Skip to content

Commit

Permalink
Allow configurable Atlas output folder structure (#26)
Browse files Browse the repository at this point in the history
* Add a scheme for output atlas too

* Spotless

* unused test

* Use proper default

* thank you spotless

* code review comments
  • Loading branch information
matthieun authored and MikeGost committed Apr 9, 2018
1 parent a8a31d9 commit 266059f
Show file tree
Hide file tree
Showing 12 changed files with 14,359 additions and 28 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package org.openstreetmap.atlas.generator;

import java.io.IOException;

import org.apache.hadoop.fs.Path;
import org.junit.Assert;
import org.junit.Test;
import org.openstreetmap.atlas.exception.CoreException;
import org.openstreetmap.atlas.generator.tools.streaming.ResourceFileSystem;
import org.openstreetmap.atlas.streaming.resource.InputStreamResource;
import org.openstreetmap.atlas.utilities.collections.StringList;

/**
* @author matthieun
*/
public class AtlasGeneratorIntegrationTest
{
public static final String BOUNDARY = "resource://test/boundaries/DMA.txt";
public static final String TREE_13 = "resource://test/sharding/tree-6-13-100000.txt";
public static final String TREE_14 = "resource://test/sharding/tree-6-14-100000.txt";
public static final String PBF = "resource://test/pbf";
public static final String PBF_233 = PBF + "/9/9-168-233.osm.pbf";
public static final String PBF_234 = PBF + "/9/9-168-234.osm.pbf";
public static final String OUTPUT = "resource://test/output";
public static final String ATLAS_OUTPUT = "resource://test/atlas/DMA";

static
{
addResource(PBF_233, "DMA_cutout.osm.pbf");
addResource(PBF_234, "DMA_cutout.osm.pbf");
addResource(TREE_13, "tree-6-13-100000.txt");
addResource(TREE_14, "tree-6-14-100000.txt");
addResource(BOUNDARY, "DMA.txt");
}

private static void addResource(final String path, final String name)
{
ResourceFileSystem.addResource(path, new InputStreamResource(
() -> AtlasGeneratorIntegrationTest.class.getResourceAsStream(name)));
}

@Test
public void testAtlasGeneration()
{
final StringList arguments = new StringList();
arguments.add("-master=local");
arguments.add("-output=" + OUTPUT);
arguments.add("-startedFolder=resource://test/started");
arguments.add("-countries=DMA");
arguments.add("-countryShapes=" + BOUNDARY);
arguments.add("-pbfs=" + PBF);
arguments.add("-pbfSharding=dynamic@" + TREE_14);
arguments.add("-pbfScheme=zz/zz-xx-yy.osm.pbf");
arguments.add("-atlasScheme=zz/");
arguments.add("-sharding=dynamic@" + TREE_13);
arguments.add(
"-sparkOptions=fs.resource.impl=" + ResourceFileSystem.class.getCanonicalName());

final String[] args = new String[arguments.size()];
for (int i = 0; i < arguments.size(); i++)
{
args[i] = arguments.get(i);
}

ResourceFileSystem.printContents();
new AtlasGenerator().runWithoutQuitting(args);
ResourceFileSystem.printContents();

try (ResourceFileSystem resourceFileSystem = new ResourceFileSystem())
{
Assert.assertTrue(
resourceFileSystem.exists(new Path(ATLAS_OUTPUT + "/9/DMA_9-168-233.atlas")));
Assert.assertTrue(
resourceFileSystem.exists(new Path(ATLAS_OUTPUT + "/9/DMA_9-168-234.atlas")));
}
catch (IllegalArgumentException | IOException e)
{
throw new CoreException("Unable to find output Atlas files.", e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DMA||MULTIPOLYGON (((-61.3490238 15.0074943, -61.3702452 15.0074207, -61.4005587 15.0115114, -61.4232934 15.0170995, -61.4499481 15.0267559, -61.4813509 15.0421834, -61.5102395 15.0630642, -61.5388572 15.0931927, -61.5609937 15.1286379, -61.573888 15.1626946, -61.5802577 15.1983142, -61.5802519 15.2182578, -61.5896846 15.2426108, -61.5979214 15.2828215, -61.6116448 15.3052907, -61.6237522 15.3347959, -61.6350822 15.3524338, -61.6486274 15.3851843, -61.6521867 15.3997221, -61.6602503 15.4157614, -61.6719498 15.4490992, -61.6756242 15.4722623, -61.6819285 15.4920564, -61.6869184 15.5259787, -61.6832167 15.6182678, -61.6798235 15.644411, -61.4425 15.7872222, -61.3236111 15.7341667, -61.1383333 15.7016667, -61.1204868 15.6804104, -61.116797 15.6720689, -61.1106596 15.6656506, -61.0968189 15.6460962, -61.083421 15.6300299, -61.0657932 15.5989765, -61.0607846 15.5846292, -61.0567601 15.5769338, -61.055243 15.5742449, -61.0436767 15.541066, -61.036387 15.4952162, -61.0344085 15.4615026, -61.037269 15.4351561, -61.0329895 15.3942999, -61.0364022 15.3589037, -61.0354401 15.3544219, -61.0346416 15.3150813, -61.0368515 15.3029501, -61.0373217 15.2768091, -61.0420306 15.2476706, -61.0442149 15.2375483, -61.0562047 15.2043716, -61.0609591 15.196169, -61.0666737 15.1779782, -61.083955 15.144618, -61.1069788 15.1151873, -61.1349236 15.0905183, -61.1589127 15.0792459, -61.2830556 15.0308333, -61.3490238 15.0074943)))#
Binary file not shown.
Loading

0 comments on commit 266059f

Please sign in to comment.