Merge pull request #757 from conveyal/dev

Prepare for v6.5 release

.github/workflows/cypress-integration.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           repository: conveyal/analysis-ui
-          ref: a869cd11919343a163e110e812b5d27f3a4ad4c8
+          ref: 1701bd9aea859a4714bc3f35f5bcf767a3256a64
           path: ui
       - uses: actions/checkout@v2
         with:

README.md

@@ -1,15 +1,12 @@
 # Conveyal R5 Routing Engine
 
 ## R5: Rapid Realistic Routing on Real-world and Reimagined networks
+R5 is the routing engine for [Conveyal](https://www.conveyal.com/learn), a web-based system that allows users to create transportation scenarios and evaluate them in terms of cumulative opportunities accessibility indicators. See the [Conveyal user manual](https://docs.conveyal.com/) for more information.
 
-R5 is Conveyal's routing engine for multimodal (transit/bike/walk/car) networks, with a particular focus on public transit. It is intended primarily for analysis applications (one-to-many trees, travel time matrices, and cumulative opportunities accessibility indicators). 
-
-We refer to the routing method as "realistic" because it works by planning many trips at different departure times in a time window, which better reflects how people use transportation system than planning a single trip at an exact departure time. R5 handles both scheduled public transit and headway-based lines, using novel methods to characterize variation and uncertainty in travel times.
+We refer to the routing method as "realistic" because it works by planning door-to-door trips at many different departure times in a time window, which better reflects how people use transportation systems than planning a single trip at an exact departure time. R5 handles both scheduled public transit and headway-based lines, using novel methods to characterize variation and uncertainty in travel times. It is designed for one-to-many and many-to-many travel-time calculations used in access indicators, offering substantially better performance than repeated calls to older tools that provide one-to-one routing results. For a comparison with OpenTripPlanner, see [this background](http://docs.opentripplanner.org/en/latest/Version-Comparison/#commentary-on-otp1-features-removed-from-otp2).
 
 We say "Real-world and Reimagined" networks because R5's networks are built from widely available open OSM and GTFS data describing baseline transportation systems, but R5 includes a system for applying light-weight patches to those networks for immediate, interactive scenario comparison.
 
-R5 is a core component of [Conveyal Analysis](https://www.conveyal.com/learn), which allows users to create transportation scenarios and evaluate them in terms of cumulative opportunities accessibility indicators. See the [methodology section](https://docs.conveyal.com/analysis/methodology) of the [Conveyal user manual](https://docs.conveyal.com/) for more information.
-
 **Please note** that the Conveyal team does not provide technical support for third-party deployments of its analysis platform. We provide paid subscriptions to a cloud-based deployment of this system, which performs these complex calculations hundreds of times faster using a compute cluster. This project is open source primarily to ensure transparency and reproducibility in public planning and decision making processes, and in hopes that it may help researchers, students, and potential collaborators to understand and build upon our methodology.
 
 ## Methodology

analysis.properties.template

@@ -9,11 +9,6 @@ database-uri=mongodb://localhost
 # The name of the database in the Mongo instance.
 database-name=analysis
 
-# The URL where the frontend is hosted.
-# In production this should point to a cached CDN for speed. e.g. https://d1uqjuy3laovxb.cloudfront.net
-# In staging this should be the underlying S3 URL so files are not cached and you see the most recent deployment.
-frontend-url=https://localhost
-
 # The S3 bucket where we can find tiles of the entire US census, built with Conveyal seamless-census.
 seamless-census-bucket=lodes-data-2014
 seamless-census-region=us-east-1
@@ -28,6 +23,11 @@ aws-region=eu-west-1
 # The port on which the server will listen for connections from clients and workers.
 server-port=7070
 
+# The origin where the frontend is hosted. When running locally, this will generally be http://localhost:3000.
+# It should be relatively safe to set this to * (allowing requests from anywhere) when authentication is enabled.
+# This increases attack surface though, so it is preferable to set this to the specific origin where the UI is hosted.
+access-control-allow-origin=http://localhost:3000
+
 # A temporary location to store scratch files. The path can be absolute or relative.
 # This allows you to locate temporary storage on an extra drive in case your main drive does not have enough space.
 # local-cache=/home/ec2-user/cache

build.gradle

@@ -125,6 +125,11 @@ repositories {
     maven { url 'https://nexus.axiomalaska.com/nexus/content/repositories/public-releases' }
 }
 
+// Exclude all JUnit 4 transitive dependencies - IntelliJ bug causes it to think we're using Junit 4 instead of 5.
+configurations.all {
+    exclude group: "junit", module: "junit"
+}
+
 dependencies {
     // Provides our logging API
     implementation 'org.slf4j:slf4j-api:1.7.30'
@@ -154,13 +159,14 @@ dependencies {
     implementation 'com.beust:jcommander:1.30'
 
     // GeoTools provides GIS functionality on top of JTS topology suite.
-    def geotoolsVersion = '21.2'
+    def geotoolsVersion = '25.2'
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-main'
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-opengis'
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-referencing'
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-shapefile'
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-coverage'
-    implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojson'
+    implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geojsondatastore'
+    implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geopkg'
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-geotiff'
     // Provides the EPSG coordinate reference system catalog as an HSQL database.
     implementation group: 'org.geotools', version: geotoolsVersion, name: 'gt-epsg-hsql'
@@ -201,10 +207,6 @@ dependencies {
     // Now used only for Seamless Census TODO eliminate this final AWS dependency
     implementation 'com.amazonaws:aws-java-sdk-s3:1.11.341'
 
-    // Old version of GraphQL-Java used by legacy gtfs-api embedded in analysis-backend.
-    // TODO eliminate GraphQL in future API revisions
-    implementation 'com.graphql-java:graphql-java:2.1.0'
-
     // Commons Math gives us FastMath, MersenneTwister, and low-discrepancy vector generators.
     implementation 'org.apache.commons:commons-math3:3.0'
 

src/main/java/com/conveyal/analysis/AnalysisServerException.java

@@ -1,12 +1,9 @@
 package com.conveyal.analysis;
 
 import com.conveyal.r5.util.ExceptionUtils;
-import graphql.GraphQLError;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.List;
-
 public class AnalysisServerException extends RuntimeException {
     private static final Logger LOG = LoggerFactory.getLogger(AnalysisServerException.class);
 
@@ -40,17 +37,6 @@ public static AnalysisServerException forbidden(String message) {
         return new AnalysisServerException(Type.FORBIDDEN, message, 403);
     }
 
-    public static AnalysisServerException graphQL(List<GraphQLError> errors) {
-        return new AnalysisServerException(
-                Type.GRAPHQL,
-                errors
-                    .stream()
-                    .map(e -> e.getMessage())
-                    .reduce("", (a, b) -> a + " " + b),
-                400
-        );
-    }
-
     public static AnalysisServerException nonce() {
         return new AnalysisServerException(Type.NONCE, "The data you attempted to change is out of date and could not be " +
                 "updated. This project may be open by another user or in another browser tab.", 400);
@@ -60,6 +46,8 @@ public static AnalysisServerException notFound(String message) {
         return new AnalysisServerException(Type.NOT_FOUND, message, 404);
     }
 
+    // Note that there is a naming mistake in the HTTP codes. 401 "unauthorized" actually means "unauthenticated".
+    // 403 "forbidden" is what is usually referred to as "unauthorized" in other contexts.
     public static AnalysisServerException unauthorized(String message) {
         return new AnalysisServerException(Type.UNAUTHORIZED, message, 401);
     }

src/main/java/com/conveyal/analysis/BackendConfig.java

@@ -4,7 +4,6 @@
 import com.conveyal.analysis.components.LocalWorkerLauncher;
 import com.conveyal.analysis.components.TaskScheduler;
 import com.conveyal.analysis.components.broker.Broker;
-import com.conveyal.analysis.controllers.OpportunityDatasetController;
 import com.conveyal.analysis.grids.SeamlessCensusGridExtractor;
 import com.conveyal.analysis.persistence.AnalysisDB;
 import com.conveyal.file.LocalFileStorage;
@@ -36,6 +35,7 @@ public class BackendConfig extends ConfigBase implements
     private final String databaseUri;
     private final String localCacheDirectory;
     private final int serverPort;
+    private final String allowOrigin;
     private final String seamlessCensusBucket;
     private final String seamlessCensusRegion;
     private final int lightThreads;
@@ -63,6 +63,7 @@ protected BackendConfig (Properties properties) {
         localCacheDirectory = strProp("local-cache");
         serverPort = intProp("server-port");
         offline = boolProp("offline");
+        allowOrigin = strProp("access-control-allow-origin");
         seamlessCensusBucket = strProp("seamless-census-bucket");
         seamlessCensusRegion = strProp("seamless-census-region");
         lightThreads = intProp("light-threads");
@@ -81,6 +82,7 @@ protected BackendConfig (Properties properties) {
     @Override public String  databaseName()         { return databaseName; }
     @Override public String  localCacheDirectory()  { return localCacheDirectory;}
     @Override public boolean testTaskRedelivery()   { return testTaskRedelivery; }
+    @Override public String  allowOrigin()          { return allowOrigin; }
     @Override public String  seamlessCensusRegion() { return seamlessCensusRegion; }
     @Override public String  seamlessCensusBucket() { return seamlessCensusBucket; }
     @Override public int     serverPort()           { return serverPort; }

src/main/java/com/conveyal/analysis/BackendMain.java

@@ -3,7 +3,6 @@
 import com.conveyal.analysis.components.BackendComponents;
 import com.conveyal.analysis.components.LocalBackendComponents;
 import com.conveyal.analysis.persistence.Persistence;
-import com.conveyal.gtfs.api.ApiMain;
 import com.conveyal.r5.SoftwareVersion;
 import com.conveyal.r5.analyst.PointSetCache;
 import com.conveyal.r5.analyst.WorkerCategory;
@@ -46,20 +45,20 @@ private static void startServerInternal (BackendComponents components, TaskActio
         // TODO remove the static ApiMain abstraction layer. We do not use it anywhere but in handling GraphQL queries.
         // TODO we could move this to something like BackendComponents.initialize()
         Persistence.initializeStatically(components.config);
-        ApiMain.initialize(components.gtfsCache);
         PointSetCache.initializeStatically(components.fileStorage);
 
         // TODO handle this via components without explicit "if (offline)"
         if (components.config.offline()) {
             LOG.info("Running in OFFLINE mode.");
             LOG.info("Pre-starting local cluster of Analysis workers...");
+            // WorkerCategory(null, null) means a worker is not on any network, and is waiting to be assigned one.
             components.workerLauncher.launch(new WorkerCategory(null, null), null, 1, 0);
         }
 
         LOG.info("Conveyal Analysis server is ready.");
         for (TaskAction taskAction : postStartupTasks) {
             components.taskScheduler.enqueue(
-                Task.create(Runnable.class.getSimpleName()).setHeavy(true).forUser("SYSTEM").withAction(taskAction)
+                Task.create(taskAction.getClass().getSimpleName()).setHeavy(true).forUser("SYSTEM").withAction(taskAction)
             );
         }
 

src/main/java/com/conveyal/analysis/SelectingGridReducer.java

@@ -20,7 +20,6 @@
  * When storing bootstrap replications of travel time, we also store the point estimate (using all Monte Carlo draws
  * equally weighted) as the first value, so a SelectingGridReducer(0) can be used to retrieve the point estimate.
  *
- * This class is not referenced within R5, but is used by the Analysis front end.
  */
 public class SelectingGridReducer {
 
@@ -64,7 +63,7 @@ public Grid compute (InputStream rawInput) throws IOException {
         // median travel time.
         int nSamples = input.readInt();
 
-        Grid outputGrid = new Grid(zoom, width, height, north, west);
+        Grid outputGrid = new Grid(west, north, width, height, zoom);
 
         int[] valuesThisOrigin = new int[nSamples];
 

src/main/java/com/conveyal/analysis/UserPermissions.java

@@ -1,5 +1,9 @@
 package com.conveyal.analysis;
 
+import spark.Request;
+
+import static com.conveyal.analysis.components.HttpApi.USER_PERMISSIONS_ATTRIBUTE;
+
 /**
  * Groups together all information about what a user is allowed to do.
  * Currently all such information is known from the group ID.
@@ -19,6 +23,16 @@ public UserPermissions (String email, boolean admin, String accessGroup) {
         this.accessGroup = accessGroup;
     }
 
+    /**
+     * From an HTTP request object, extract a strongly typed UserPermissions object containing the user's email and
+     * access group. This should be used almost everywhere instead of String email and accessGroup variables. Use this
+     * method to encapsulate all calls to req.attribute(String) because those calls are not typesafe (they cast an Object
+     * to whatever type seems appropriate in the context, or is supplied by the "req.<T>attribute(String)" syntax).
+     */
+    public static UserPermissions from (Request req) {
+        return req.attribute(USER_PERMISSIONS_ATTRIBUTE);
+    }
+
     @Override
     public String toString () {
         return "UserPermissions{" +

src/main/java/com/conveyal/analysis/components/BackendComponents.java

@@ -7,14 +7,12 @@
 import com.conveyal.analysis.controllers.BrokerController;
 import com.conveyal.analysis.controllers.BundleController;
 import com.conveyal.analysis.controllers.FileStorageController;
-import com.conveyal.analysis.controllers.GTFSGraphQLController;
+import com.conveyal.analysis.controllers.GtfsController;
 import com.conveyal.analysis.controllers.GtfsTileController;
 import com.conveyal.analysis.controllers.HttpController;
-import com.conveyal.analysis.controllers.ModificationController;
 import com.conveyal.analysis.controllers.OpportunityDatasetController;
-import com.conveyal.analysis.controllers.ProjectController;
 import com.conveyal.analysis.controllers.RegionalAnalysisController;
-import com.conveyal.analysis.controllers.TimetableController;
+import com.conveyal.analysis.controllers.DataSourceController;
 import com.conveyal.analysis.controllers.UserActivityController;
 import com.conveyal.analysis.grids.SeamlessCensusGridExtractor;
 import com.conveyal.analysis.persistence.AnalysisDB;
@@ -86,23 +84,21 @@ public List<HttpController> standardHttpControllers () {
         return Lists.newArrayList(
                 // These handlers are at paths beginning with /api
                 // and therefore subject to authentication and authorization.
-                new ModificationController(),
-                new ProjectController(),
-                new GTFSGraphQLController(gtfsCache),
+                new GtfsController(gtfsCache),
                 new BundleController(this),
-                new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor),
+                new OpportunityDatasetController(fileStorage, taskScheduler, censusExtractor, database),
                 new RegionalAnalysisController(broker, fileStorage),
-                new AggregationAreaController(fileStorage),
-                new TimetableController(),
                 new FileStorageController(fileStorage, database),
+                new AggregationAreaController(fileStorage, database, taskScheduler),
                 // This broker controller registers at least one handler at URL paths beginning with /internal, which
                 // is exempted from authentication and authorization, but should be hidden from the world
                 // outside the cluster by the reverse proxy. Perhaps we should serve /internal on a separate
                 // port so they can't be accidentally exposed by the reverse proxy. It could even be a separate
                 // InternalHttpApi component with its own spark service, renaming this ExternalHttpApi.
                 new BrokerController(broker, eventBus),
                 new UserActivityController(taskScheduler),
-                new GtfsTileController(gtfsCache)
+                new GtfsTileController(gtfsCache),
+                new DataSourceController(fileStorage, database, taskScheduler, censusExtractor)
         );
     }