Merge pull request #136 from matsim-scenarios/6.x

6.4 Release

.github/workflows/build.yaml

@@ -11,8 +11,8 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
         with:
           java-version: 21
           architecture: x64
@@ -29,8 +29,8 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
         with:
           java-version: 21
           architecture: x64
@@ -54,8 +54,8 @@ jobs:
         java: [21]
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           architecture: x64
@@ -64,7 +64,7 @@ jobs:
 
       - name: Test
         run: mvn --batch-mode --update-snapshots test -Dmatsim.preferLocalDtds=true -Dmaven.javadoc.skip -e
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: test-coverage
           path: target/site/jacoco/
@@ -77,8 +77,8 @@ jobs:
 
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
         with:
           java-version: 21
           architecture: x64
@@ -88,7 +88,7 @@ jobs:
       - name: Package
         run: mvn --batch-mode --update-snapshots package -DskipTests -Dmatsim.preferLocalDtds=true -Dmaven.javadoc.skip -e
       - run: mkdir staging && cp *.jar staging
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: Package
           path: staging

CHANGELOG.md

@@ -2,6 +2,34 @@
 
 All notable changes to this project will be documented in this file. 
 
+### 6.4 (2025-01)
+- Improved facility locations
+  - OSM tag filtering has been improved to reduce number of wrongly assigned facility types
+- Update work location assignment within Berlin
+  - Work locations now use weighted sampling during location choice
+  - The weight is based on an attraction factor and zone specific probabilities determined from survey data
+- Updated GTFS schedule
+  - The schedule is now based on the 2024-11-19
+  - Note that there are major changes how stops and links between them are created
+    - Stops having the same gtfs parent_id and route types are merged together, allowing agents to find better PT connections
+    - The PT network is created with loop links (on each PT stop) instead of duplicating stops, which also improves connections
+    - The Berlin Ringbahn is manually adjusted so that each train drives multiple loops the whole day
+- New income calculation
+  - In previous versions income was used directly as household income from the survey data
+  - Now, the income is calculated as personal equivalent income, which is the household income divided by equivalent household size
+    - See https://en.wikipedia.org/wiki/Equivalisation
+  - Corresponding attributes have been added to the population file
+- Bike mode updated and recalibrated
+  - Bike is now routed on the network, which results in more realistic travel distances
+  - Bikes are not simulated on the network, and no link events generated yet. This will likely be added in future versions.
+  - The road network includes bike infrastructure and corresponding attributes
+  - The bike infrastructure is not fully complete yet, and has to be carefully evaluated first for bike centric studies
+  - The avg. bike speed has been set to match SrV2018 survey data (~10.3 km/h)
+- New dashboards 
+  - PT Transit viewer
+  - Emissions
+  - Noise 
+
 ### 6.3 (2024-07)
 - Include additional trip analysis and updated dashboard
   - Mode share is now analyzed by age, income, employment, economic_status

Makefile

@@ -1,7 +1,7 @@
 
 
 JAR := matsim-berlin-*.jar
-V := v6.3
+V := v6.4
 CRS := EPSG:25832
 
 p := input/$V
@@ -38,6 +38,12 @@ input/facilities.gpkg: input/brandenburg.osm.pbf
 	 --input $<\
 	 --output $@
 
+# This facility file is using an older version that matches the reference visitations more closely
+input/ref_facilities.gpkg: input/facilities.osm.pbf
+	$(sc) prepare facility-shp\
+	 --activity-mapping input/activity_mapping.json\
+	 --input $<\
+	 --output $@
 
 input/PLR_2013_2020.csv:
 	curl https://instantatlas.statistik-berlin-brandenburg.de/instantatlas/interaktivekarten/kommunalatlas/Kommunalatlas.zip --insecure -o atlas.zip
@@ -53,8 +59,9 @@ $(berlin)/input/shp/Planungsraum_EPSG_25833.shp:
 
 input/network.osm: input/brandenburg.osm.pbf
 
+	# Detailed network includes bikes as well
 	$(osmosis) --rb file=$<\
-	 --tf accept-ways highway=motorway,motorway_link,trunk,trunk_link,primary,primary_link,secondary_link,secondary,tertiary,motorway_junction,residential,living_street,unclassified\
+	 --tf accept-ways bicycle=designated highway=motorway,motorway_link,trunk,trunk_link,primary,primary_link,secondary_link,secondary,tertiary,motorway_junction,residential,living_street,unclassified,cycleway\
 	 --bounding-polygon file="$p/area/area.poly"\
 	 --used-node --wb input/network-detailed.osm.pbf
 
@@ -81,17 +88,17 @@ input/sumo.net.xml: input/network.osm
 	 --no-internal-links --keep-edges.by-vclass passenger,truck,bicycle\
 	 --remove-edges.by-vclass hov,tram,rail,rail_urban,rail_fast,pedestrian\
 	 --output.original-names --output.street-names\
-	 --osm.lane-access true	--osm.bike-access true\
+	 --osm.lane-access false --osm.bike-access false\
 	 --osm.all-attributes\
-	 --osm.extra-attributes tunnel,highway,traffic_sign,bus:lanes,bus:lanes:forward,bus:lanes:backward,cycleway,cycleway:right,cycleway:left\
+	 --osm.extra-attributes smoothness,surface,crossing,tunnel,traffic_sign,bus:lanes,bus:lanes:forward,bus:lanes:backward,cycleway,cycleway:right,cycleway:left,bicycle\
 	 --proj "+proj=utm +zone=32 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs"\
 	 --osm-files $< -o=$@
 
 
 $p/berlin-$V-network.xml.gz: input/sumo.net.xml
 	$(sc) prepare network-from-sumo $< --target-crs $(CRS) --lane-restrictions REDUCE_CAR_LANES --output $@
 
-	$(sc) prepare clean-network $@  --output $@ --modes car,ride,truck --remove-turn-restrictions
+	$(sc) prepare clean-network $@  --output $@ --modes car,bike,ride,truck --remove-turn-restrictions
 
 	$(sc) prepare reproject-network\
 	 --input $@	--output $@\
@@ -114,10 +121,23 @@ $p/berlin-$V-network.xml.gz: input/sumo.net.xml
 
 $p/berlin-$V-network-with-pt.xml.gz: $p/berlin-$V-network.xml.gz
 	$(sc) prepare transit-from-gtfs --network $< --output=$p\
-	 --name berlin-$V --date "2023-06-07" --target-crs $(CRS) \
-	 $(germany)/gtfs/complete-pt-2023-06-06.zip\
+	 --name berlin-$V --date "2024-11-19" --target-crs $(CRS) \
+	 $(germany)/gtfs/complete-pt-2024-10-27.zip\
+	 --copy-late-early\
+	 --transform-stops org.matsim.prepare.pt.CorrectStopLocations\
+	 --transform-routes org.matsim.prepare.pt.CorrectRouteTypes\
+	 --transform-schedule org.matsim.application.prepare.pt.AdjustSameDepartureTimes\
+	 --pseudo-network withLoopLinks\
+	 --merge-stops mergeToParentAndRouteTypes\
 	 --shp $p/pt-area/pt-area.shp
 
+	$(sc) prepare endless-circle-line\
+ 	  --network $p/berlin-$V-network-with-pt.xml.gz\
+ 	  --transit-schedule $p/berlin-$V-transitSchedule.xml.gz\
+ 	  --transit-vehicles $p/berlin-$V-transitVehicles.xml.gz\
+ 	  --output-transit-schedule $p/berlin-$V-transitSchedule.xml.gz\
+	  --output-transit-vehicles $p/berlin-$V-transitVehicles.xml.gz
+
 $p/berlin-$V-counts-vmz.xml.gz: $p/berlin-$V-network.xml.gz
 	$(sc) prepare counts-from-vmz\
 	 --excel ../shared-svn/projects/matsim-berlin/berlin-v5.5/original_data/vmz_counts_2018/Datenexport_2018_TU_Berlin.xlsx\
@@ -128,10 +148,20 @@ $p/berlin-$V-counts-vmz.xml.gz: $p/berlin-$V-network.xml.gz
 	 --target-crs $(CRS)\
 	 --counts-mapping input/counts_mapping.csv
 
-$p/berlin-$V-facilities.xml.gz: $p/berlin-$V-network.xml.gz input/facilities.gpkg
+$p/berlin-$V-facilities.xml.gz: $p/berlin-$V-network.xml.gz input/facilities.gpkg $(berlin)/input/shp/Planungsraum_EPSG_25833.shp
 	$(sc) prepare facilities --network $< --shp $(word 2,$^)\
+	 --facility-mapping input/facility_mapping.json\
+	 --zones-shp $(word 3,$^)\
 	 --output $@
 
+$p/berlin-only-$V-100pct.plans.xml.gz: input/PLR_2013_2020.csv $(berlin)/input/shp/Planungsraum_EPSG_25833.shp input/facilities.gpkg
+	$(sc) prepare berlin-population\
+		--input $<\
+		--sample 1.0\
+		--shp $(word 2,$^) --shp-crs EPSG:25833\
+		--facilities $(word 3,$^) --facilities-attr resident\
+		--output $@
+
 $p/berlin-only-$V-25pct.plans.xml.gz: input/PLR_2013_2020.csv $(berlin)/input/shp/Planungsraum_EPSG_25833.shp input/facilities.gpkg
 	$(sc) prepare berlin-population\
 		--input $<\
@@ -174,6 +204,7 @@ $p/berlin-initial-$V-25pct.plans.xml.gz: $p/berlin-activities-$V-25pct.plans.xml
 	 --network $(word 3,$^)\
 	 --shp $(germany)/vg5000/vg5000_ebenen_0101/VG5000_GEM.shp\
 	 --commuter $(germany)/regionalstatistik/commuter.csv\
+	 --berlin-commuter input/berlin-work-commuter.csv
 
 	# For debugging and visualization
 	$(sc) prepare downsample-population $@\
@@ -269,11 +300,15 @@ $p/berlin-$V-25pct.plans_cadyts.xml.gz:
 
 # These depend on the output of optimization runs
 $p/berlin-$V-25pct.plans-initial.xml.gz: $p/berlin-$V-facilities.xml.gz $p/berlin-$V-network.xml.gz $p/berlin-longHaulFreight-$V-25pct.plans.xml.gz
-	$(sc) prepare filter-relevant-agents\
-	 --input $p/berlin-$V-25pct.plans_log_error.xml.gz --output $@\
-	 --shp input/$V/area/area.shp\
+	$(sc) prepare scenario-cutout\
+	 --population $p/berlin-$V-25pct.plans_cadyts.xml.gz\
 	 --facilities $<\
-	 --network $(word 2,$^)
+	 --network $(word 2,$^)\
+	 --output-population $@\
+	 --output-network $p/network-cutout.xml.gz\
+	 --output-facilities $p/facilities-cutout.xml.gz\
+	 --input-crs $(CRS)\
+	 --shp input/$V/area/area.shp
 
 	$(sc) prepare split-activity-types-duration\
  	 --exclude commercial_start,commercial_end,freight_start,freight_end\
@@ -294,13 +329,18 @@ $p/berlin-$V-25pct.plans-initial.xml.gz: $p/berlin-$V-facilities.xml.gz $p/berli
 
 $p/berlin-$V-10pct.plans.xml.gz:
 	$(sc) prepare clean-population\
-	 --plans mode-choice-10pct-default-v2/runs/008/008.output_plans.xml.gz\
+	 --plans mode-choice-10pct-baseline/runs/008/008.output_plans.xml.gz\
 	 --remove-unselected-plans\
 	 --output $@
 
 	$(sc) prepare downsample-population $@\
 		--sample-size 0.1\
-		--samples 0.03 0.01 0.001\
+		--samples 0.01 0.001\
+
+	$(sc) prepare clean-population\
+	 	--plans choice-experiments/baseline/runs/008/008.output_plans.xml.gz\
+	 	--remove-unselected-plans\
+	 	--output $(subst 10pct,3pct,$@)
 
 
 $p/berlin-$V.drt-by-rndLocations-10000vehicles-4seats.xml.gz: $p/berlin-$V-network.xml.gz
@@ -311,6 +351,14 @@ $p/berlin-$V.drt-by-rndLocations-10000vehicles-4seats.xml.gz: $p/berlin-$V-netwo
 	 --vehicles 10000\
 	 --seats 4
 
+	$(sc) prepare create-drt-vehicles\
+	 --network $<\
+	 --shp "https://svn.vsp.tu-berlin.de/repos/public-svn/matsim/scenarios/countries/de/berlin/berlin-$V/input/shp/berlin_inner_city.gpkg"\
+	 --output $p/berlin-$V.\
+	 --vehicles 500\
+	 --seats 4
+
+
 prepare-calibration: $p/berlin-cadyts-input-$V-25pct.plans.xml.gz $p/berlin-$V-network-with-pt.xml.gz $p/berlin-$V-counts-vmz.xml.gz
 	echo "Done"
 

README.md

@@ -1,9 +1,10 @@
 [![Build Status](https://github.com/matsim-scenarios/matsim-berlin/actions/workflows/build.yaml/badge.svg?branch=main)](https://github.com/matsim-scenarios/matsim-berlin/actions/workflows/build.yaml)
 ![license](https://img.shields.io/github/license/matsim-scenarios/matsim-berlin.svg)
-![JDK](https://img.shields.io/badge/JDK-17+-green.svg)
+![JDK](https://img.shields.io/badge/JDK-21+-green.svg)
 
 # The MATSim Open Berlin Scenario
-![Berlin MATSim network and agents)](scenarios/berlin-v5.5-10pct/visualization-berlin.png "Berlin MATSim network and agents")
+
+![MATSim Open Berlin network, vehicles and activities](input/visualization.png "MATSim Open Berlin")
 
 ## About this project
 
@@ -13,9 +14,9 @@ This repository provides an open MATSim transport model for Berlin, provided by
 
 Currently, there are multiple versions of the MATSim Open Berlin model:
 
-### 10pct scenario (`input/v6.3`)
+### 10pct scenario (`input/v6.4`)
 
-This scenario contains both 10pct and 1pct sample of the Greater Berlin population; road capacities are accordingly reduced. The scenario is calibrated taking into consideration the traffic counts, modal split and mode-specific trip distance distributions.
+This scenario contains a 10pct, 3pct and 1pct sample of the Greater Berlin population; road capacities are accordingly reduced. The scenario is calibrated taking into consideration the traffic counts, modal split and mode-specific trip distance distributions.
 
 
 ## Licenses
@@ -38,14 +39,14 @@ Handling of large files within git is not without problems (git lfs files are no
 
 1. Open SimWrapper at https://simwrapper.github.io/site/ .
 1. Select Explore Data Sources -> VSP Public-SVN .
-1. Navigate to de -> berlin -> berlin-v6.3 (https://simwrapper.github.io/site/public/de/berlin/berlin-v6.3).
+1. Navigate to de -> berlin -> berlin-v6.4 -> output (https://simwrapper.github.io/site/public/de/berlin/berlin-v6.4/output/berlin-v6.4-10pct/).
 1. Enjoy the dashboards and visualizations.
 
 ### Run VIA on output files
 
 1. Get VIA from https://www.simunto.com/via/.  (There is a free license for a small number of agents; that will probably work but only display a small number of vehicles/agents.)
 1. Go to https://svn.vsp.tu-berlin.de/repos/public-svn/matsim/scenarios/countries/de/berlin/ .
-1. Decide for a scenario that you find interesting (e.g. `berlin-v6.3`) and go into that directory.
+1. Decide for a scenario that you find interesting (e.g. `berlin-v6.4`) and go into that directory.
 1. Inside there, look for an `output-*` directory that you find interesting and go into that directory.
 1. Download `*.output_network.xml.gz` and `*.output_events.xml.gz`.  Best make sure that they do not uncompress, e.g. by "Download linked file as ...".
 1. Get these files into VIA.  This can be achieved in various ways; one is to open VIA and then drag the files from a file browser into VIA.
@@ -98,7 +99,7 @@ This will download all necessary dependencies (it might take a while the first t
 java -jar [FILENAME].jar
 ``
 1. A simple GUI should open.
-1. In the GUI, click on the "Choose" button for configuration file.  Navigate to one of the `scenario` directories and load one of the configuration files.
+1. In the GUI, click on the "Choose" button for configuration file.  Navigate to one of the `input` directories and load one of the configuration files.
 1. Increase memory in the GUI.
 1. Press the "Start MATSim" button.  This should run MATSim.  Note that MATSim accepts URLs as filenames in its config, so while the config files are part of the git repo, running them will pull additional material from our server.
 1. "Open" the output directory.  You can drag files into VIA as was already done above.
@@ -112,7 +113,7 @@ java -jar [FILENAME].jar
 
 1. Set up the project in your IDE.
 1. Make sure the project is configured as maven project.
-1. Run the JAVA class `src/main/java/org/matsim/run/RunOpenBerlinScenario.java` or `src/main/java/org/matsim/gui/RunOpenBerlinScenarioGUI.java`.
+1. Run the JAVA class `src/main/java/org/matsim/run/RunOpenBerlinScenario.java`.
 1. "Open" the output directory.  You can drag files into VIA as was already done above.
 1. Edit the config file or adjust the run class. Re-run MATSim.
 </details>

input/activity_mapping.json

@@ -1,7 +1,7 @@
 {
   "building": {
     "hotel": ["work"],
-    "commercial": ["shop", "work", "delivery"],
+    "commercial": ["work", "delivery"],
     "retail": ["shop", "work", "delivery"],
     "supermarket": ["shop", "shop_daily", "dining", "work", "delivery"],
     "industrial": ["work", "delivery"],
@@ -26,7 +26,14 @@
     "college": ["edu_higher", "work"],
     "sports_hall": ["leisure", "work"],
     "stadium": ["leisure", "work"],
-    "apartments": ["resident"]
+    "garage": ["parking"],
+    "apartments": ["resident"],
+    "house": ["resident"],
+    "detached": ["resident"],
+    "semidetached_house": ["resident"],
+    "bungalow": ["resident"],
+    "terrace": ["resident"],
+    "residential": ["resident"]
   },
   "amenity": {
     "bar": ["leisure", "work", "delivery", "dining"],
@@ -71,11 +78,12 @@
     "post_depot": ["p_business", "work"],
     "post_office": ["p_business", "work"],
     "prison": ["p_business", "work"],
-    "townhall": ["p_business", "work"]
+    "townhall": ["p_business", "work"],
+    "parking": ["parking"]
   },
   "landuse": {
-    "commercial": ["shop", "work", "delivery"],
-    "industrial": ["shop", "work", "delivery", "depot"],
+    "commercial": ["work", "delivery"],
+    "industrial": ["work", "delivery", "depot"],
     "retail": ["shop", "work", "delivery"],
     "depot": ["depot"],
     "port": ["depot"],

input/facility_mapping.json

@@ -0,0 +1,14 @@
+{
+  "work": ["work", "work_business"],
+  "shop": ["shop_other"],
+  "shop_daily": ["shop_other", "shop_daily"],
+  "leisure": ["leisure"],
+  "dining": ["dining"],
+  "edu_higher": ["edu_higher"],
+  "edu_prim": ["edu_primary", "edu_secondary"],
+  "edu_kiga": ["edu_kiga"],
+  "edu_other": ["edu_other"],
+  "p_business": ["personal_business", "work_business"],
+  "medical": ["personal_business"],
+  "religious": ["personal_business"]
+}