From bb89a407d4cc054e314e3b7a92f46c845fe7b436 Mon Sep 17 00:00:00 2001 From: Harry Wood Date: Wed, 27 Feb 2013 12:00:07 +0000 Subject: [PATCH 1/5] CSS styling Style it up! Mostly cosmetic. It also now has the Kuona title and all buttons and text are to the right of the time images (fits on small screens better) --- kuona.css | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.html | 14 ++++---------- 2 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 kuona.css diff --git a/kuona.css b/kuona.css new file mode 100644 index 0000000..eec8f2e --- /dev/null +++ b/kuona.css @@ -0,0 +1,56 @@ +body { + font-family: Verdana, Helvetica, Arial, sans-serif; + color: black; +} +#imagerytiles { + float:left; + margin-bottom:30px; + margin-right:30px; + padding:5px; + background:#EEE; +} + +#imagerytiles table input[type=image] { + display: block; +} + +#content { + padding:5px; +} +#content h1 { + font-size:3em; + margin:0px; +} +#instructions { + background:#FFE; + margin:10px; + padding:10px; +} +#instructions h2 { + font-size:1.1em; + margin:0px; +} +#instructions ul { + list-style:inside; + margin:0px; + padding:0px; +} + +#footer { + clear:both; + border-top:solid 2px #EEE; + padding:5px; + text-align:right +} +#footer a { + color:#AAA; +} +#stats { + font-size: 9pt; +} + +input[type=submit] { + padding:5px; + font-weight:BOLD; + font-size:1em; +} diff --git a/main.html b/main.html index dd5cf6b..744055e 100644 --- a/main.html +++ b/main.html @@ -8,12 +8,9 @@ #} - - + Kuona - Imagery micro-tasking for OpenStreetMap + + {% if got_tile %} @@ -52,10 +49,7 @@ {% if remaining %}

Pictures remaining: {{remaining}}

{% endif %} - {#

This application is currently running on my home computer, - on a residential Internet con­nec­tion. Lately it's being very unreliable. - If you get a connection error, just reload the page.

-

I will try to move it to an OpenStreetMap dev server soon.

#} + From 6d27cb8d396c9674ecec20be8c0dd87e3ca53a38 Mon Sep 17 00:00:00 2001 From: Harry Wood Date: Wed, 27 Feb 2013 12:24:17 +0000 Subject: [PATCH 2/5] more HTML changes in line with style managed to miss this out of my commit somehow. Hopefully that's everything --- main.html | 75 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/main.html b/main.html index 744055e..1f6b4f9 100644 --- a/main.html +++ b/main.html @@ -8,19 +8,19 @@ #} - Kuona - Imagery micro-tasking for OpenStreetMap + Kuona - Any easy way to help OpenStreetMap - {% if got_tile %} + {% if got_tile %}
-

Click on any buildings you see. Especially on large areas of them (like this).

- - -
+ + +
@@ -31,26 +31,59 @@
-
- DO Mark
  • Villages
  • Dams
- DO Not Mark
  • Roads
  • Cattle Pens
  • Fields
-
-

-
+ + +
+

Kuona

+ +
+

Mali building mapping

+

+ Click on any buildings you see. Especially on large areas of them + (like this) +

+ + DO Mark +
    +
  • Villages
    • +
  • Dams
    • +
+
+ DO Not Mark +
    +
  • Roads
    • +
  • Cattle Pens
    • +
  • Fields
    • +
+
+ +

+ + +
+ {% if confirmation %} +

Currently verifying already-seen tiles.

+ {% endif %} + {% if remaining %} +

Pictures remaining: {{remaining}}

+ {% endif %} + + {#

This application is currently running on my home computer, + on a residential Internet con­nec­tion. Lately it's being very unreliable. + If you get a connection error, just reload the page.

+

I will try to move it to an OpenStreetMap dev server soon.

#} + +
+ +
+ {% else %}

There are currently no tiles to process. Come back later!

{% endif %} - -
- {% if confirmation %} -

Currently verifying already-seen tiles.

- {% endif %} - {% if remaining %} -

Pictures remaining: {{remaining}}

- {% endif %} -
+ {# vim: set filetype=jinja: #} From 0ea3faa4e61e105b99e24e01e0ed4fa20115b6f2 Mon Sep 17 00:00:00 2001 From: Harry Wood Date: Wed, 4 Jun 2014 00:33:30 +0100 Subject: [PATCH 3/5] description. Not currently running anywhere --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd205d3 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +kuona +===== + +Kuona is/was a micro-tasking tool to help with H.O.T. mapping of villages in Mali. It presented users with a bing imagery square, and asked them to click if they see any buildings/villages. More advanced OpenStreetMap mappers could then use the results to find missing villages across a very sparesely populated saharan landscape. + +It was developed by nicolas17 (AKA "PovAddict") as quick proof of concept which was used quite effectively during the [2012 Mail Crisis](http://wiki.openstreetmap.org/wiki/2012_Mali_Crisis) response mapping. + +Output from the tool was a GPX file showing where everybody had been clicking. Intially these points were just viewed directly in JOSM. Harry developed a ruby script to eliminate points which already appeared to be mapped. Pierre G set up [a special task manager job](http://tasks.hotosm.org/job/198) using the data. This presented task squares spread out in an interesting pattern (unlike the usual grid). + +The tool was initially deployed here: http://stuff.povaddict.com.ar/mali-crowdsource/ From 9c5ea5264ed81ef13830310aff63b648b46c9efb Mon Sep 17 00:00:00 2001 From: Harry Wood Date: Wed, 4 Jun 2014 00:40:49 +0100 Subject: [PATCH 4/5] more detail on the behind-the-scenes count stats --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dd205d3..b7e4575 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ kuona ===== -Kuona is/was a micro-tasking tool to help with H.O.T. mapping of villages in Mali. It presented users with a bing imagery square, and asked them to click if they see any buildings/villages. More advanced OpenStreetMap mappers could then use the results to find missing villages across a very sparesely populated saharan landscape. +Kuona is/was a micro-tasking tool to help with H.O.T. mapping of villages in Mali. It presented users with a bing imagery square, and asked them to click if they see any buildings/villages. This micro task was super-simple, and didn't even require a login. Much easier to learn than even the most simplistic editing of OpenStreetMap. -It was developed by nicolas17 (AKA "PovAddict") as quick proof of concept which was used quite effectively during the [2012 Mail Crisis](http://wiki.openstreetmap.org/wiki/2012_Mali_Crisis) response mapping. +More advanced OpenStreetMap mappers could then use the results to find missing villages (which was useful across this very sparesely populated saharan landscape) + +It was developed by nicolas17 (AKA "PovAddict") as quick proof of concept which was used quite effectively during the [2012 Mail Crisis](http://wiki.openstreetmap.org/wiki/2012_Mali_Crisis) response mapping. +The tool was initially deployed here: http://stuff.povaddict.com.ar/mali-crowdsource/ Output from the tool was a GPX file showing where everybody had been clicking. Intially these points were just viewed directly in JOSM. Harry developed a ruby script to eliminate points which already appeared to be mapped. Pierre G set up [a special task manager job](http://tasks.hotosm.org/job/198) using the data. This presented task squares spread out in an interesting pattern (unlike the usual grid). -The tool was initially deployed here: http://stuff.povaddict.com.ar/mali-crowdsource/ +The tool also had a behind-the-scenes admin interface displaying a count of how many times a tiles had been seen vs "hits" (times users had decided to click) and nicolas was able to direct attention onto re-checking tiles or looking at new ones as desired. From 7e87a4ec69406c66a023fa53ce9cfb5772ef8add Mon Sep 17 00:00:00 2001 From: Harry Wood Date: Tue, 10 Feb 2015 01:25:48 +0000 Subject: [PATCH 5/5] Add my missingplaces ruby script e message[B for your changes. Lines starting --- missingplaces/README.md | 7 ++ missingplaces/fetchplaces.sh | 46 ++++++++ missingplaces/missingplaces.rb | 191 +++++++++++++++++++++++++++++++++ missingplaces/missingplaces.sh | 7 ++ 4 files changed, 251 insertions(+) create mode 100644 missingplaces/README.md create mode 100755 missingplaces/fetchplaces.sh create mode 100644 missingplaces/missingplaces.rb create mode 100644 missingplaces/missingplaces.sh diff --git a/missingplaces/README.md b/missingplaces/README.md new file mode 100644 index 0000000..4e11b0f --- /dev/null +++ b/missingplaces/README.md @@ -0,0 +1,7 @@ +# Missing places script + +This was Harry's weird way of solving a simple buffering geo-calculation as a script which could be re-run repeatedly. + +We fetch the latest OSM data on places (Mali villages) using fetchplaces.sh + +...and then we read in the GPX file of places people have clicked, and write out a GPX file of places which seem to be missing in OSM. diff --git a/missingplaces/fetchplaces.sh b/missingplaces/fetchplaces.sh new file mode 100755 index 0000000..edbe84b --- /dev/null +++ b/missingplaces/fetchplaces.sh @@ -0,0 +1,46 @@ +set -0 + +# Bash script to fetch villages data in Mali +# - fetched as several .osm files for different data types +# - merged into one using osmconvert +# - centroid nodes only using osmconvert +# - converted to CSV using osmconvert +# +# ...but the whole thing couldve been done by Overpass + +#These steps can actually be done as +# one magical OverpassAPI query. + +#xapi="http://jxapi.osm.rambler.ru/xapi/api/0.6/*" +#xapi="http://jxapi.openstreetmap.org/xapi/api/0.6/*" +xapi="http://www.overpass-api.de/api/xapi?*" + +#mali +bbox="[bbox=-12.46545,9.4758,6.34314,25.74516]" + +# test region +#bbox="[bbox=-4.7010513,14.5520723,-4.5533712,14.637127]" + +echo "calling xapi for places" +wget $xapi[place=*]$bbox -O "places.osm" +echo "calling xapi for residential landuse" +wget $xapi[landuse=*]$bbox -O "landuse.osm" +wget $xapi[building=*]$bbox -O "buildings.osm" +wget $xapi[barrier=*]$bbox -O "barriers.osm" + + +#echo "adding fake version=1" +#./osmconvert places.osm --fake-version > places-fv.osm +#./osmconvert residential.osm --fake-version > residential-fv.osm + +./osmconvert places.osm landuse.osm buildings.osm barriers.osm -o=places-ways-nodes.osm + + +echo "converting to nodes" +./osmconvert places-ways-nodes.osm --all-to-nodes >places-nodes.osm +#./osmconvert places-ways-nodes.osm --all-to-nodes | grep -v "" >places-nodes.osm + +echo "converting to CSV" +./osmconvert places-nodes.osm --csv="@lat @lon name" >places.csv + +echo "DONE places.csv" diff --git a/missingplaces/missingplaces.rb b/missingplaces/missingplaces.rb new file mode 100644 index 0000000..ddd1f5f --- /dev/null +++ b/missingplaces/missingplaces.rb @@ -0,0 +1,191 @@ +require "nokogiri" +include Math + +STDOUT.sync = true + +# +# This script takes a GPX file which was output from +# kuona, representing places people clicked when they +# thought they could see a village in the imagery +# +# It also takes a CSV file representing places nodes +# from OSM (i.e. data for villages already in OSM) +# +# It then performs a spatial buffer operation to drop +# any GPX points for villages which seem to be in +# OSM already. +# +# There's probably a two line GRASS command which +# could have done this. Some nice bare ruby logic but +# there's probably some GRASS two-liner which could +# have done this :-) +# + +Radius = 6371 # rough radius of the Earth, in kilometers +Buffer = 0.1 +OutputGPX = "missingplaces-new.gpx" + +# SAX parser for processing a GPX file +# calls gotPoint on each element +# subclass of Nokogiri::XML::SAX::Document +class MyDoc < Nokogiri::XML::SAX::Document + def start_element name, attrs = [] + if name=="wpt" + + $count+=1 + + lat = attrs[0][1].to_f + lon = attrs[1][1].to_f + puts lat.to_s + " " + lon.to_s if $count<2 + gotPoint(lat,lon) + # test region + # if lon>-4.7010513 and + # lon<-4.5533712 and + # lat>14.5520723 and + # lat<14.637127 + + end + end + + #def end_element name + # puts "ending: #{name}" + #end +end + +#quick test if two points are within 0.02 degrees of eachother +def is_close(start_coords, end_coords) + lat1, long1 = deg2rad *start_coords + lat2, long2 = deg2rad *end_coords + if (long1 - long2 < 0.02) and + (long1 - long2 > -0.02) and + (lat1 - lat2 < 0.02) and + (lat1 - lat2 > -0.02) + return true + else + return false + end +end + +#great circle distance betwen two points +def spherical_distance(start_coords, end_coords) + lat1, long1 = deg2rad *start_coords + lat2, long2 = deg2rad *end_coords + 2 * Radius * asin(sqrt(sin((lat2-lat1)/2)**2 + cos(lat1) * cos(lat2) * sin((long2 - long1)/2)**2)) +end + +def deg2rad(lat, long) + [lat * PI / 180, long * PI / 180] +end + +#Get nearby places (OSM villages) +#Do this by looking up any places within the same grid +#square, or neighbouring grid squares +def get_close_places(lat,lon) + + square = get_square(lat,lon) + sqlat = square[0] + sqlon = square[1] + places = [] + places.concat $places[ [sqlat-1,sqlon+1 ] ] || [] + places.concat $places[ [sqlat ,sqlon+1 ] ] || [] + places.concat $places[ [sqlat+1,sqlon+1 ] ] || [] + places.concat $places[ [sqlat-1,sqlon ] ] || [] + places.concat $places[ [sqlat ,sqlon ] ] || [] + places.concat $places[ [sqlat+1,sqlon ] ] || [] + places.concat $places[ [sqlat-1,sqlon-1 ] ] || [] + places.concat $places[ [sqlat ,sqlon-1 ] ] || [] + places.concat $places[ [sqlat+1,sqlon-1 ] ] || [] + + return places +end + +#decide which square a lat lon coordinate is within +#reutrn the unique square reference which is actually +#a [lat,lon] pair just with the values rounded. +def get_square(lat,lon) + return [ ((lat * 100.0).round)/100.0, + ((lon * 100.0).round)/100.0 ] + +end + +#Called for each waypoint found in the input GPX +#(where people think they've seen villages) +def gotPoint(lat,lon) + + nearby_place_found = false + + # call get_close_places to get nearby OSM places + # (as per the grid squares) + get_close_places(lat,lon).each do |place| + + #for each place + plat,plon,name = place + + if is_close([lat, lon], [plat.to_f, plon.to_f]) + #It's vaguely close by. Let's do the actual distance calculation + dist = spherical_distance([lat, lon], [plat.to_f, plon.to_f]) + if dist < Buffer + nearby_place_found = true + break + end + end + end + if not nearby_place_found + #No nearby places found (which is the intersting case!) + $missingcount+=1 + puts "gpx point " + $count.to_s + " - No nearby place. " + lat.to_s + " " + lon.to_s #+ " " + shortest_dist.to_s + "km (" + shortest_place.to_s + " " + splat.to_s + " " + splon.to_s + ")" + + #Write a waypoint to the output GPX file + File.open(OutputGPX, 'a') {|f| f.write( "\n" ) } + + end +end + +#Load openstreetmap places (villages) data from CSv file +#can populate the grid squares index +def load_places_CSV() + puts "loading CSV" + $places = {} + File.open('places.csv').each do |line| + place = line.split("\t") + plat,plon,name = place + if plat.to_f>14.04 + square = get_square(plat.to_f, plon.to_f) + if $places[square].nil? + p square + $places[square] = [] + end + $places[square] << place + end + end + puts $places.size.to_s + " places loaded" +end + +start_time = Time.now + +$count = 0 +$missingcount = 0 +$places = {} +load_places_CSV() + + + +filename = ARGV[0] +raise('no file param') if filename.nil? + +# Create our parser +parser = Nokogiri::XML::SAX::Parser.new(MyDoc.new) + +File.open(OutputGPX, 'w') {|f| f.write( "\n" ) } + +# Send some XML to the parser +parser.parse(File.open(ARGV[0])) + +File.open(OutputGPX, 'a') {|f| f.write( "\n" ) } + +report = $count.to_s + " points processed from GPX file\n" + + $missingcount.to_s + " points found a long way from anything\n" + + "Excution time:" + (Time.now - start_time).round.to_s + " s" + +puts report +File.open(OutputGPX, 'a') {|f| f.write( "" ) } diff --git a/missingplaces/missingplaces.sh b/missingplaces/missingplaces.sh new file mode 100644 index 0000000..7ab4b57 --- /dev/null +++ b/missingplaces/missingplaces.sh @@ -0,0 +1,7 @@ +set -0 +./fetchplaces.sh + +rm mali-crowdsource.gpx +wget http://stuff.povaddict.com.ar/mali-crowdsource.gpx + +ruby missingplaces.rb mali-crowdsource.gpx