Skip to content

Commit

Permalink
Merge pull request #64 from pelias/binary_encoding
Browse files Browse the repository at this point in the history
improved binary encoding
  • Loading branch information
orangejulius authored Jul 13, 2018
2 parents 37ef362 + aafe5e6 commit 636d702
Show file tree
Hide file tree
Showing 6 changed files with 125 additions and 42 deletions.
Binary file modified build/pbf2json.darwin-x64
Binary file not shown.
Binary file modified build/pbf2json.linux-arm
Binary file not shown.
Binary file modified build/pbf2json.linux-x64
Binary file not shown.
Binary file modified build/pbf2json.win32-x64
Binary file not shown.
57 changes: 57 additions & 0 deletions encoding_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package main

import (
"testing"

"github.com/qedus/osmpbf"
"github.com/stretchr/testify/assert"
)

func TestEncodingSimple(t *testing.T) {

var node = &osmpbf.Node{ID: 100, Lat: -50, Lon: 77}
var expectedBytes = []byte{0xc0, 0x49, 0x0, 0x0, 0x0, 0x0, 0x40, 0x53, 0x40, 0x0, 0x0, 0x0}
var expectedLatlon = map[string]string{"lon": "77.0000000", "lat": "-50.0000000"}

// encode
var stringid, byteval = nodeToBytes(node)
assert.Equal(t, "100", stringid)
assert.Equal(t, expectedBytes, byteval)

// decode
var latlon = bytesToLatLon(byteval)
assert.Equal(t, expectedLatlon, latlon)
}

func TestEncodingFloatPrecision(t *testing.T) {

var node = &osmpbf.Node{ID: 100, Lat: -50.555555555, Lon: 77.777777777}
var expectedBytes = []byte{0xc0, 0x49, 0x47, 0x1c, 0x71, 0xc5, 0x40, 0x53, 0x71, 0xc7, 0x1c, 0x70}
var expectedLatlon = map[string]string{"lon": "77.7777778", "lat": "-50.5555556"}

// encode
var stringid, byteval = nodeToBytes(node)
assert.Equal(t, "100", stringid)
assert.Equal(t, expectedBytes, byteval)

// decode
var latlon = bytesToLatLon(byteval)
assert.Equal(t, expectedLatlon, latlon)
}

func TestEncodingBitmaskValues(t *testing.T) {

var tags = map[string]string{"entrance": "main", "wheelchair": "yes"}
var node = &osmpbf.Node{ID: 100, Lat: -50, Lon: 77, Tags: tags}
var expectedBytes = []byte{0xc0, 0x49, 0x0, 0x0, 0x0, 0x0, 0x40, 0x53, 0x40, 0x0, 0x0, 0x0, 0xa0}
var expectedLatlon = map[string]string{"lon": "77.0000000", "lat": "-50.0000000", "entrance": "2", "wheelchair": "2"}

// encode
var stringid, byteval = nodeToBytes(node)
assert.Equal(t, "100", stringid)
assert.Equal(t, expectedBytes, byteval)

// decode
var latlon = bytesToLatLon(byteval)
assert.Equal(t, expectedLatlon, latlon)
}
110 changes: 68 additions & 42 deletions pbf2json.go
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
package main

import "encoding/json"
import "fmt"
import "flag"
import "bytes"
import "os"
import "log"
import "io"

import "runtime"
import "strings"
import "strconv"
import "github.com/qedus/osmpbf"
import "github.com/syndtr/goleveldb/leveldb"
import "github.com/paulmach/go.geo"
import (
"bytes"
"encoding/binary"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"math"
"os"
"runtime"
"strconv"
"strings"

"github.com/paulmach/go.geo"
"github.com/qedus/osmpbf"
"github.com/syndtr/goleveldb/leveldb"
)

type settings struct {
PbfPath string
Expand Down Expand Up @@ -238,7 +242,7 @@ func isWheelchairAccessibleNode(node *osmpbf.Node) uint8 {

// write to leveldb immediately
func cacheStore(db *leveldb.DB, node *osmpbf.Node) {
id, val := formatLevelDB(node)
id, val := nodeToBytes(node)
err := db.Put([]byte(id), []byte(val), nil)
if err != nil {
log.Fatal(err)
Expand All @@ -247,7 +251,7 @@ func cacheStore(db *leveldb.DB, node *osmpbf.Node) {

// queue a leveldb write in a batch
func cacheQueue(batch *leveldb.Batch, node *osmpbf.Node) {
id, val := formatLevelDB(node)
id, val := nodeToBytes(node)
batch.Put([]byte(id), []byte(val))
}

Expand All @@ -273,47 +277,69 @@ func cacheLookup(db *leveldb.DB, way *osmpbf.Way) ([]map[string]string, error) {
return container, err
}

s := string(data)
spl := strings.Split(s, ":")
container = append(container, bytesToLatLon(data))
}

latlon := make(map[string]string)
lat, lon := spl[0], spl[1]
latlon["lat"] = lat
latlon["lon"] = lon
return container, nil
}

// check for third & fourth fields which indicate an entrance
// and the level of wheelchair accessibility
if len(spl) == 4 {
latlon["entrance"] = spl[2]
latlon["wheelchair"] = spl[3]
}
// decode bytes to a 'latlon' type object
func bytesToLatLon(data []byte) map[string]string {

container = append(container, latlon)
var latlon = make(map[string]string)

}
// first 6 bytes are the latitude
var latBytes = append([]byte{}, data[0:6]...)
var lat64 = math.Float64frombits(binary.BigEndian.Uint64(append(latBytes, []byte{0x0, 0x0}...)))
latlon["lat"] = strconv.FormatFloat(lat64, 'f', 7, 64)

return container, nil
// next 6 bytes are the longitude
var lonBytes = append([]byte{}, data[6:12]...)
var lon64 = math.Float64frombits(binary.BigEndian.Uint64(append(lonBytes, []byte{0x0, 0x0}...)))
latlon["lon"] = strconv.FormatFloat(lon64, 'f', 7, 64)

// fmt.Println(way.NodeIDs)
// fmt.Println(container)
// os.Exit(1)
}
// check for the bitmask byte which indicates things like an
// entrance and the level of wheelchair accessibility
if len(data) > 12 {
latlon["entrance"] = fmt.Sprintf("%d", (data[12]&0xC0)>>6)
latlon["wheelchair"] = fmt.Sprintf("%d", (data[12]&0x30)>>4)
}

func formatLevelDB(node *osmpbf.Node) (id string, val []byte) {
return latlon
}

stringid := strconv.FormatInt(node.ID, 10)
// encode a node as bytes (between 12 & 13 bytes used)
func nodeToBytes(node *osmpbf.Node) (string, []byte) {

var bufval bytes.Buffer
bufval.WriteString(strconv.FormatFloat(node.Lat, 'f', 7, 64))
bufval.WriteString(":")
bufval.WriteString(strconv.FormatFloat(node.Lon, 'f', 7, 64))

// encode lat/lon as 64 bit floats packed in to 8 bytes,
// each float is then truncated to 6 bytes because we don't
// need the additional precision (> 8 decimal places)

var latBytes = make([]byte, 8)
binary.BigEndian.PutUint64(latBytes, math.Float64bits(node.Lat))
bufval.Write(latBytes[0:6])

var lonBytes = make([]byte, 8)
binary.BigEndian.PutUint64(lonBytes, math.Float64bits(node.Lon))
bufval.Write(lonBytes[0:6])

// generate a bitmask for relevant tag features
var isEntrance = isEntranceNode(node)
if isEntrance > 0 {
bufval.WriteString(fmt.Sprintf(":%d:%d", isEntrance, isWheelchairAccessibleNode(node)))
// leftmost two bits are for the entrance, next two bits are accessibility
// remaining 4 rightmost bits are reserved for future use.
var bitmask = isEntrance << 6
var isWheelchairAccessible = isWheelchairAccessibleNode(node)
if isWheelchairAccessible > 0 {
bitmask |= isWheelchairAccessible << 4
}
bufval.WriteByte(bitmask)
}

byteval := []byte(bufval.String())
stringid := strconv.FormatInt(node.ID, 10)
byteval := bufval.Bytes()

return stringid, byteval
}
Expand Down

0 comments on commit 636d702

Please sign in to comment.