Merge pull request mysteriumnetwork#281 from mysteriumnetwork/improve…

…ment/build-in-geoip-db

Compile in geoip2 database and avoid external file reading

bin/package/config/common/README

@@ -0,0 +1 @@
+Any files inside this directory will be available for all OS distributions as part of config-dir

cmd/di.go

@@ -248,8 +248,10 @@ func (di *Dependencies) bootstrapLocationComponents(options node.OptionsLocation
 
 	switch {
 	case options.Country != "":
-		di.LocationResolver = location.NewResolverFake(options.Country)
+		di.LocationResolver = location.NewStaticResolver(options.Country)
+	case options.ExternalDb != "":
+		di.LocationResolver = location.NewExternalDbResolver(filepath.Join(configDirectory, options.ExternalDb))
 	default:
-		di.LocationResolver = location.NewResolver(filepath.Join(configDirectory, options.Database))
+		di.LocationResolver = location.NewBuiltInResolver()
 	}
 }

cmd/flags_location.go

@@ -31,7 +31,7 @@ var (
 	locationDatabaseFlag = cli.StringFlag{
 		Name:  "location.database",
 		Usage: "Service location autodetect database of GeoLite2 format e.g. http://dev.maxmind.com/geoip/geoip2/geolite2/",
-		Value: "GeoLite2-Country.mmdb",
+		Value: "",
 	}
 	// LocationCountryFlag allows to configure service country manually
 	LocationCountryFlag = cli.StringFlag{
@@ -49,8 +49,8 @@ func RegisterFlagsLocation(flags *[]cli.Flag) {
 // ParseFlagsLocation function fills in location options from CLI context
 func ParseFlagsLocation(ctx *cli.Context) node.OptionsLocation {
 	return node.OptionsLocation{
-		ctx.GlobalString(ipifyUrlFlag.Name),
-		ctx.GlobalString(locationDatabaseFlag.Name),
-		ctx.GlobalString(LocationCountryFlag.Name),
+		IpifyUrl:   ctx.GlobalString(ipifyUrlFlag.Name),
+		ExternalDb: ctx.GlobalString(locationDatabaseFlag.Name),
+		Country:    ctx.GlobalString(LocationCountryFlag.Name),
 	}
 }

core/location/built_in_resolver.go

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2017 The "MysteriumNetwork/node" Authors.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package location
+
+import (
+	"github.com/mysteriumnetwork/node/core/location/gendb"
+	"github.com/oschwald/geoip2-golang"
+)
+
+//go:generate go run generator/generator.go --dbname db/GeoLite2-Country.mmdb --output gendb --compress
+
+// NewBuiltInResolver returns new db resolver initialized from built in data
+func NewBuiltInResolver() Resolver {
+	dbBytes, err := gendb.LoadData()
+	if err != nil {
+		return NewFailingResolver(err)
+	}
+
+	dbReader, err := geoip2.FromBytes(dbBytes)
+	if err != nil {
+		return NewFailingResolver(err)
+	}
+	return &DbResolver{dbReader: dbReader}
+}

core/location/built_in_resolver_test.go

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2017 The "MysteriumNetwork/node" Authors.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package location
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuiltInResolverWorks(t *testing.T) {
+	country, err := NewBuiltInResolver().ResolveCountry("46.111.111.99")
+	assert.NoError(t, err)
+	assert.Equal(t, "RU", country)
+}

core/location/cache_test.go

@@ -46,7 +46,7 @@ func TestLocationCacheFirstSecondCalls(t *testing.T) {
 }
 
 func TestLocationCacheWithError(t *testing.T) {
-	locationErr := errors.New("location resolver error")
+	locationErr := errors.New("location DbResolver error")
 	locationDetector := NewDetectorFakeFailing(locationErr)
 	locationCache := NewLocationCache(locationDetector)
 

core/location/resolver.go → core/location/db_resolver.go

@@ -24,31 +24,31 @@ import (
 	"github.com/oschwald/geoip2-golang"
 )
 
-type resolver struct {
-	databasePath string
+// DbResolver struct represents ip -> country resolver which uses geoip2 data reader
+type DbResolver struct {
+	dbReader *geoip2.Reader
 }
 
-// NewResolver returns Resolver which uses country database
-func NewResolver(databasePath string) Resolver {
-	return &resolver{
-		databasePath: databasePath,
+// NewExternalDbResolver returns Resolver which uses external country database
+func NewExternalDbResolver(databasePath string) Resolver {
+	db, err := geoip2.Open(databasePath)
+	if err != nil {
+		return NewFailingResolver(err)
 	}
-}
 
-// ResolveCountry maps given ip to country
-func (r *resolver) ResolveCountry(ip string) (string, error) {
-	db, err := geoip2.Open(r.databasePath)
-	if err != nil {
-		return "", err
+	return &DbResolver{
+		dbReader: db,
 	}
-	defer db.Close()
+}
 
+// ResolveCountry maps given ip to country
+func (r *DbResolver) ResolveCountry(ip string) (string, error) {
 	ipObject := net.ParseIP(ip)
 	if ipObject == nil {
 		return "", errors.New("failed to parse IP")
 	}
 
-	countryRecord, err := db.Country(ipObject)
+	countryRecord, err := r.dbReader.Country(ipObject)
 	if err != nil {
 		return "", err
 	}

core/location/resolver_test.go → core/location/db_resolver_test.go

@@ -38,7 +38,7 @@ func TestResolverResolveCountry(t *testing.T) {
 		{"asd", "", "failed to parse IP"},
 	}
 
-	resolver := NewResolver("../../bin/package/config/common/GeoLite2-Country.mmdb")
+	resolver := NewExternalDbResolver("db/GeoLite2-Country.mmdb")
 	for _, tt := range tests {
 		got, err := resolver.ResolveCountry(tt.ip)
 

core/location/detector.go

@@ -33,15 +33,15 @@ func NewDetector(ipResolver ip.Resolver, locationResolver Resolver) Detector {
 func NewDetectorFake(ipAddress string, country string) Detector {
 	return &detector{
 		ipResolver:       ip.NewResolverFake(ipAddress),
-		locationResolver: NewResolverFake(country),
+		locationResolver: NewStaticResolver(country),
 	}
 }
 
 // NewDetectorFakeFailing constructs Detector instance with faked error
 func NewDetectorFakeFailing(err error) Detector {
 	return &detector{
 		ipResolver:       ip.NewResolverFake(""),
-		locationResolver: NewResolverFakeFailing(err),
+		locationResolver: NewFailingResolver(err),
 	}
 }
 

core/location/detector_test.go

@@ -35,10 +35,10 @@ func TestNewDetector(t *testing.T) {
 }
 
 func TestWithIpResolverFailing(t *testing.T) {
-	ipErr := errors.New("ip resolver error")
+	ipErr := errors.New("ip DbResolver error")
 	detector := NewDetector(
 		ip.NewResolverFakeFailing(ipErr),
-		NewResolverFake(""),
+		NewStaticResolver(""),
 	)
 
 	location, err := detector.DetectLocation()
@@ -47,10 +47,10 @@ func TestWithIpResolverFailing(t *testing.T) {
 }
 
 func TestWithLocationResolverFailing(t *testing.T) {
-	locationErr := errors.New("location resolver error")
+	locationErr := errors.New("location DbResolver error")
 	detector := NewDetector(
 		ip.NewResolverFake(""),
-		NewResolverFakeFailing(locationErr),
+		NewFailingResolver(locationErr),
 	)
 
 	location, err := detector.DetectLocation()

core/location/gendb/db_index.go

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2017 The "MysteriumNetwork/node" Authors.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package gendb
+
+// generated by generator.go - DO NOT EDIT
+
+const originalSize = 3303090
+const dbData = "" + dbDataPart0 + dbDataPart1
+
+// LoadData returns embedded database as byte array
+func LoadData() ([]byte, error) {
+	return EncodedDataLoader(dbData, originalSize, true)
+}

core/location/gendb/loader.go

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2017 The "MysteriumNetwork/node" Authors.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package gendb
+
+import (
+	"compress/gzip"
+	"encoding/base64"
+	"errors"
+	"io/ioutil"
+	"strings"
+)
+
+// EncodedDataLoader returns emmbeded database as byte array
+func EncodedDataLoader(data string, originalSize int, compressed bool) (decompressed []byte, err error) {
+	reader := base64.NewDecoder(base64.RawStdEncoding, strings.NewReader(data))
+
+	if compressed {
+		reader, err = gzip.NewReader(reader)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	decompressed, err = ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+	if len(decompressed) != originalSize {
+		return nil, errors.New("original and decompressed data size mismatch")
+	}
+	return decompressed, nil
+}