From b1d497a365f8b4386f0117ce40a5212feaebee87 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sat, 12 Mar 2022 15:12:46 +0800 Subject: [PATCH] fix dashboard (#396) --- README.md | 7 +-- cmd/README.md | 94 ------------------------------------- config/config.toml.template | 1 + docker/README.md | 58 ----------------------- docker/config.toml | 51 ++++++++++++++++++-- docker/docker-compose.yml | 32 ++++++++++--- go.mod | 2 +- go.sum | 4 +- master/rest.go | 2 +- master/rest_test.go | 2 +- server/rest.go | 11 +++-- server/rest_test.go | 1 + 12 files changed, 89 insertions(+), 176 deletions(-) delete mode 100644 cmd/README.md delete mode 100644 docker/README.md diff --git a/README.md b/README.md index 1e74246ca..09a36dc36 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,8 @@ Gorse is an open-source recommendation system written in Go. Gorse aims to be a ## Quick Start -- [Run Gorse manually](https://github.com/zhenghaoz/gorse/tree/master/cmd) -- [Run Gorse with Docker Compose](https://github.com/zhenghaoz/gorse/tree/master/docker) -- [Use Gorse to recommend awesome GitHub repositories](https://github.com/zhenghaoz/gitrec) - [Read official documents](https://docs.gorse.io/) -- [Read develop roadmap](https://github.com/zhenghaoz/gorse/projects) +- [Visit official demo](https://gitrec.gorse.io/) ## Architecture @@ -49,7 +46,7 @@ In addition, the administrator can perform system monitoring, data import and ex -Any contribution is appreciated: report a bug, give advice or even create a pull request. You are welcome to submit [*Gorse In-Production User Registration*](https://forms.office.com/r/q04Fgkh18t) to add your product to the case studies section of [gorse.io](https://gorse.io/) if you are using Gorse in production. +Any contribution is appreciated: report a bug, give advice or even create a pull request. ## Acknowledgments diff --git a/cmd/README.md b/cmd/README.md deleted file mode 100644 index 3239a3456..000000000 --- a/cmd/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# Run Gorse Manually - -Binary distributions have been provided for 64-bit Windows/Linux/Mac OS on the [release](https://github.com/zhenghaoz/gorse/releases) page. Due to the demand on large memories, 64-bit machines are highly recommended to deploy Gorse. - -## Prerequisite - -Gorse depends on following software: - -- *Redis* is used to store caches. -- One of *MySQL/PostgresSQL/ClickHouse/MongoDB* is used to store data. - -The minimal versions of dependent software are as follows: - -| Software | Minimal Version | -|-------------|-----------------| -| Redis | 5.0 | -| MySQL | 5.7 | -| PostgresSQL | 10.0 | -| ClickHouse | 21.10 | -| MongoDB | 4.0 | - -## Run Gorse - -- Install Gorse - -**Option 1:** Download binary distributions (Linux) - -```bash -wget https://github.com/zhenghaoz/gorse/releases/latest/download/gorse_linux_amd64.zip -unzip gorse_linux_amd64.zip -``` - -For Windows and MacOS (Intel Chip or Apple Silicon), download [gorse_windows_amd64.zip](https://github.com/zhenghaoz/gorse/releases/latest/download/gorse_windows_amd64.zip), [gorse_darwin_amd64.zip](https://github.com/zhenghaoz/gorse/releases/latest/download/gorse_darwin_amd64.zip) or [gorse_darwin_arm64.zip](https://github.com/zhenghaoz/gorse/releases/latest/download/gorse_darwin_arm64.zip) respectively. - -**Option 2:** Build executable files via `go get` - -```bash -go get github.com/zhenghaoz/gorse/... -``` - -Built binaries locate at `$(go env GOPATH)/bin`. - -- Configuration - -Create a configuration file [config.toml](https://github.com/zhenghaoz/gorse/blob/master/config/config.toml.template) in the working directory. Set `cache_store` and `data_store` in the configuration file [config.toml](https://github.com/zhenghaoz/gorse/blob/master/config/config.toml.template). - -```toml -# This section declares settings for the database. -[database] -# database for caching (support Redis only) -cache_store = "redis://localhost:6379" -# database for persist data (support MySQL/MongoDB) -data_store = "mysql://root@tcp(localhost:3306)/gorse?parseTime=true" -``` - -- Download the SQL file [github.sql](https://cdn.gorse.io/example/github.sql) and import to the MySQL instance. - -```bash -mysql -h 127.0.0.1 -u gorse -pgorse_pass gorse < github.sql -``` - -- Start the master node - -```bash -./gorse-master -c config.toml -``` - -`-c` specify the path of the configuration file. - -- Start the server node and worker node - -```bash -./gorse-server --master-host 127.0.0.1 --master-port 8086 \ - --http-host 127.0.0.1 --http-port 8087 -``` - -`--master-host` and `--master-port` are the RPC host and port of the master node. `--http-host` and `--http-port` are the HTTP host and port for RESTful APIs and metrics reporting of this server node. - -```bash -./gorse-worker --master-host 127.0.0.1 --master-port 8086 \ - --http-host 127.0.0.1 --http-port 8089 -j 4 -``` - -`--master-host` and `--master-port` are the RPC host and port of the master node. `--http-host` and `--http-port` are the HTTP host and port for metrics reporting of this worker node. `-j` is the number of working threads. - -- Play with Gorse: - -| Entry | Link | -| --- | --- | -| Master Dashboard | http://127.0.0.1:8088/ | -| Server RESTful API | http://127.0.0.1:8087/apidocs | -| Server Prometheus Metrics | http://127.0.0.1:8087/metrics | -| Worker Prometheus Metrics | http://127.0.0.1:8089/metrics | - diff --git a/config/config.toml.template b/config/config.toml.template index c51bd2c08..429c31b88 100644 --- a/config/config.toml.template +++ b/config/config.toml.template @@ -10,6 +10,7 @@ cache_store = "redis://localhost:6379/0" # postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full # clickhouse://user:password@host[:port]/database?param1=value1&...¶mN=valueN # mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]] +# mongodb+srv://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]] data_store = "mysql://gorse:gorse_pass@tcp(localhost:3306)/gorse?parseTime=true" # The cache size for recommended/popular/latest items. The default value is 100. diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index c9ddd7343..000000000 --- a/docker/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Run Gorse with Docker Compose - -The best practice to manage Gorse nodes is using orchestration tools such as Docker Compose, etc.. There are Docker images of the master node, the server node and the worker node. - -| Docker Image | Image Size | -| ------------ | -------- | -| gorse-master | [![](https://img.shields.io/docker/image-size/zhenghaoz/gorse-master)](https://hub.docker.com/repository/docker/zhenghaoz/gorse-master) | -| gorse-server | [![](https://img.shields.io/docker/image-size/zhenghaoz/gorse-server)](https://hub.docker.com/repository/docker/zhenghaoz/gorse-server) | -| gorse-worker | [![](https://img.shields.io/docker/image-size/zhenghaoz/gorse-worker)](https://hub.docker.com/repository/docker/zhenghaoz/gorse-worker) | - -## Prerequisite - -Gorse depends on following software: - -- *Redis* is used to store caches. -- One of *MySQL/PostgresSQL/ClickHouse/MongoDB* is used to store data. - -The minimal versions of dependent software are as follows: - -| Software | Minimal Version | -|-------------|-----------------| -| Redis | 5.0 | -| MySQL | 5.7 | -| PostgresSQL | 10.0 | -| ClickHouse | 21.10 | -| MongoDB | 4.0 | - -## Quick Start - -There is an example [docker-compose.yml](https://github.com/zhenghaoz/gorse/blob/master/docker/docker-compose.yml) consists of a master node, a server node and a worker node, a Redis instance, and a MySQL instance. - -- Create a configuration file [config.toml](https://github.com/zhenghaoz/gorse/blob/master/docker/config.toml) (Docker Compose version) in the working directory. -- Setup the Gorse cluster using Docker Compose. - -```bash -docker-compose up -d -``` - -- Download the SQL file [github.sql](https://cdn.gorse.io/example/github.sql) and import to the MySQL instance. - -``` -mysql -h 127.0.0.1 -u gorse -pgorse_pass gorse < github.sql -``` - -- Restart the master node to apply imported data. - -```bash -docker-compose restart -``` - -- Play with Gorse: - -| Entry | Link | -| --- | --- | -| Master Dashboard | http://127.0.0.1:8088/ | -| Server RESTful API | http://127.0.0.1:8087/apidocs | -| Server Prometheus Metrics | http://127.0.0.1:8087/metrics | -| Worker Prometheus Metrics | http://127.0.0.1:8089/metrics | diff --git a/docker/config.toml b/docker/config.toml index 45f33ef5f..e74685fce 100644 --- a/docker/config.toml +++ b/docker/config.toml @@ -10,10 +10,11 @@ cache_store = "redis://redis:6379" # postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full # clickhouse://user:password@host[:port]/database?param1=value1&...¶mN=valueN # mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]] +# mongodb+srv://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]] data_store = "mysql://gorse:gorse_pass@tcp(mysql:3306)/gorse?parseTime=true" # The cache size for recommended/popular/latest items. The default value is 100. -cache_size = 200 +cache_size = 100 # Insert new users while inserting feedback. The default value is true. auto_insert_user = true @@ -44,7 +45,7 @@ meta_timeout = 10 # cluster meta timeout (second) # This section declares settings for the server node. [server] -default_n = 20 # default number of returned items +default_n = 10 # default number of returned items api_key = "" # secret key for RESTful APIs (SSL required) # This section declares settings for recommendation. @@ -54,7 +55,7 @@ api_key = "" # secret key for RESTful APIs (SSL required) popular_window = 30 # The time period for model fitting (minutes). The default values is 60. -fit_period = 10 +fit_period = 360 # The time period for model searching (minutes). The default values is 100. search_period = 60 @@ -65,6 +66,9 @@ search_epoch = 100 # The number of trials for model searching. The default values is 10. search_trials = 10 +# The time period to check recommendation for users (minutes). The default values is 1. +check_recommend_period = 1 + # The time period to refresh recommendation for inactive users (days). The default values is 5. refresh_recommend_period = 1 @@ -75,6 +79,9 @@ refresh_recommend_period = 1 # Recommenders are used in order. The default values is ["latest"]. fallback_recommend = ["item_based", "latest"] +# The number of feedback used in fallback item-based similar recommendation. The default values is 10. +num_feedback_fallback_item_based = 10 + # The type of neighbors for items. There are three types: # similar: Neighbors are found by number of common labels. # related: Neighbors are found by number of common users. @@ -83,6 +90,15 @@ fallback_recommend = ["item_based", "latest"] # The default values is "auto". item_neighbor_type = "similar" +# Enable approximate item neighbor searching using vector index. +enable_item_neighbor_index = false + +# Minimal recall for approximate item neighbor searching. +item_neighbor_index_recall = 0.8 + +# Maximal number of fit epochs for approximate item neighbor searching vector index. +item_neighbor_index_fit_epoch = 3 + # The type of neighbors for users. There are three types: # similar: Neighbors are found by number of common labels. # related: Neighbors are found by number of common liked items. @@ -91,6 +107,15 @@ item_neighbor_type = "similar" # The default values is "auto". user_neighbor_type = "similar" +# Enable approximate user neighbor searching using vector index. +enable_user_neighbor_index = false + +# Minimal recall for approximate user neighbor searching. +user_neighbor_index_recall = 0.8 + +# Maximal number of fit epochs for approximate user neighbor searching vector index. +user_neighbor_index_fit_epoch = 3 + # Enable latest recommendation during offline recommendation. The default values is false. enable_latest_recommend = true @@ -106,6 +131,15 @@ enable_item_based_recommend = false # Enable collaborative filtering recommendation during offline recommendation. The default values is true. enable_collaborative_recommend = true +# Enable approximate collaborative filtering recommend using vector index. +enable_collaborative_index = false + +# Minimal recall for approximate collaborative filtering recommend. +collaborative_index_recall = 0.9 + +# Maximal number of fit epochs for approximate collaborative filtering recommend vector index. +collaborative_index_fit_epoch = 3 + # Enable click-though rate prediction during offline recommendation. Otherwise, results from multi-way recommendation # would be merged randomly. The default values is true. enable_click_through_prediction = true @@ -113,5 +147,14 @@ enable_click_through_prediction = true # The explore recommendation method is used to inject popular items or latest items into recommended result: # popular: Recommend popular items to cold-start users. # latest: Recommend latest items to cold-start users. -# Recommenders are used in order. The default values is { popular = 0.0, latest = 0.0 }. +# The default values is { popular = 0.0, latest = 0.0 }. explore_recommend = { popular = 0.1, latest = 0.2 } + +# Replace historical items back to recommendations. +enable_replacement = false + +# Decay the weights of replaced items from positive feedbacks. +positive_replacement_decay = 0.8 + +# Decay the weights of replaced items from read feedbacks. +read_replacement_decay = 0.6 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 92214e460..6074247e8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,6 +5,7 @@ services: restart: unless-stopped ports: - 6379:6379 + mysql: image: mysql/mysql-server restart: unless-stopped @@ -16,7 +17,8 @@ services: MYSQL_USER: gorse MYSQL_PASSWORD: gorse_pass volumes: - - ./var/lib/mysql:/var/lib/mysql + - mysql_data:/var/lib/mysql + worker: image: zhenghaoz/gorse-worker restart: unless-stopped @@ -26,8 +28,11 @@ services: --master-host master --master-port 8086 --http-host 0.0.0.0 --http-port 8089 --log-path /var/log/gorse/worker.log --cache-path /var/lib/gorse/worker_cache.data volumes: - - ./var/log/gorse:/var/log/gorse - - ./var/lib/gorse:/var/lib/gorse + - gorse_log:/var/log/gorse + - worker_data:/var/lib/gorse + depends_on: + - master + server: image: zhenghaoz/gorse-server restart: unless-stopped @@ -37,8 +42,11 @@ services: --master-host master --master-port 8086 --http-host 0.0.0.0 --http-port 8087 --log-path /var/log/gorse/server.log --cache-path /var/lib/gorse/server_cache.data volumes: - - ./var/log/gorse:/var/log/gorse - - ./var/lib/gorse:/var/lib/gorse + - gorse_log:/var/log/gorse + - server_data:/var/lib/gorse + depends_on: + - master + master: image: zhenghaoz/gorse-master restart: unless-stopped @@ -48,5 +56,15 @@ services: command: -c /etc/gorse/config.toml --log-path /var/log/gorse/master.log --cache-path /var/lib/gorse/master_cache.data volumes: - ./config.toml:/etc/gorse/config.toml - - ./var/log/gorse:/var/log/gorse - - ./var/lib/gorse:/var/lib/gorse + - gorse_log:/var/log/gorse + - master_data:/var/lib/gorse + depends_on: + - redis + - mysql + +volumes: + mysql_data: + worker_data: + server_data: + master_data: + gorse_log: diff --git a/go.mod b/go.mod index bc7549c67..c3950a498 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/go-redis/redis/v8 v8.11.0 github.com/go-sql-driver/mysql v1.6.0 github.com/gorilla/securecookie v1.1.1 - github.com/gorse-io/dashboard v0.0.0-20220221123215-3c20525d5e96 + github.com/gorse-io/dashboard v0.0.0-20220312053149-edd363d0232a github.com/haxii/go-swagger-ui v3.19.4+incompatible github.com/json-iterator/go v1.1.12 github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 diff --git a/go.sum b/go.sum index c2e708f3a..f6b144091 100644 --- a/go.sum +++ b/go.sum @@ -325,8 +325,8 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorse-io/dashboard v0.0.0-20220221123215-3c20525d5e96 h1:z5toIn/jqYvojGx474O96yvieXiO+nWcw/GzOBeE3DY= -github.com/gorse-io/dashboard v0.0.0-20220221123215-3c20525d5e96/go.mod h1:14j0KVMmOAAN0xvsa2wxFoV/X8Aa/gCeNFRLjtYP6+M= +github.com/gorse-io/dashboard v0.0.0-20220312053149-edd363d0232a h1:LlFvpY6VVr2oBpnRsL8eK+snwj25ib7gc66G0LNpJ1g= +github.com/gorse-io/dashboard v0.0.0-20220312053149-edd363d0232a/go.mod h1:14j0KVMmOAAN0xvsa2wxFoV/X8Aa/gCeNFRLjtYP6+M= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= diff --git a/master/rest.go b/master/rest.go index dccdea74b..ba79235f9 100644 --- a/master/rest.go +++ b/master/rest.go @@ -596,7 +596,7 @@ func (m *Master) getRecommend(request *restful.Request, response *restful.Respon results, err = m.Recommend(userId, category, n, m.RecommendUserBased) case "item_based": results, err = m.Recommend(userId, category, n, m.RecommendItemBased) - case "": + case "_": recommenders := []server.Recommender{m.RecommendOffline} for _, recommender := range m.GorseConfig.Recommend.FallbackRecommend { switch recommender { diff --git a/master/rest_test.go b/master/rest_test.go index d5cd35074..f0e4c2948 100644 --- a/master/rest_test.go +++ b/master/rest_test.go @@ -679,7 +679,7 @@ func TestServer_GetRecommends(t *testing.T) { s.GorseConfig.Recommend.FallbackRecommend = []string{"collaborative", "item_based", "user_based", "latest", "popular"} apitest.New(). Handler(s.handler). - Get("/api/dashboard/recommend/0/"). + Get("/api/dashboard/recommend/0/_"). Header("Cookie", cookie). Expect(t). Status(http.StatusOK). diff --git a/server/rest.go b/server/rest.go index 3ae22a1b6..7ba5250e8 100644 --- a/server/rest.go +++ b/server/rest.go @@ -806,9 +806,14 @@ func (s *RestServer) RecommendItemBased(ctx *recommendContext) error { start := time.Now() // truncate user feedback data.SortFeedbacks(ctx.userFeedback) - userFeedback := ctx.userFeedback - if s.GorseConfig.Recommend.NumFeedbackFallbackItemBased < len(userFeedback) { - userFeedback = userFeedback[:s.GorseConfig.Recommend.NumFeedbackFallbackItemBased] + userFeedback := make([]data.Feedback, 0, s.GorseConfig.Recommend.NumFeedbackFallbackItemBased) + for _, feedback := range ctx.userFeedback { + if s.GorseConfig.Recommend.NumFeedbackFallbackItemBased <= len(userFeedback) { + break + } + if funk.ContainsString(s.GorseConfig.Database.PositiveFeedbackType, feedback.FeedbackType) { + userFeedback = append(userFeedback, feedback) + } } // collect candidates candidates := make(map[string]float64) diff --git a/server/rest_test.go b/server/rest_test.go index a54b9f1a6..f24321736 100644 --- a/server/rest_test.go +++ b/server/rest_test.go @@ -1074,6 +1074,7 @@ func TestServer_GetRecommends_Replacement(t *testing.T) { func TestServer_GetRecommends_Fallback_ItemBasedSimilar(t *testing.T) { s := newMockServer(t) s.GorseConfig.Recommend.NumFeedbackFallbackItemBased = 4 + s.GorseConfig.Database.PositiveFeedbackType = []string{"a"} defer s.Close(t) // insert recommendation err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"),