fix database version (#334)

gorse-io · Dec 14, 2021 · 2c95589 · 2c95589
1 parent 8aa5c53
commit 2c95589
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 17 deletions.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -15,15 +15,15 @@ jobs:
 
     services:
       mysql:
-        image: mysql
+        image: mysql:8.0
         ports:
           - 3306
         env:
           MYSQL_ROOT_PASSWORD: password
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
 
       postgres:
-        image: postgres
+        image: postgres:10.0
         ports:
           - 5432
         env:
@@ -36,18 +36,28 @@ jobs:
           --health-retries 5
 
       mongo:
-        image: healthcheck/mongo
+        image: mongo:4.0
         ports:
           - 27017
         env:
           MONGO_INITDB_ROOT_USERNAME: root
           MONGO_INITDB_ROOT_PASSWORD: password
+        options: >-
+          --health-cmd mongo
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
       clickhouse:
-        image: yandex/clickhouse-server
+        image: yandex/clickhouse-server:21.10
         ports:
           - 8123
 
+      redis:
+        image: redis:5.0
+        ports:
+          - 6379
+
     steps:
     - name: Set up dataset
       run: |
@@ -95,6 +105,8 @@ jobs:
         MONGO_URI: mongodb://root:password@localhost:${{ job.services.mongo.ports[27017] }}/
         # ClickHouse
         CLICKHOUSE_URI: clickhouse://localhost:${{ job.services.clickhouse.ports[8123] }}/
+        # Redis
+        REDIS_URI: redis://localhost:${{ job.services.redis.ports[6379] }}/
 
     - name: Upload 
       run: bash <(curl -s https://codecov.io/bash)
@@ -133,12 +145,17 @@ jobs:
 
     services:
       mariadb:
-        image: mariadb
+        image: mariadb:10.2
         ports:
           - 3306
         env:
           MYSQL_ROOT_PASSWORD: password
 
+      pika:
+        image: zhenghaoz/pika
+        ports:
+          - 9221
+
     steps:
       - name: Set up Go 1.17.x
         uses: actions/setup-go@v2
@@ -153,3 +170,8 @@ jobs:
         run: go test ./storage/data -run ^TestMySQL_
         env:
           MYSQL_URI: mysql://root:password@tcp(localhost:${{ job.services.mariadb.ports[3306] }})/
+
+      - name: Test Pika
+        run: go test ./storage/cache -run ^TestRedis_
+        env:
+          REDIS_URI: redis://localhost:${{ job.services.pika.ports[9221] }}/
diff --git a/cmd/README.md b/cmd/README.md
@@ -7,7 +7,17 @@ Binary distributions have been provided for 64-bit Windows/Linux/Mac OS on the [
 Gorse depends on following software:
 
 - *Redis* is used to store caches.
-- *MySQL/MongoDB* is used to store data.
+- One of *MySQL/PostgresSQL/ClickHouse/MongoDB* is used to store data.
+
+The minimal versions of dependent software are as follows:
+
+| Software    | Minimal Version |
+|-------------|-----------------|
+| Redis       | 5.0             |
+| MySQL       | 5.7             |
+| PostgresSQL | 10.0            |
+| ClickHouse  | 21.10           |
+| MongoDB     | 4.0             |
 
 ## Run Gorse
 

diff --git a/docker/README.md b/docker/README.md
@@ -8,6 +8,23 @@ The best practice to manage Gorse nodes is using orchestration tools such as Doc
 | gorse-server | [![](https://img.shields.io/docker/image-size/zhenghaoz/gorse-server)](https://hub.docker.com/repository/docker/zhenghaoz/gorse-server) |
 | gorse-worker | [![](https://img.shields.io/docker/image-size/zhenghaoz/gorse-worker)](https://hub.docker.com/repository/docker/zhenghaoz/gorse-worker) |
 
+## Prerequisite
+
+Gorse depends on following software:
+
+- *Redis* is used to store caches.
+- One of *MySQL/PostgresSQL/ClickHouse/MongoDB* is used to store data.
+
+The minimal versions of dependent software are as follows:
+
+| Software    | Minimal Version |
+|-------------|-----------------|
+| Redis       | 5.0             |
+| MySQL       | 5.7             |
+| PostgresSQL | 10.0            |
+| ClickHouse  | 21.10           |
+| MongoDB     | 4.0             |
+
 ## Quick Start
 
 There is an example [docker-compose.yml](https://github.com/zhenghaoz/gorse/blob/master/docker/docker-compose.yml) consists of a master node, a server node and a worker node, a Redis instance, and a MySQL instance.

diff --git a/misc/database_test/docker-compose.yml b/misc/database_test/docker-compose.yml
@@ -1,34 +1,34 @@
 version: "3"
 services:
   redis:
-    image: redis
+    image: redis:5.0
     ports:
       - 6379:6379
 
   mysql:
-    image: mysql
+    image: mysql:8.0
     ports:
       - 3306:3306
     environment:
       MYSQL_ROOT_PASSWORD: password
 
   postgres:
-    image: postgres
+    image: postgres:10.0
     ports:
       - 5432:5432
     environment:
       POSTGRES_USER: gorse
       POSTGRES_PASSWORD: gorse_pass
 
   mongo:
-    image: healthcheck/mongo
+    image: mongo:4.0
     ports:
       - 27017:27017
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: password
 
   clickhouse:
-    image: yandex/clickhouse-server
+    image: yandex/clickhouse-server:21.10
     ports:
       - 8123:8123
diff --git a/storage/cache/redis.go b/storage/cache/redis.go
@@ -132,7 +132,12 @@ func (r *Redis) AppendScores(prefix, name string, items ...Scored) error {
 func (r *Redis) PopScores(prefix, name string, n int) error {
 	var ctx = context.Background()
 	key := prefix + "/" + name
-	return r.client.LPopCount(ctx, key, n).Err()
+	for i := 0; i < n; i++ {
+		if err := r.client.LPop(ctx, key).Err(); err != nil {
+			return errors.Trace(err)
+		}
+	}
+	return nil
 }
 
 // GetString returns a string from Redis.

diff --git a/storage/cache/redis_test.go b/storage/cache/redis_test.go
@@ -15,30 +15,40 @@
 package cache
 
 import (
-	"github.com/alicebob/miniredis/v2"
 	"github.com/stretchr/testify/assert"
+	"os"
 	"testing"
 )
 
+var redisDSN string
+
+func init() {
+	// get environment variables
+	env := func(key, defaultValue string) string {
+		if value := os.Getenv(key); value != "" {
+			return value
+		}
+		return defaultValue
+	}
+	redisDSN = env("REDIS_URI", "redis://127.0.0.1:6379/")
+}
+
 type mockRedis struct {
 	Database
-	server *miniredis.Miniredis
 }
 
 func newMockRedis(t *testing.T) *mockRedis {
 	var err error
 	db := new(mockRedis)
-	db.server, err = miniredis.Run()
 	assert.NoError(t, err)
-	db.Database, err = Open(redisPrefix + db.server.Addr())
+	db.Database, err = Open(redisDSN)
 	assert.NoError(t, err)
 	return db
 }
 
 func (db *mockRedis) Close(t *testing.T) {
 	err := db.Database.Close()
 	assert.NoError(t, err)
-	db.server.Close()
 }
 
 func TestRedis_Meta(t *testing.T) {