From 132ec36d058d8bd75d879aa7f366f11a09862d0d Mon Sep 17 00:00:00 2001 From: Fangxin Lou Date: Mon, 6 Jan 2025 10:02:44 +0800 Subject: [PATCH 1/4] add more connection options to sql meta engine --- cmd/flags.go | 20 ++++++++++++++++++++ cmd/mount.go | 5 +++++ pkg/meta/config.go | 5 +++++ pkg/meta/sql.go | 18 ++++++++++++++++-- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index 5b94095e55dd..b4be95c29f64 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -297,6 +297,26 @@ func metaFlags() []cli.Flag { Name: "sort-dir", Usage: "sort entries within a directory by name", }, + &cli.IntFlag{ + Name: "max-open-conns", + Value: 0, + Usage: "maximum number of connections allowed for meta engine (used for SQL engine only)", + }, + &cli.IntFlag{ + Name: "max-idle-conns", + Value: 10, + Usage: "maximum number of idle connections allowed for meta engine (used for SQL engine only)", + }, + &cli.IntFlag{ + Name: "max-idle-time", + Value: 300, + Usage: "maximum time allowed for a idle connection to be closed (used for SQL engine only)", + }, + &cli.IntFlag{ + Name: "max-life-time", + Value: 3600, + Usage: "maximum life time allowed for a connection to be closed (used for SQL engine only)", + }, }) } diff --git a/cmd/mount.go b/cmd/mount.go index bde167c91048..15d1f7f4310a 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -317,6 +317,11 @@ func getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config { conf.Sid, _ = strconv.ParseUint(os.Getenv("_JFS_META_SID"), 10, 64) conf.SortDir = c.Bool("sort-dir") + conf.SqlMaxOpenConns = c.Int("max-open-conns") + conf.SqlMaxIdleConns = c.Int("max-idle-conns") + conf.SqlMaxIdleTime = c.Int("max-idle-time") + conf.SqlMaxLifeTime = c.Int("max-life-time") + atimeMode := c.String("atime-mode") if atimeMode != meta.RelAtime && atimeMode != meta.StrictAtime && atimeMode != meta.NoAtime { logger.Warnf("unknown atime-mode \"%s\", changed to %s", atimeMode, meta.NoAtime) diff --git a/pkg/meta/config.go b/pkg/meta/config.go index 950b15eb4b2f..2285444df4a5 100644 --- a/pkg/meta/config.go +++ b/pkg/meta/config.go @@ -50,6 +50,11 @@ type Config struct { SkipDirMtime time.Duration Sid uint64 SortDir bool + + SqlMaxOpenConns int + SqlMaxIdleConns int + SqlMaxIdleTime int + SqlMaxLifeTime int } func DefaultConf() *Config { diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index c43c7454e6a3..bba2dc70903d 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -325,8 +325,22 @@ func newSQLMeta(driver, addr string, conf *Config) (Meta, error) { if searchPath != "" { engine.SetSchema(searchPath) } - engine.DB().SetMaxIdleConns(runtime.GOMAXPROCS(-1) * 2) - engine.DB().SetConnMaxIdleTime(time.Minute * 5) + if conf.SqlMaxOpenConns > 0 { + engine.DB().SetMaxOpenConns(conf.SqlMaxOpenConns) + } + if conf.SqlMaxIdleConns > 0 { + engine.DB().SetMaxIdleConns(conf.SqlMaxIdleConns) + } else { + engine.DB().SetMaxIdleConns(runtime.GOMAXPROCS(-1) * 2) + } + if conf.SqlMaxIdleTime > 0 { + engine.DB().SetConnMaxIdleTime(time.Second * time.Duration(conf.SqlMaxIdleTime)) + } else { + engine.DB().SetConnMaxIdleTime(time.Minute * 5) + } + if conf.SqlMaxLifeTime > 0 { + engine.DB().SetConnMaxLifetime(time.Second * time.Duration(conf.SqlMaxLifeTime)) + } engine.SetTableMapper(names.NewPrefixMapper(engine.GetTableMapper(), "jfs_")) m := &dbMeta{ baseMeta: newBaseMeta(addr, conf), From a7c9cc086b02f31851305e8b9b14066d6ed056d0 Mon Sep 17 00:00:00 2001 From: Fangxin Lou Date: Mon, 6 Jan 2025 15:27:44 +0800 Subject: [PATCH 2/4] fix pass conn options through meta url --- cmd/flags.go | 20 ---------------- cmd/mount.go | 5 ---- pkg/meta/config.go | 5 ---- pkg/meta/sql.go | 59 +++++++++++++++++++++++++++++++++++++++------- 4 files changed, 51 insertions(+), 38 deletions(-) diff --git a/cmd/flags.go b/cmd/flags.go index b4be95c29f64..5b94095e55dd 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -297,26 +297,6 @@ func metaFlags() []cli.Flag { Name: "sort-dir", Usage: "sort entries within a directory by name", }, - &cli.IntFlag{ - Name: "max-open-conns", - Value: 0, - Usage: "maximum number of connections allowed for meta engine (used for SQL engine only)", - }, - &cli.IntFlag{ - Name: "max-idle-conns", - Value: 10, - Usage: "maximum number of idle connections allowed for meta engine (used for SQL engine only)", - }, - &cli.IntFlag{ - Name: "max-idle-time", - Value: 300, - Usage: "maximum time allowed for a idle connection to be closed (used for SQL engine only)", - }, - &cli.IntFlag{ - Name: "max-life-time", - Value: 3600, - Usage: "maximum life time allowed for a connection to be closed (used for SQL engine only)", - }, }) } diff --git a/cmd/mount.go b/cmd/mount.go index 15d1f7f4310a..bde167c91048 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -317,11 +317,6 @@ func getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config { conf.Sid, _ = strconv.ParseUint(os.Getenv("_JFS_META_SID"), 10, 64) conf.SortDir = c.Bool("sort-dir") - conf.SqlMaxOpenConns = c.Int("max-open-conns") - conf.SqlMaxIdleConns = c.Int("max-idle-conns") - conf.SqlMaxIdleTime = c.Int("max-idle-time") - conf.SqlMaxLifeTime = c.Int("max-life-time") - atimeMode := c.String("atime-mode") if atimeMode != meta.RelAtime && atimeMode != meta.StrictAtime && atimeMode != meta.NoAtime { logger.Warnf("unknown atime-mode \"%s\", changed to %s", atimeMode, meta.NoAtime) diff --git a/pkg/meta/config.go b/pkg/meta/config.go index 2285444df4a5..950b15eb4b2f 100644 --- a/pkg/meta/config.go +++ b/pkg/meta/config.go @@ -50,11 +50,6 @@ type Config struct { SkipDirMtime time.Duration Sid uint64 SortDir bool - - SqlMaxOpenConns int - SqlMaxIdleConns int - SqlMaxIdleTime int - SqlMaxLifeTime int } func DefaultConf() *Config { diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index bba2dc70903d..50c631b2df82 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -266,10 +266,53 @@ func recoveryMysqlPwd(addr string) string { return addr } +func retriveUrlConnsOptions(murl string) (string, int, int, int, int) { + optIndex := strings.Index(murl, "?") + + var vOpenConns int = 0 + var vIdleConns int = 10 + var vIdleTime int = 300 + var vLifeTime int = 3600 + + if optIndex != -1 { + baseurl := murl[:optIndex] + optsurl := murl[optIndex+1:] + if vals, err := url.ParseQuery(optsurl); err == nil { + if vals.Has("MaxOpenConns") { + vOpenConns, _ = strconv.Atoi(vals.Get("MaxOpenConns")) + vals.Del("MaxOpenConns") + } + if vals.Has("MaxIdleConns") { + vIdleConns, _ = strconv.Atoi(vals.Get("MaxIdleConns")) + vals.Del("MaxIdleConns") + } + if vals.Has("MaxIdleTime") { + vIdleTime, _ = strconv.Atoi(vals.Get("MaxIdleTime")) + vals.Del("MaxIdleTime"); + } + if vals.Has("MaxLifeTime") { + vLifeTime, _ = strconv.Atoi(vals.Get("MaxLifeTime")) + vals.Del("MaxLifeTime"); + } + optsurl = vals.Encode() + } + if optsurl != "" { + return fmt.Sprintf("%s?%s", baseurl, optsurl), vOpenConns, vIdleConns, vIdleTime, vLifeTime + } else { + return baseurl, vOpenConns, vIdleConns, vIdleTime, vLifeTime + } + } + + return murl, vOpenConns, vIdleConns, vIdleTime, vLifeTime +} + var setTransactionIsolation func(dns string) (string, error) func newSQLMeta(driver, addr string, conf *Config) (Meta, error) { var searchPath string + + addr, vOpenConns, vIdleConns, vIdleTime, vLifeTime := retriveUrlConnsOptions(addr) + if driver == "postgres" { addr = driver + "://" + addr driver = "pgx" @@ -325,21 +368,21 @@ func newSQLMeta(driver, addr string, conf *Config) (Meta, error) { if searchPath != "" { engine.SetSchema(searchPath) } - if conf.SqlMaxOpenConns > 0 { - engine.DB().SetMaxOpenConns(conf.SqlMaxOpenConns) + if vOpenConns > 0 { + engine.DB().SetMaxOpenConns(vOpenConns) } - if conf.SqlMaxIdleConns > 0 { - engine.DB().SetMaxIdleConns(conf.SqlMaxIdleConns) + if vIdleConns > 0 { + engine.DB().SetMaxIdleConns(vIdleConns) } else { engine.DB().SetMaxIdleConns(runtime.GOMAXPROCS(-1) * 2) } - if conf.SqlMaxIdleTime > 0 { - engine.DB().SetConnMaxIdleTime(time.Second * time.Duration(conf.SqlMaxIdleTime)) + if vIdleTime > 0 { + engine.DB().SetConnMaxIdleTime(time.Second * time.Duration(vIdleTime)) } else { engine.DB().SetConnMaxIdleTime(time.Minute * 5) } - if conf.SqlMaxLifeTime > 0 { - engine.DB().SetConnMaxLifetime(time.Second * time.Duration(conf.SqlMaxLifeTime)) + if vLifeTime > 0 { + engine.DB().SetConnMaxLifetime(time.Second * time.Duration(vLifeTime)) } engine.SetTableMapper(names.NewPrefixMapper(engine.GetTableMapper(), "jfs_")) m := &dbMeta{ From 8794f76bb2069e5081783ab61e5f9e98bd7b3574 Mon Sep 17 00:00:00 2001 From: Fangxin Lou Date: Mon, 6 Jan 2025 15:51:08 +0800 Subject: [PATCH 3/4] fix rename the options --- pkg/meta/sql.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 50c631b2df82..ae1a7e250e26 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -278,21 +278,21 @@ func retriveUrlConnsOptions(murl string) (string, int, int, int, int) { baseurl := murl[:optIndex] optsurl := murl[optIndex+1:] if vals, err := url.ParseQuery(optsurl); err == nil { - if vals.Has("MaxOpenConns") { - vOpenConns, _ = strconv.Atoi(vals.Get("MaxOpenConns")) - vals.Del("MaxOpenConns") + if vals.Has("max_open_conns") { + vOpenConns, _ = strconv.Atoi(vals.Get("max_open_conns")) + vals.Del("max_open_conns") } - if vals.Has("MaxIdleConns") { - vIdleConns, _ = strconv.Atoi(vals.Get("MaxIdleConns")) - vals.Del("MaxIdleConns") + if vals.Has("max_idle_conns") { + vIdleConns, _ = strconv.Atoi(vals.Get("max_idle_conns")) + vals.Del("max_idle_conns") } - if vals.Has("MaxIdleTime") { - vIdleTime, _ = strconv.Atoi(vals.Get("MaxIdleTime")) - vals.Del("MaxIdleTime"); + if vals.Has("max_idle_time") { + vIdleTime, _ = strconv.Atoi(vals.Get("max_idle_time")) + vals.Del("max_idle_time"); } - if vals.Has("MaxLifeTime") { - vLifeTime, _ = strconv.Atoi(vals.Get("MaxLifeTime")) - vals.Del("MaxLifeTime"); + if vals.Has("max_life_time") { + vLifeTime, _ = strconv.Atoi(vals.Get("max_life_time")) + vals.Del("max_life_time"); } optsurl = vals.Encode() } From 9d9102b8c85720f8eccc2a0014071fd5e49e4a18 Mon Sep 17 00:00:00 2001 From: Fangxin Lou Date: Tue, 7 Jan 2025 11:20:04 +0800 Subject: [PATCH 4/4] fix restor the old default values --- pkg/meta/sql.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index ae1a7e250e26..b031c9677b04 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -270,9 +270,9 @@ func retriveUrlConnsOptions(murl string) (string, int, int, int, int) { optIndex := strings.Index(murl, "?") var vOpenConns int = 0 - var vIdleConns int = 10 + var vIdleConns int = runtime.GOMAXPROCS(-1) * 2 var vIdleTime int = 300 - var vLifeTime int = 3600 + var vLifeTime int = 0 if optIndex != -1 { baseurl := murl[:optIndex] @@ -296,6 +296,12 @@ func retriveUrlConnsOptions(murl string) (string, int, int, int, int) { } optsurl = vals.Encode() } + if vIdleConns <= 0 { + vIdleConns = runtime.GOMAXPROCS(-1) * 2 + } + if vIdleTime <= 0 { + vIdleTime = 300 + } if optsurl != "" { return fmt.Sprintf("%s?%s", baseurl, optsurl), vOpenConns, vIdleConns, vIdleTime, vLifeTime } else { @@ -371,19 +377,11 @@ func newSQLMeta(driver, addr string, conf *Config) (Meta, error) { if vOpenConns > 0 { engine.DB().SetMaxOpenConns(vOpenConns) } - if vIdleConns > 0 { - engine.DB().SetMaxIdleConns(vIdleConns) - } else { - engine.DB().SetMaxIdleConns(runtime.GOMAXPROCS(-1) * 2) - } - if vIdleTime > 0 { - engine.DB().SetConnMaxIdleTime(time.Second * time.Duration(vIdleTime)) - } else { - engine.DB().SetConnMaxIdleTime(time.Minute * 5) - } if vLifeTime > 0 { engine.DB().SetConnMaxLifetime(time.Second * time.Duration(vLifeTime)) } + engine.DB().SetMaxIdleConns(vIdleConns) + engine.DB().SetConnMaxIdleTime(time.Second * time.Duration(vIdleTime)) engine.SetTableMapper(names.NewPrefixMapper(engine.GetTableMapper(), "jfs_")) m := &dbMeta{ baseMeta: newBaseMeta(addr, conf),