feat(sharding): added shardid (#12)

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - added `BitBool` for mysql bit type (#11)
+- added `sharding` feature (#12)
+
+### Fixed
+- fixed parameterized placeholder for postgresql(#12)
 
 ## [1.1.0] - 2024-02-13
 ### Added

README.md

@@ -25,6 +25,19 @@ You’ll find the SQLE package useful if you’re not a fan of full-featured ORM
 
 ## Tutorials
 > All examples on https://go.dev/doc/tutorial/database-access can directly work with `sqle.DB` instance. 
+>
+
+### Install SQLE
+- install latest commit from `main` branch
+```
+go get github.com/yaitoo/sqle@main
+```
+
+- install latest release
+```
+go get github.com/yaitoo/sqle@latest
+```
+
 ### Connecting to a Database
 SQLE directly connects to a database by `sql.DB` instance. 
 ```
@@ -235,7 +248,7 @@ func albumsByArtist(name string) ([]Album, error) {
     // An albums slice to hold data from returned rows.
     var albums []Album
 
-    cmd := sql.New("SELECT * FROM album").Where().
+    cmd := sql.New().Select("album").Where().
         If(name != "").And("artist = {artist}").
         Param("artist",name)
 
@@ -291,7 +304,7 @@ func albumByID(id int64) (Album, error) {
 func albumByID(id int64) (Album, error) {
     // An album to hold data from the returned row.
     var alb Album
-    cmd := sqle.New("SELECT * FROM album").
+    cmd := sqle.New().Select("album").
         Where("id = {id}").
         Param("id",id)
 
@@ -383,7 +396,7 @@ func deleteAlbumByID(id int64) error {
 - delete album by named sql statement
 ```
 func deleteAlbumByID(id int64) error {
-    _, err := db.ExecBuilder(context.TODO(), sqle.New("DELETE FROM album WHERE id = {id}").
+    _, err := db.ExecBuilder(context.TODO(), sqle.New().Delete("album").Where("id = {id}").
         Param("id",id))
 
     return err 
@@ -395,7 +408,7 @@ perform a set of operations within a transaction
 ```
 func deleteAlbums(ids []int64) error {
 
-    return db.Transaction(ctx, &sql.TxOptions{}, func(tx *sqle.Tx) error {
+    return db.Transaction(ctx, &sql.TxOptions{}, func(ctx context.Context,tx *sqle.Tx) error {
         var err error
         for _, id := range ids {
             _, err = tx.Exec("DELETE FROM album WHERE id=?",id)

sharding/README.md

@@ -0,0 +1,49 @@
+# sharding
+
+## sid-64-bit
+// +----------+-------------------+------------+----------------+----------------------+---------------+
+// | signed 1 |      millis (39)  |  worker(2) | db-sharding(10)|   table-rotate(2)    | sequence(10)  |
+// +----------+-------------------+------------+----------------+----------------------+---------------+
+                39 = 17 years        2 = 4      10=1024         0: none    :table         10=1024
+                                                                1: monthly :table-[YYYYMM]
+                                                                2: weekly  :table-[YYYY0XX]
+                                                                3: daily   :table-[YYYYMMDD]
+- signed(1): sid is always positive number
+- millis(39): 2^39 (17years) unix milliseconds since 2024-02-19 00:00:00
+- workers(4): 2^4(16) workers
+- db-sharding(10): 2^10 (1024) db instances
+- table-rotate(2): 2^2(4) table rotate: none/by year/by month/by day
+- sequence(10): 2^10(1024) per milliseconds
+
+## TPS:
+  - ID: 1000(ms)*1024(seq)*4 = 4096000  409.6W/s
+        1000*1024            = 1024000  102.4W/s
+
+  - DB : 
+        10   * 1000   =   10000       1W/s
+        1024 * 1000   = 1024000   102.4W/s
+
+        10   * 2000   =   20000       2W/s
+        1024 * 2000   = 2048000   204.8W/s
+
+        10   * 3000   =   30000       3W/s
+        1024 * 3000   = 3072000   307.2W/s
+
+## mysql-benchmark 
+  - https://docs.aws.amazon.com/whitepapers/latest/optimizing-mysql-on-ec2-using-amazon-ebs/mysql-benchmark-observations-and-considerations.html
+  - https://github.com/MinervaDB/MinervaDB-Sysbench
+  - https://www.percona.com/blog/assessing-mysql-performance-amongst-aws-options-part-one/
+
+## issues
+- Overflow capacity
+  waiting for next microsecond.
+
+- System Clock Dependency
+  You should use NTP to keep your system clock accurate.
+
+- Time move backwards   
+  + if sequence doesn't overflow, let's use last timestamp and next sequence. system clock might moves forward and greater than last timestamp on next id generation 
+  + if sequence overflows, and has to be reset. let's built-in clock to get timestamp till system clock moves forward and greater than built-in clock
+
+- Built-in clock
+  record last timestamp in memory/database, increase it when it is requested to send current timestamp instead of system clock

sharding/generator.go

@@ -0,0 +1,101 @@
+package sharding
+
+import (
+	"sync"
+	"time"
+)
+
+type Generator struct {
+	sync.Mutex
+	_ noCopy // nolint: unused
+
+	workerID      int8
+	databaseTotal int16
+	tableRotate   TableRotate
+	now           func() time.Time
+
+	lastMillis     int64
+	nextSequence   int16
+	nextDatabaseID int16
+}
+
+func New(options ...Option) *Generator {
+	g := &Generator{
+		now:           time.Now,
+		databaseTotal: 1,
+		tableRotate:   None,
+		workerID:      acquireWorkerID(),
+	}
+	for _, option := range options {
+		option(g)
+	}
+	return g
+}
+
+func (g *Generator) Next() int64 {
+	g.Lock()
+
+	defer func() {
+		g.nextSequence++
+		g.Unlock()
+	}()
+
+	nowMillis := g.now().UnixMilli()
+	if nowMillis < g.lastMillis {
+		if g.nextSequence > MaxSequence {
+			// time move backwards,and sequence overflows capacity, waiting system clock to move forward
+			g.nextSequence = 0
+			nowMillis = g.tillNextMillis()
+		} else {
+			// time move backwards,but sequence doesn't overflow capacity, use Built-in clock to move forward
+			nowMillis = g.moveNextMillis()
+		}
+	}
+
+	// sequence overflows capacity
+	if g.nextSequence > MaxSequence {
+		if nowMillis == g.lastMillis {
+			nowMillis = g.tillNextMillis()
+		}
+
+		g.nextSequence = 0
+	}
+
+	g.lastMillis = nowMillis
+
+	return Build(nowMillis, g.workerID, g.getNextDatabaseID(), g.tableRotate, g.nextSequence)
+
+}
+
+func (g *Generator) getNextDatabaseID() int16 {
+	if g.databaseTotal <= 1 {
+		return 0
+	}
+
+	defer func() {
+		g.nextDatabaseID++
+	}()
+
+	if g.nextDatabaseID < g.databaseTotal {
+		return g.nextDatabaseID
+	}
+
+	g.nextDatabaseID = 0
+	return 0
+}
+
+func (g *Generator) tillNextMillis() int64 {
+	lastMillis := g.now().UnixMilli()
+	for {
+		if lastMillis > g.lastMillis {
+			break
+		}
+
+		lastMillis = g.now().UnixMilli()
+	}
+
+	return lastMillis
+}
+func (g *Generator) moveNextMillis() int64 {
+	return g.lastMillis + 1
+}