Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: Syncano/pkg-go
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v0.1.1
Choose a base ref
...
head repository: Syncano/pkg-go
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref

Commits on May 25, 2020

  1. fix: remove orion dep

    23doors committed May 25, 2020
    Copy the full SHA
    17bce8f View commit details
  2. feat: rename rediscache

    23doors committed May 25, 2020
    Copy the full SHA
    1d8f3f7 View commit details
  3. add redis and db shutdown

    23doors committed May 25, 2020
    Copy the full SHA
    0496a33 View commit details

Commits on Jun 4, 2020

  1. Copy the full SHA
    45379dc View commit details

Commits on Jun 5, 2020

  1. Copy the full SHA
    f7338f8 View commit details

Commits on Jun 12, 2020

  1. Copy the full SHA
    f8e8f11 View commit details
  2. feat: change package names

    23doors committed Jun 12, 2020
    Copy the full SHA
    17d42da View commit details
  3. chore: querymgr -> manager

    23doors committed Jun 12, 2020
    Copy the full SHA
    f4f573f View commit details
  4. chore: manager separate

    23doors committed Jun 12, 2020
    Copy the full SHA
    23d6fbc View commit details
  5. feat: add manager dbcontext

    23doors committed Jun 12, 2020
    Copy the full SHA
    29b0577 View commit details
  6. chore: update golangci-lint

    23doors committed Jun 12, 2020
    Copy the full SHA
    088e4c2 View commit details

Commits on Jun 15, 2020

  1. Copy the full SHA
    cc08b15 View commit details
  2. Copy the full SHA
    61924c0 View commit details

Commits on Jun 16, 2020

  1. Copy the full SHA
    47253b4 View commit details
  2. fix: newdb instances opts nil

    23doors committed Jun 16, 2020
    Copy the full SHA
    cecfe3e View commit details

Commits on Jul 8, 2020

  1. feat: limiter info (#7)

    23doors authored Jul 8, 2020
    Copy the full SHA
    ae56702 View commit details

Commits on Jul 30, 2020

  1. Copy the full SHA
    1ebb215 View commit details
  2. fix: log options

    23doors committed Jul 30, 2020
    Copy the full SHA
    22fa9fa View commit details

Commits on Aug 4, 2020

  1. Copy the full SHA
    edea5c1 View commit details

Commits on Aug 10, 2020

  1. Copy the full SHA
    50430bd View commit details
  2. fix: re-add dbcontext()

    23doors committed Aug 10, 2020
    Copy the full SHA
    fc544df View commit details

Commits on Aug 13, 2020

  1. Copy the full SHA
    91b7989 View commit details
  2. fix: JSON as string

    23doors committed Aug 13, 2020
    Copy the full SHA
    ff68e9e View commit details

Commits on Aug 17, 2020

  1. Copy the full SHA
    6a0ecc0 View commit details

Commits on Aug 19, 2020

  1. Copy the full SHA
    b3649fa View commit details

Commits on Aug 21, 2020

  1. fix: nil cache support (#14)

    23doors authored Aug 21, 2020
    Copy the full SHA
    96b2575 View commit details

Commits on Aug 27, 2020

  1. feat: jobs system (#15)

    * feat: jobs system
    
    * lint
    23doors authored Aug 27, 2020
    Copy the full SHA
    1415298 View commit details

Commits on Aug 28, 2020

  1. Copy the full SHA
    ab750a8 View commit details
  2. feat: add addr support

    23doors committed Aug 28, 2020
    Copy the full SHA
    f3b36c8 View commit details
  3. Copy the full SHA
    71c1c04 View commit details
  4. Copy the full SHA
    37c806a View commit details
  5. fix: use milliseconds

    23doors committed Aug 28, 2020
    Copy the full SHA
    d79bb5d View commit details
Showing with 1,718 additions and 546 deletions.
  1. +1 −2 Makefile
  2. +61 −37 cache/cache.go
  3. +5 −5 cache/cache_test.go
  4. +24 −26 cache/lru.go
  5. +14 −5 cache/lru_set.go
  6. +3 −1 cache/lru_set_test.go
  7. +4 −3 cache/lru_test.go
  8. +1 −1 celery/celery_test.go
  9. +70 −23 {storage → database}/database.go
  10. +7 −7 {storage → database}/db_hooks.go
  11. +58 −0 database/fields/date.go
  12. +32 −0 database/fields/hstore.go
  13. +68 −0 database/fields/json.go
  14. +68 −0 database/fields/jsonb.go
  15. +50 −0 database/fields/time.go
  16. +6 −11 {storage → database}/interface.go
  17. +52 −0 database/manager/live_manager.go
  18. +120 −0 database/manager/manager.go
  19. +42 −0 database/manager/util.go
  20. +7 −7 {storage → database}/model_hooks.go
  21. +41 −0 database/wrapper.go
  22. +101 −0 echo_middleware/handlers.go
  23. +60 −0 echo_middleware/opencensus.go
  24. +23 −0 echo_middleware/request.go
  25. +25 −19 go.mod
  26. +148 −131 go.sum
  27. +24 −0 jobs/interface.go
  28. +88 −0 jobs/jobs.go
  29. +185 −0 jobs/runner.go
  30. +17 −0 limiter/info.go
  31. +48 −23 limiter/limiter.go
  32. +18 −9 limiter/limiter_test.go
  33. +49 −7 log/log.go
  34. +83 −59 rediscache/cache.go
  35. +32 −16 rediscache/func.go
  36. +1 −1 rediscache/interface.go
  37. +17 −17 rediscache/model.go
  38. +1 −1 {storage → rediscli}/pubsub.go
  39. +6 −2 {storage → rediscli}/redis.go
  40. +1 −1 redisdb/dbctx.go
  41. +1 −1 redisdb/table.go
  42. +3 −2 storage/data.go
  43. +1 −1 storage/data_gcloud.go
  44. +1 −1 storage/data_local.go
  45. +1 −1 storage/data_s3.go
  46. +0 −51 storage/mocks/DBContext.go
  47. +0 −45 storage/mocks/Databaser.go
  48. +1 −1 sys/systemcheck.go
  49. +39 −22 util/downloader.go
  50. +4 −4 util/downloader_test.go
  51. +1 −1 util/mocks/Downloader.go
  52. +5 −2 util/util.go
3 changes: 1 addition & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
CURRENTPACKAGE := github.com/Syncano/pkg-go
EXECNAME := orion

PATH := $(PATH):$(GOPATH)/bin
GOFILES=$(shell find . -mindepth 2 -type f -name '*.go' ! -path "./.*" ! -path "./dev/*" ! -path "*/proto/*")
@@ -28,7 +27,7 @@ lint: ## Run lint checks
echo "=== lint ==="
if ! hash golangci-lint 2>/dev/null; then \
echo "Installing golangci-lint"; \
curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $$(go env GOPATH)/bin v1.24.0; \
curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $$(go env GOPATH)/bin v1.27.0; \
fi
golangci-lint run $(ARGS)

98 changes: 61 additions & 37 deletions cache/cache.go
Original file line number Diff line number Diff line change
@@ -4,16 +4,14 @@ import (
"container/list"
"sync"
"time"

"github.com/imdario/mergo"
)

// Cache is an abstract raw struct used for LIFO-like LRU cache with ttl.
// As it provides no value access interface, it should be extended depending on what's needed.
// All exported methods should be thread-safe and use internal mutex.
// See LRUCache for example implementation.
type Cache struct {
options Options
cfg Config

// Maintain both valueMap and valuesList in sync.
// valueMap is used as a storage for key->list
@@ -29,18 +27,38 @@ type Cache struct {
deleteHandler DeleteHandler
}

// Options holds settable options for cache.
type Options struct {
// Config holds settable config for cache.
type Config struct {
TTL time.Duration
CleanupInterval time.Duration
Capacity int
}

var defaultOptions = &Options{
var DefaultConfig = Config{
TTL: 30 * time.Second,
CleanupInterval: 15 * time.Second,
}

type Option func(*Config)

func WithTTL(ttl time.Duration) Option {
return func(config *Config) {
config.TTL = ttl
}
}

func WithCleanupInterval(val time.Duration) Option {
return func(config *Config) {
config.CleanupInterval = val
}
}

func WithCapacity(c int) Option {
return func(config *Config) {
config.Capacity = c
}
}

// Item is used as a single element in cache.
type Item struct {
object interface{}
@@ -68,21 +86,21 @@ type DeleteHandler func(*valuesItem) *keyValue
type EvictionHandler func(string, interface{})

// Init initializes cache struct fields and starts janitor process.
func (c *Cache) Init(options *Options, deleteHandler DeleteHandler) {
func (c *Cache) Init(deleteHandler DeleteHandler, opts ...Option) {
c.mu.Lock()
defer c.mu.Unlock()

if c.janitor != nil {
panic("init on cache cannot be called twice")
}

if options != nil {
mergo.Merge(options, defaultOptions) // nolint - error not possible
} else {
options = defaultOptions
cfg := DefaultConfig

for _, opt := range opts {
opt(&cfg)
}

c.options = *options
c.cfg = cfg

if deleteHandler == nil {
deleteHandler = c.defaultDeleteHandler
@@ -92,16 +110,16 @@ func (c *Cache) Init(options *Options, deleteHandler DeleteHandler) {
c.valueMap = make(map[string]interface{})
c.valuesList = list.New()
c.janitor = &janitor{
interval: options.CleanupInterval,
interval: cfg.CleanupInterval,
stop: make(chan struct{}),
}

go c.janitor.Run(c)
}

// Options returns a copy of options struct.
func (c *Cache) Options() Options {
return c.options
// Config returns a copy of config struct.
func (c *Cache) Config() Config {
return c.cfg
}

// StopJanitor is meant to be called when cache is no longer needed to avoid leaking goroutine.
@@ -134,15 +152,24 @@ func (c *Cache) OnValueEvicted(f EvictionHandler) {
// Calls onValueEvicted.
func (c *Cache) DeleteLRU() bool {
c.mu.Lock()
defer c.mu.Unlock()

return c.deleteLRU()
evicted := c.deleteLRU()

c.mu.Unlock()

if evicted != nil {
c.handleEviction(evicted)

return true
}

return false
}

func (c *Cache) deleteLRU() bool {
func (c *Cache) deleteLRU() *keyValue {
e := c.valuesList.Front()
if e == nil {
return false
return nil
}

return c.delete(e)
@@ -179,33 +206,30 @@ func (c *Cache) DeleteByTime(now int64) {
evictedValues = append(evictedValues, valueEvicted)
}

c.muHandler.RLock()
if c.onValueEvicted != nil {
for _, v := range evictedValues {
c.onValueEvicted(v.key, v.value)
}
}
c.muHandler.RUnlock()
c.mu.Unlock()

c.handleEviction(evictedValues...)
}

func (c *Cache) defaultDeleteHandler(item *valuesItem) *keyValue {
delete(c.valueMap, item.key)
return &keyValue{key: item.key, value: item.item.object}
}

func (c *Cache) delete(e *list.Element) bool {
if v := c.deleteValue(e, 0); v != nil {
c.muHandler.RLock()
if c.onValueEvicted != nil {
func (c *Cache) delete(e *list.Element) *keyValue {
return c.deleteValue(e, 0)
}

func (c *Cache) handleEviction(evictedValues ...*keyValue) {
c.muHandler.RLock()

if c.onValueEvicted != nil {
for _, v := range evictedValues {
c.onValueEvicted(v.key, v.value)
}
c.muHandler.RUnlock()

return true
}

return false
c.muHandler.RUnlock()
}

// deleteValue deletes exactly one element from valuesList. If now != 0, it is checked against expiration time.
@@ -228,8 +252,8 @@ func (c *Cache) deleteValue(e *list.Element, now int64) (valueEvicted *keyValue)

func (c *Cache) checkLength() {
// If we are over the capacity, delete one closest to expiring.
if c.options.Capacity > 0 {
for c.valuesList.Len() > c.options.Capacity {
if c.cfg.Capacity > 0 {
for c.valuesList.Len() > c.cfg.Capacity {
c.deleteLRU()
}
}
10 changes: 5 additions & 5 deletions cache/cache_test.go
Original file line number Diff line number Diff line change
@@ -20,12 +20,12 @@ func TestCache(t *testing.T) {
c := new(Cache)

Convey("Init sets default delete handler and starts janitor", func() {
c.Init(&Options{}, nil)
c.Init(nil)
So(c.janitor, ShouldNotBeNil)
So(c.deleteHandler, ShouldEqual, c.defaultDeleteHandler)

Convey("and cannot be called twice", func() {
So(func() { c.Init(&Options{}, nil) }, ShouldPanic)
So(func() { c.Init(nil) }, ShouldPanic)
})
Convey("delete returns false when delete handler returns nil", func() {
c.deleteHandler = func(item *valuesItem) *keyValue {
@@ -42,9 +42,9 @@ func TestCache(t *testing.T) {
})
})

Convey("Options returns a copy of options struct", func() {
So(c.Options(), ShouldNotEqual, c.options)
So(c.Options(), ShouldResemble, c.options)
Convey("Config returns a copy of config struct", func() {
So(c.Config(), ShouldNotEqual, c.cfg)
So(c.Config(), ShouldResemble, c.cfg)
})

})
50 changes: 24 additions & 26 deletions cache/lru.go
Original file line number Diff line number Diff line change
@@ -2,46 +2,29 @@ package cache

import (
"time"

"github.com/imdario/mergo"
)

// LRUCache describes a struct for caching.
type LRUCache struct {
Cache
lruOptions LRUOptions
}

// LRUOptions holds settable options for cache.
type LRUOptions struct {
AutoRefresh bool
}

var defaultLRUOptions = &LRUOptions{
AutoRefresh: true,
autoRefresh bool
}

// NewLRUCache creates and initializes a new cache object.
// This one is based on LRU KV with TTL
func NewLRUCache(options *Options, lruOptions *LRUOptions) *LRUCache {
if lruOptions != nil {
mergo.Merge(lruOptions, defaultLRUOptions) // nolint - error not possible
} else {
lruOptions = defaultLRUOptions
}

func NewLRUCache(autoRefresh bool, opts ...Option) *LRUCache {
cache := LRUCache{
lruOptions: *lruOptions,
autoRefresh: autoRefresh,
}

cache.Cache.Init(options, cache.deleteHandler)
cache.Cache.Init(cache.deleteHandler, opts...)

return &cache
}

// Get returns an item at given key. It automatically extends the expiration if auto refresh is true. Returns the item or nil.
func (c *LRUCache) Get(key string) interface{} {
return c.get(key, c.lruOptions.AutoRefresh)
return c.get(key, c.autoRefresh)
}

func (c *LRUCache) get(key string, refresh bool) interface{} {
@@ -73,7 +56,7 @@ func (c *LRUCache) Refresh(key string) bool {

func (c *LRUCache) set(key string, val interface{}, ttl time.Duration) {
if ttl == 0 {
ttl = c.options.TTL
ttl = c.cfg.TTL
}

cItem := &Item{object: val, expiration: time.Now().Add(ttl).UnixNano(), ttl: ttl}
@@ -90,15 +73,21 @@ func (c *LRUCache) Set(key string, val interface{}) {
}

func (c *LRUCache) SetTTL(key string, val interface{}, ttl time.Duration) {
var evicted *keyValue

c.mu.Lock()

curVal, ok := c.valueMap[key]
if ok {
c.delete(curVal.(*Item).valuesListElement)
evicted = c.delete(curVal.(*Item).valuesListElement)
}

c.set(key, val, ttl)
c.mu.Unlock()

if evicted != nil {
c.handleEviction(evicted)
}
}

// Add assigns a new value to an item at given key if it doesn't exist.
@@ -129,13 +118,22 @@ func (c *LRUCache) AddTTL(key string, val interface{}, ttl time.Duration) bool {
// Delete removes an item at given key.
func (c *LRUCache) Delete(key string) bool {
c.mu.Lock()
defer c.mu.Unlock()

curVal, ok := c.valueMap[key]
if ok {
return c.delete(curVal.(*Item).valuesListElement)
evicted := c.delete(curVal.(*Item).valuesListElement)

c.mu.Unlock()

if evicted != nil {
c.handleEviction(evicted)

return true
}
}

c.mu.Unlock()

return false
}

19 changes: 14 additions & 5 deletions cache/lru_set.go
Original file line number Diff line number Diff line change
@@ -11,9 +11,9 @@ type LRUSetCache struct {

// NewLRUSetCache creates and initializes a new cache object.
// This one is based on LRU K->List with TTL
func NewLRUSetCache(options *Options) *LRUSetCache {
func NewLRUSetCache(opts ...Option) *LRUSetCache {
cache := LRUSetCache{}
cache.Cache.Init(options, cache.deleteHandler)
cache.Cache.Init(cache.deleteHandler, opts...)

return &cache
}
@@ -77,7 +77,7 @@ func (c *LRUSetCache) Add(key string, val interface{}) bool {

func (c *LRUSetCache) AddTTL(key string, val interface{}, ttl time.Duration) bool {
if ttl == 0 {
ttl = c.options.TTL
ttl = c.cfg.TTL
}

c.mu.Lock()
@@ -111,15 +111,24 @@ func (c *LRUSetCache) AddTTL(key string, val interface{}, ttl time.Duration) boo

// Delete removes an item at given key.
func (c *LRUSetCache) Delete(key string, val interface{}) bool {
var evicted *keyValue

c.mu.Lock()
defer c.mu.Unlock()

if curVal, ok := c.valueMap[key]; ok {
if v, ok := curVal.(map[interface{}]*Item)[val]; ok {
return c.delete(v.valuesListElement)
evicted = c.delete(v.valuesListElement)
}
}

c.mu.Unlock()

if evicted != nil {
c.handleEviction(evicted)

return true
}

return false
}

Loading