avoid bigint converted into float64 when unmarshaling

* rebase upstream

* rebase

* trim no need line

* trim no need line

* trim no need line

* update doc

* remove update

* remove no need

* remove no need

* goctl add jwt support

* goctl add jwt support

* goctl add jwt support

* goctl support import

* goctl support import

* support return ()

* revert

* refactor and rename folder to group

* parser add test

Co-authored-by: kingxt <dream4kingxt@163.com>

.github/workflows/go.yml

@@ -7,7 +7,6 @@ on:
     branches: [ master ]
 
 jobs:
-
   build:
     name: Build
     runs-on: ubuntu-latest

core/bloom/bloom.go

@@ -13,15 +13,13 @@ const (
 	// maps as k in the error rate table
 	maps      = 14
 	setScript = `
-local key = KEYS[1]
 for _, offset in ipairs(ARGV) do
-	redis.call("setbit", key, offset, 1)
+	redis.call("setbit", KEYS[1], offset, 1)
 end
 `
 	testScript = `
-local key = KEYS[1]
 for _, offset in ipairs(ARGV) do
-	if tonumber(redis.call("getbit", key, offset)) == 0 then
+	if tonumber(redis.call("getbit", KEYS[1], offset)) == 0 then
 		return false
 	end
 end

core/breaker/breaker.go

@@ -5,17 +5,12 @@ import (
 	"fmt"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/tal-tech/go-zero/core/mathx"
 	"github.com/tal-tech/go-zero/core/proc"
 	"github.com/tal-tech/go-zero/core/stat"
 	"github.com/tal-tech/go-zero/core/stringx"
-)
-
-const (
-	StateClosed State = iota
-	StateOpen
+	"github.com/tal-tech/go-zero/core/timex"
 )
 
 const (
@@ -27,7 +22,6 @@ const (
 var ErrServiceUnavailable = errors.New("circuit breaker is open")
 
 type (
-	State      = int32
 	Acceptable func(err error) bool
 
 	Breaker interface {
@@ -195,23 +189,23 @@ type errorWindow struct {
 
 func (ew *errorWindow) add(reason string) {
 	ew.lock.Lock()
-	ew.reasons[ew.index] = fmt.Sprintf("%s %s", time.Now().Format(timeFormat), reason)
+	ew.reasons[ew.index] = fmt.Sprintf("%s %s", timex.Time().Format(timeFormat), reason)
 	ew.index = (ew.index + 1) % numHistoryReasons
 	ew.count = mathx.MinInt(ew.count+1, numHistoryReasons)
 	ew.lock.Unlock()
 }
 
 func (ew *errorWindow) String() string {
-	var builder strings.Builder
+	var reasons []string
 
 	ew.lock.Lock()
-	for i := ew.index + ew.count - 1; i >= ew.index; i-- {
-		builder.WriteString(ew.reasons[i%numHistoryReasons])
-		builder.WriteByte('\n')
+	// reverse order
+	for i := ew.index - 1; i >= ew.index-ew.count; i-- {
+		reasons = append(reasons, ew.reasons[(i+numHistoryReasons)%numHistoryReasons])
 	}
 	ew.lock.Unlock()
 
-	return builder.String()
+	return strings.Join(reasons, "\n")
 }
 
 type promiseWithReason struct {

core/breaker/breaker_test.go

@@ -2,7 +2,9 @@ package breaker
 
 import (
 	"errors"
+	"fmt"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -33,6 +35,84 @@ func TestLogReason(t *testing.T) {
 	assert.Equal(t, numHistoryReasons, errs.count)
 }
 
+func TestErrorWindow(t *testing.T) {
+	tests := []struct {
+		name    string
+		reasons []string
+	}{
+		{
+			name: "no error",
+		},
+		{
+			name:    "one error",
+			reasons: []string{"foo"},
+		},
+		{
+			name:    "two errors",
+			reasons: []string{"foo", "bar"},
+		},
+		{
+			name:    "five errors",
+			reasons: []string{"first", "second", "third", "fourth", "fifth"},
+		},
+		{
+			name:    "six errors",
+			reasons: []string{"first", "second", "third", "fourth", "fifth", "sixth"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var ew errorWindow
+			for _, reason := range test.reasons {
+				ew.add(reason)
+			}
+			var reasons []string
+			if len(test.reasons) > numHistoryReasons {
+				reasons = test.reasons[len(test.reasons)-numHistoryReasons:]
+			} else {
+				reasons = test.reasons
+			}
+			for _, reason := range reasons {
+				assert.True(t, strings.Contains(ew.String(), reason), fmt.Sprintf("actual: %s", ew.String()))
+			}
+		})
+	}
+}
+
+func TestPromiseWithReason(t *testing.T) {
+	tests := []struct {
+		name   string
+		reason string
+		expect string
+	}{
+		{
+			name: "success",
+		},
+		{
+			name:   "success",
+			reason: "fail",
+			expect: "fail",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			promise := promiseWithReason{
+				promise: new(mockedPromise),
+				errWin:  new(errorWindow),
+			}
+			if len(test.reason) == 0 {
+				promise.Accept()
+			} else {
+				promise.Reject(test.reason)
+			}
+
+			assert.True(t, strings.Contains(promise.errWin.String(), test.expect))
+		})
+	}
+}
+
 func BenchmarkGoogleBreaker(b *testing.B) {
 	br := NewBreaker()
 	for i := 0; i < b.N; i++ {
@@ -41,3 +121,12 @@ func BenchmarkGoogleBreaker(b *testing.B) {
 		})
 	}
 }
+
+type mockedPromise struct {
+}
+
+func (m *mockedPromise) Accept() {
+}
+
+func (m *mockedPromise) Reject() {
+}

core/breaker/breakers.go

@@ -41,10 +41,13 @@ func GetBreaker(name string) Breaker {
 	}
 
 	lock.Lock()
-	defer lock.Unlock()
+	b, ok = breakers[name]
+	if !ok {
+		b = NewBreaker(WithName(name))
+		breakers[name] = b
+	}
+	lock.Unlock()
 
-	b = NewBreaker()
-	breakers[name] = b
 	return b
 }
 
@@ -55,20 +58,5 @@ func NoBreakFor(name string) {
 }
 
 func do(name string, execute func(b Breaker) error) error {
-	lock.RLock()
-	b, ok := breakers[name]
-	lock.RUnlock()
-	if ok {
-		return execute(b)
-	}
-
-	lock.Lock()
-	b, ok = breakers[name]
-	if !ok {
-		b = NewBreaker(WithName(name))
-		breakers[name] = b
-	}
-	lock.Unlock()
-
-	return execute(b)
+	return execute(GetBreaker(name))
 }

core/breaker/googlebreaker.go

@@ -2,7 +2,6 @@ package breaker
 
 import (
 	"math"
-	"sync/atomic"
 	"time"
 
 	"github.com/tal-tech/go-zero/core/collection"
@@ -21,7 +20,6 @@ const (
 // see Client-Side Throttling section in https://landing.google.com/sre/sre-book/chapters/handling-overload/
 type googleBreaker struct {
 	k     float64
-	state int32
 	stat  *collection.RollingWindow
 	proba *mathx.Proba
 }
@@ -32,7 +30,6 @@ func newGoogleBreaker() *googleBreaker {
 	return &googleBreaker{
 		stat:  st,
 		k:     k,
-		state: StateClosed,
 		proba: mathx.NewProba(),
 	}
 }
@@ -43,15 +40,9 @@ func (b *googleBreaker) accept() error {
 	// https://landing.google.com/sre/sre-book/chapters/handling-overload/#eq2101
 	dropRatio := math.Max(0, (float64(total-protection)-weightedAccepts)/float64(total+1))
 	if dropRatio <= 0 {
-		if atomic.LoadInt32(&b.state) == StateOpen {
-			atomic.CompareAndSwapInt32(&b.state, StateOpen, StateClosed)
-		}
 		return nil
 	}
 
-	if atomic.LoadInt32(&b.state) == StateClosed {
-		atomic.CompareAndSwapInt32(&b.state, StateClosed, StateOpen)
-	}
 	if b.proba.TrueOnProba(dropRatio) {
 		return ErrServiceUnavailable
 	}

core/breaker/googlebreaker_test.go

@@ -27,7 +27,6 @@ func getGoogleBreaker() *googleBreaker {
 	return &googleBreaker{
 		stat:  st,
 		k:     5,
-		state: StateClosed,
 		proba: mathx.NewProba(),
 	}
 }

core/cmdline/input_test.go

@@ -0,0 +1,80 @@
+package cmdline
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/iox"
+	"github.com/tal-tech/go-zero/core/lang"
+)
+
+func TestEnterToContinue(t *testing.T) {
+	restore, err := iox.RedirectInOut()
+	assert.Nil(t, err)
+	defer restore()
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		fmt.Println()
+	}()
+	go func() {
+		defer wg.Done()
+		EnterToContinue()
+	}()
+
+	wait := make(chan lang.PlaceholderType)
+	go func() {
+		wg.Wait()
+		close(wait)
+	}()
+
+	select {
+	case <-time.After(time.Second):
+		t.Error("timeout")
+	case <-wait:
+	}
+}
+
+func TestReadLine(t *testing.T) {
+	r, w, err := os.Pipe()
+	assert.Nil(t, err)
+	ow := os.Stdout
+	os.Stdout = w
+	or := os.Stdin
+	os.Stdin = r
+	defer func() {
+		os.Stdin = or
+		os.Stdout = ow
+	}()
+
+	const message = "hello"
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		fmt.Println(message)
+	}()
+	go func() {
+		defer wg.Done()
+		input := ReadLine("")
+		assert.Equal(t, message, input)
+	}()
+
+	wait := make(chan lang.PlaceholderType)
+	go func() {
+		wg.Wait()
+		close(wait)
+	}()
+
+	select {
+	case <-time.After(time.Second):
+		t.Error("timeout")
+	case <-wait:
+	}
+}

core/codec/dh_test.go

@@ -71,3 +71,12 @@ func TestDiffieHellmanMiddleManAttack(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, string(src), string(decryptedSrc))
 }
+
+func TestKeyBytes(t *testing.T) {
+	var empty DhKey
+	assert.Equal(t, 0, len(empty.Bytes()))
+
+	key, err := GenerateKey()
+	assert.Nil(t, err)
+	assert.True(t, len(key.Bytes()) > 0)
+}

core/codec/hmac_test.go

@@ -0,0 +1,19 @@
+package codec
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHmac(t *testing.T) {
+	ret := Hmac([]byte("foo"), "bar")
+	assert.Equal(t, "f9320baf0249169e73850cd6156ded0106e2bb6ad8cab01b7bbbebe6d1065317",
+		fmt.Sprintf("%x", ret))
+}
+
+func TestHmacBase64(t *testing.T) {
+	ret := HmacBase64([]byte("foo"), "bar")
+	assert.Equal(t, "+TILrwJJFp5zhQzWFW3tAQbiu2rYyrAbe7vr5tEGUxc=", ret)
+}

core/codec/rsa_test.go

@@ -0,0 +1,59 @@
+package codec
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/fs"
+)
+
+const (
+	priKey = `-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQC4TJk3onpqb2RYE3wwt23J9SHLFstHGSkUYFLe+nl1dEKHbD+/
+Zt95L757J3xGTrwoTc7KCTxbrgn+stn0w52BNjj/kIE2ko4lbh/v8Fl14AyVR9ms
+fKtKOnhe5FCT72mdtApr+qvzcC3q9hfXwkyQU32pv7q5UimZ205iKSBmgQIDAQAB
+AoGAM5mWqGIAXj5z3MkP01/4CDxuyrrGDVD5FHBno3CDgyQa4Gmpa4B0/ywj671B
+aTnwKmSmiiCN2qleuQYASixes2zY5fgTzt+7KNkl9JHsy7i606eH2eCKzsUa/s6u
+WD8V3w/hGCQ9zYI18ihwyXlGHIgcRz/eeRh+nWcWVJzGOPUCQQD5nr6It/1yHb1p
+C6l4fC4xXF19l4KxJjGu1xv/sOpSx0pOqBDEX3Mh//FU954392rUWDXV1/I65BPt
+TLphdsu3AkEAvQJ2Qay/lffFj9FaUrvXuftJZ/Ypn0FpaSiUh3Ak3obBT6UvSZS0
+bcYdCJCNHDtBOsWHnIN1x+BcWAPrdU7PhwJBAIQ0dUlH2S3VXnoCOTGc44I1Hzbj
+Rc65IdsuBqA3fQN2lX5vOOIog3vgaFrOArg1jBkG1wx5IMvb/EnUN2pjVqUCQCza
+KLXtCInOAlPemlCHwumfeAvznmzsWNdbieOZ+SXVVIpR6KbNYwOpv7oIk3Pfm9sW
+hNffWlPUKhW42Gc+DIECQQDmk20YgBXwXWRM5DRPbhisIV088N5Z58K9DtFWkZsd
+OBDT3dFcgZONtlmR1MqZO0pTh30lA4qovYj3Bx7A8i36
+-----END RSA PRIVATE KEY-----`
+	pubKey = `-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC4TJk3onpqb2RYE3wwt23J9SHL
+FstHGSkUYFLe+nl1dEKHbD+/Zt95L757J3xGTrwoTc7KCTxbrgn+stn0w52BNjj/
+kIE2ko4lbh/v8Fl14AyVR9msfKtKOnhe5FCT72mdtApr+qvzcC3q9hfXwkyQU32p
+v7q5UimZ205iKSBmgQIDAQAB
+-----END PUBLIC KEY-----`
+	testBody      = `this is the content`
+	encryptedBody = `49e7bc15640e5d927fd3f129b749536d0755baf03a0f35fc914ff1b7b8ce659e5fe3a598442eb908c5995e28bacd3d76e4420bb05b6bfc177040f66c6976f680f7123505d626ab96a9db1151f45c93bc0262db9087b9fb6801715f76f902e644a20029262858f05b0d10540842204346ac1d6d8f29cc5d47dab79af75d922ef2`
+)
+
+func TestCryption(t *testing.T) {
+	enc, err := NewRsaEncrypter([]byte(pubKey))
+	assert.Nil(t, err)
+	ret, err := enc.Encrypt([]byte(testBody))
+	assert.Nil(t, err)
+
+	file, err := fs.TempFilenameWithText(priKey)
+	assert.Nil(t, err)
+	dec, err := NewRsaDecrypter(file)
+	assert.Nil(t, err)
+	actual, err := dec.Decrypt(ret)
+	assert.Nil(t, err)
+	assert.Equal(t, testBody, string(actual))
+
+	actual, err = dec.DecryptBase64(base64.StdEncoding.EncodeToString(ret))
+	assert.Nil(t, err)
+	assert.Equal(t, testBody, string(actual))
+}
+
+func TestBadPubKey(t *testing.T) {
+	_, err := NewRsaEncrypter([]byte("foo"))
+	assert.Equal(t, ErrPublicKey, err)
+}

core/collection/cache.go

@@ -82,12 +82,7 @@ func (c *Cache) Del(key string) {
 }
 
 func (c *Cache) Get(key string) (interface{}, bool) {
-	c.lock.Lock()
-	value, ok := c.data[key]
-	if ok {
-		c.lruCache.add(key)
-	}
-	c.lock.Unlock()
+	value, ok := c.doGet(key)
 	if ok {
 		c.stats.IncrementHit()
 	} else {
@@ -113,12 +108,25 @@ func (c *Cache) Set(key string, value interface{}) {
 }
 
 func (c *Cache) Take(key string, fetch func() (interface{}, error)) (interface{}, error) {
-	val, fresh, err := c.barrier.DoEx(key, func() (interface{}, error) {
+	if val, ok := c.doGet(key); ok {
+		c.stats.IncrementHit()
+		return val, nil
+	}
+
+	var fresh bool
+	val, err := c.barrier.Do(key, func() (interface{}, error) {
+		// because O(1) on map search in memory, and fetch is an IO query
+		// so we do double check, cache might be taken by another call
+		if val, ok := c.doGet(key); ok {
+			return val, nil
+		}
+
 		v, e := fetch()
 		if e != nil {
 			return nil, e
 		}
 
+		fresh = true
 		c.Set(key, v)
 		return v, nil
 	})
@@ -137,6 +145,18 @@ func (c *Cache) Take(key string, fetch func() (interface{}, error)) (interface{}
 	return val, nil
 }
 
+func (c *Cache) doGet(key string) (interface{}, bool) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	value, ok := c.data[key]
+	if ok {
+		c.lruCache.add(key)
+	}
+
+	return value, ok
+}
+
 func (c *Cache) onEvict(key string) {
 	// already locked
 	delete(c.data, key)

core/collection/cache_test.go

@@ -1,6 +1,7 @@
 package collection
 
 import (
+	"errors"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -10,6 +11,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var errDummy = errors.New("dummy")
+
 func TestCacheSet(t *testing.T) {
 	cache, err := NewCache(time.Second*2, WithName("any"))
 	assert.Nil(t, err)
@@ -63,6 +66,54 @@ func TestCacheTake(t *testing.T) {
 	assert.Equal(t, int32(1), atomic.LoadInt32(&count))
 }
 
+func TestCacheTakeExists(t *testing.T) {
+	cache, err := NewCache(time.Second * 2)
+	assert.Nil(t, err)
+
+	var count int32
+	var wg sync.WaitGroup
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			cache.Set("first", "first element")
+			cache.Take("first", func() (interface{}, error) {
+				atomic.AddInt32(&count, 1)
+				time.Sleep(time.Millisecond * 100)
+				return "first element", nil
+			})
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	assert.Equal(t, 1, cache.size())
+	assert.Equal(t, int32(0), atomic.LoadInt32(&count))
+}
+
+func TestCacheTakeError(t *testing.T) {
+	cache, err := NewCache(time.Second * 2)
+	assert.Nil(t, err)
+
+	var count int32
+	var wg sync.WaitGroup
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			_, err := cache.Take("first", func() (interface{}, error) {
+				atomic.AddInt32(&count, 1)
+				time.Sleep(time.Millisecond * 100)
+				return "", errDummy
+			})
+			assert.Equal(t, errDummy, err)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	assert.Equal(t, 0, cache.size())
+	assert.Equal(t, int32(1), atomic.LoadInt32(&count))
+}
+
 func TestCacheWithLruEvicts(t *testing.T) {
 	cache, err := NewCache(time.Minute, WithLimit(3))
 	assert.Nil(t, err)

core/collection/set.go

@@ -15,6 +15,7 @@ const (
 	stringType
 )
 
+// Set is not thread-safe, for concurrent use, make sure to use it with synchronization.
 type Set struct {
 	data map[interface{}]lang.PlaceholderType
 	tp   int
@@ -182,10 +183,7 @@ func (s *Set) add(i interface{}) {
 }
 
 func (s *Set) setType(i interface{}) {
-	if s.tp != untyped {
-		return
-	}
-
+	// s.tp can only be untyped here
 	switch i.(type) {
 	case int:
 		s.tp = intType

core/collection/set_test.go

@@ -5,8 +5,13 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/logx"
 )
 
+func init() {
+	logx.Disable()
+}
+
 func BenchmarkRawSet(b *testing.B) {
 	m := make(map[interface{}]struct{})
 	for i := 0; i < b.N; i++ {
@@ -147,3 +152,51 @@ func TestCount(t *testing.T) {
 	// then
 	assert.Equal(t, set.Count(), 3)
 }
+
+func TestKeysIntMismatch(t *testing.T) {
+	set := NewSet()
+	set.add(int64(1))
+	set.add(2)
+	vals := set.KeysInt()
+	assert.EqualValues(t, []int{2}, vals)
+}
+
+func TestKeysInt64Mismatch(t *testing.T) {
+	set := NewSet()
+	set.add(1)
+	set.add(int64(2))
+	vals := set.KeysInt64()
+	assert.EqualValues(t, []int64{2}, vals)
+}
+
+func TestKeysUintMismatch(t *testing.T) {
+	set := NewSet()
+	set.add(1)
+	set.add(uint(2))
+	vals := set.KeysUint()
+	assert.EqualValues(t, []uint{2}, vals)
+}
+
+func TestKeysUint64Mismatch(t *testing.T) {
+	set := NewSet()
+	set.add(1)
+	set.add(uint64(2))
+	vals := set.KeysUint64()
+	assert.EqualValues(t, []uint64{2}, vals)
+}
+
+func TestKeysStrMismatch(t *testing.T) {
+	set := NewSet()
+	set.add(1)
+	set.add("2")
+	vals := set.KeysStr()
+	assert.EqualValues(t, []string{"2"}, vals)
+}
+
+func TestSetType(t *testing.T) {
+	set := NewUnmanagedSet()
+	set.add(1)
+	set.add("2")
+	vals := set.Keys()
+	assert.ElementsMatch(t, []interface{}{1, "2"}, vals)
+}

core/conf/config_test.go

@@ -0,0 +1,58 @@
+package conf
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/hash"
+)
+
+func TestConfigJson(t *testing.T) {
+	tests := []string{
+		".json",
+		".yaml",
+		".yml",
+	}
+	text := `{
+	"a": "foo",
+	"b": 1
+}`
+	for _, test := range tests {
+		test := test
+		t.Run(test, func(t *testing.T) {
+			t.Parallel()
+
+			tmpfile, err := createTempFile(test, text)
+			assert.Nil(t, err)
+			defer os.Remove(tmpfile)
+
+			var val struct {
+				A string `json:"a"`
+				B int    `json:"b"`
+			}
+			MustLoad(tmpfile, &val)
+			assert.Equal(t, "foo", val.A)
+			assert.Equal(t, 1, val.B)
+		})
+	}
+}
+
+func createTempFile(ext, text string) (string, error) {
+	tmpfile, err := ioutil.TempFile(os.TempDir(), hash.Md5Hex([]byte(text))+"*"+ext)
+	if err != nil {
+		return "", err
+	}
+
+	if err := ioutil.WriteFile(tmpfile.Name(), []byte(text), os.ModeTemporary); err != nil {
+		return "", err
+	}
+
+	filename := tmpfile.Name()
+	if err = tmpfile.Close(); err != nil {
+		return "", err
+	}
+
+	return filename, nil
+}

core/conf/properties.go

@@ -30,12 +30,12 @@ type mapBasedProperties struct {
 	lock       sync.RWMutex
 }
 
-// Loads the properties into a properties configuration instance. May return the
-// configuration itself along with an error that indicates if there was a problem loading the configuration.
+// Loads the properties into a properties configuration instance.
+// Returns an error that indicates if there was a problem loading the configuration.
 func LoadProperties(filename string) (Properties, error) {
 	lines, err := iox.ReadTextLines(filename, iox.WithoutBlank(), iox.OmitWithPrefix("#"))
 	if err != nil {
-		return nil, nil
+		return nil, err
 	}
 
 	raw := make(map[string]string)

core/conf/properties_test.go

@@ -41,3 +41,8 @@ func TestSetInt(t *testing.T) {
 	props.SetInt(key, value)
 	assert.Equal(t, value, props.GetInt(key))
 }
+
+func TestLoadBadFile(t *testing.T) {
+	_, err := LoadProperties("nosuchfile")
+	assert.NotNil(t, err)
+}

core/discov/internal/listener.go

@@ -1,4 +1,3 @@
-//go:generate mockgen -package internal -destination listener_mock.go -source listener.go Listener
 package internal
 
 type Listener interface {

core/discov/internal/listener_mock.go

@@ -1,45 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: listener.go
-
-// Package internal is a generated GoMock package.
-package internal
-
-import (
-	gomock "github.com/golang/mock/gomock"
-	reflect "reflect"
-)
-
-// MockListener is a mock of Listener interface
-type MockListener struct {
-	ctrl     *gomock.Controller
-	recorder *MockListenerMockRecorder
-}
-
-// MockListenerMockRecorder is the mock recorder for MockListener
-type MockListenerMockRecorder struct {
-	mock *MockListener
-}
-
-// NewMockListener creates a new mock instance
-func NewMockListener(ctrl *gomock.Controller) *MockListener {
-	mock := &MockListener{ctrl: ctrl}
-	mock.recorder = &MockListenerMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use
-func (m *MockListener) EXPECT() *MockListenerMockRecorder {
-	return m.recorder
-}
-
-// OnUpdate mocks base method
-func (m *MockListener) OnUpdate(keys, values []string, newKey string) {
-	m.ctrl.T.Helper()
-	m.ctrl.Call(m, "OnUpdate", keys, values, newKey)
-}
-
-// OnUpdate indicates an expected call of OnUpdate
-func (mr *MockListenerMockRecorder) OnUpdate(keys, values, newKey interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnUpdate", reflect.TypeOf((*MockListener)(nil).OnUpdate), keys, values, newKey)
-}

core/discov/kubernetes/etcd.yaml

@@ -40,6 +40,7 @@ spec:
         - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
+        - --auto-compaction-retention=1
       image: quay.io/coreos/etcd:latest
       name: etcd0
       ports:
@@ -111,6 +112,7 @@ spec:
         - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
+        - --auto-compaction-retention=1
       image: quay.io/coreos/etcd:latest
       name: etcd1
       ports:
@@ -182,6 +184,7 @@ spec:
         - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
+        - --auto-compaction-retention=1
       image: quay.io/coreos/etcd:latest
       name: etcd2
       ports:
@@ -253,6 +256,7 @@ spec:
         - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
+        - --auto-compaction-retention=1
       image: quay.io/coreos/etcd:latest
       name: etcd3
       ports:
@@ -324,6 +328,7 @@ spec:
         - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380,etcd4=http://etcd4:2380
         - --initial-cluster-state
         - new
+        - --auto-compaction-retention=1
       image: quay.io/coreos/etcd:latest
       name: etcd4
       ports:

core/discov/publisher_test.go

@@ -111,6 +111,10 @@ func TestPublisher_keepAliveAsyncQuit(t *testing.T) {
 	defer ctrl.Finish()
 	const id clientv3.LeaseID = 1
 	cli := internal.NewMockEtcdClient(ctrl)
+	cli.EXPECT().ActiveConnection()
+	cli.EXPECT().Close()
+	defer cli.Close()
+	cli.ActiveConnection()
 	restore := setMockClient(cli)
 	defer restore()
 	cli.EXPECT().Ctx().AnyTimes()

core/errorx/atomicerror.go

@@ -1,21 +1,18 @@
 package errorx
 
-import "sync"
+import "sync/atomic"
 
 type AtomicError struct {
-	err  error
-	lock sync.Mutex
+	err atomic.Value // error
 }
 
 func (ae *AtomicError) Set(err error) {
-	ae.lock.Lock()
-	ae.err = err
-	ae.lock.Unlock()
+	ae.err.Store(err)
 }
 
 func (ae *AtomicError) Load() error {
-	ae.lock.Lock()
-	err := ae.err
-	ae.lock.Unlock()
-	return err
+	if v := ae.err.Load(); v != nil {
+		return v.(error)
+	}
+	return nil
 }

core/errorx/atomicerror_test.go

@@ -2,6 +2,8 @@ package errorx
 
 import (
 	"errors"
+	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -19,3 +21,53 @@ func TestAtomicErrorNil(t *testing.T) {
 	var err AtomicError
 	assert.Nil(t, err.Load())
 }
+
+func BenchmarkAtomicError(b *testing.B) {
+	var aerr AtomicError
+	wg := sync.WaitGroup{}
+
+	b.Run("Load", func(b *testing.B) {
+		var done uint32
+		go func() {
+			for {
+				if atomic.LoadUint32(&done) != 0 {
+					break
+				}
+				wg.Add(1)
+				go func() {
+					aerr.Set(errDummy)
+					wg.Done()
+				}()
+			}
+		}()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = aerr.Load()
+		}
+		b.StopTimer()
+		atomic.StoreUint32(&done, 1)
+		wg.Wait()
+	})
+	b.Run("Set", func(b *testing.B) {
+		var done uint32
+		go func() {
+			for {
+				if atomic.LoadUint32(&done) != 0 {
+					break
+				}
+				wg.Add(1)
+				go func() {
+					_ = aerr.Load()
+					wg.Done()
+				}()
+			}
+		}()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			aerr.Set(errDummy)
+		}
+		b.StopTimer()
+		atomic.StoreUint32(&done, 1)
+		wg.Wait()
+	})
+}

core/fx/fn.go

@@ -84,6 +84,14 @@ func (p Stream) Buffer(n int) Stream {
 	return Range(source)
 }
 
+// Count counts the number of elements in the result.
+func (p Stream) Count() (count int) {
+	for range p.source {
+		count++
+	}
+	return
+}
+
 // Distinct removes the duplicated items base on the given KeyFunc.
 func (p Stream) Distinct(fn KeyFunc) Stream {
 	source := make(chan interface{})

core/fx/fn_test.go

@@ -49,6 +49,36 @@ func TestBufferNegative(t *testing.T) {
 	assert.Equal(t, 10, result)
 }
 
+func TestCount(t *testing.T) {
+	tests := []struct {
+		name     string
+		elements []interface{}
+	}{
+		{
+			name: "no elements with nil",
+		},
+		{
+			name:     "no elements",
+			elements: []interface{}{},
+		},
+		{
+			name:     "1 element",
+			elements: []interface{}{1},
+		},
+		{
+			name:     "multiple elements",
+			elements: []interface{}{1, 2, 3},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			val := Just(test.elements...).Count()
+			assert.Equal(t, len(test.elements), val)
+		})
+	}
+}
+
 func TestDone(t *testing.T) {
 	var count int32
 	Just(1, 2, 3).Walk(func(item interface{}, pipe chan<- interface{}) {

core/iox/pipe.go

@@ -0,0 +1,23 @@
+package iox
+
+import "os"
+
+// RedirectInOut redirects stdin to r, stdout to w, and callers need to call restore afterwards.
+func RedirectInOut() (restore func(), err error) {
+	var r, w *os.File
+	r, w, err = os.Pipe()
+	if err != nil {
+		return
+	}
+
+	ow := os.Stdout
+	os.Stdout = w
+	or := os.Stdin
+	os.Stdin = r
+	restore = func() {
+		os.Stdin = or
+		os.Stdout = ow
+	}
+
+	return
+}

core/iox/pipe_test.go

@@ -0,0 +1,13 @@
+package iox
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRedirectInOut(t *testing.T) {
+	restore, err := RedirectInOut()
+	assert.Nil(t, err)
+	defer restore()
+}

core/load/adaptiveshedder_test.go

@@ -135,6 +135,7 @@ func TestAdaptiveShedderShouldDrop(t *testing.T) {
 		passCounter:     passCounter,
 		rtCounter:       rtCounter,
 		windows:         buckets,
+		dropTime:        syncx.NewAtomicDuration(),
 		droppedRecently: syncx.NewAtomicBool(),
 	}
 	// cpu >=  800, inflight < maxPass
@@ -160,6 +161,40 @@ func TestAdaptiveShedderShouldDrop(t *testing.T) {
 	}
 	shedder.avgFlying = 80
 	assert.False(t, shedder.shouldDrop())
+
+	// cpu >=  800, inflight < maxPass
+	systemOverloadChecker = func(int64) bool {
+		return true
+	}
+	shedder.avgFlying = 80
+	shedder.flying = 80
+	_, err := shedder.Allow()
+	assert.NotNil(t, err)
+}
+
+func TestAdaptiveShedderStillHot(t *testing.T) {
+	logx.Disable()
+	passCounter := newRollingWindow()
+	rtCounter := newRollingWindow()
+	for i := 0; i < 10; i++ {
+		if i > 0 {
+			time.Sleep(bucketDuration)
+		}
+		passCounter.Add(float64((i + 1) * 100))
+		for j := i*10 + 1; j <= i*10+10; j++ {
+			rtCounter.Add(float64(j))
+		}
+	}
+	shedder := &adaptiveShedder{
+		passCounter:     passCounter,
+		rtCounter:       rtCounter,
+		windows:         buckets,
+		dropTime:        syncx.NewAtomicDuration(),
+		droppedRecently: syncx.ForAtomicBool(true),
+	}
+	assert.False(t, shedder.stillHot())
+	shedder.dropTime.Set(-coolOffDuration * 2)
+	assert.False(t, shedder.stillHot())
 }
 
 func BenchmarkAdaptiveShedder_Allow(b *testing.B) {

core/load/sheddergroup_test.go

@@ -13,3 +13,8 @@ func TestGroup(t *testing.T) {
 		assert.NotNil(t, limiter)
 	})
 }
+
+func TestShedderClose(t *testing.T) {
+	var nop nopCloser
+	assert.Nil(t, nop.Close())
+}

core/logx/durationlogger.go

@@ -8,55 +8,60 @@ import (
 	"github.com/tal-tech/go-zero/core/timex"
 )
 
-const customCallerDepth = 3
+const durationCallerDepth = 3
 
-type customLog logEntry
+type durationLogger logEntry
 
 func WithDuration(d time.Duration) Logger {
-	return customLog{
+	return &durationLogger{
 		Duration: timex.ReprOfDuration(d),
 	}
 }
 
-func (l customLog) Error(v ...interface{}) {
+func (l *durationLogger) Error(v ...interface{}) {
 	if shouldLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), customCallerDepth))
+		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
 	}
 }
 
-func (l customLog) Errorf(format string, v ...interface{}) {
+func (l *durationLogger) Errorf(format string, v ...interface{}) {
 	if shouldLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), customCallerDepth))
+		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
 	}
 }
 
-func (l customLog) Info(v ...interface{}) {
+func (l *durationLogger) Info(v ...interface{}) {
 	if shouldLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprint(v...))
 	}
 }
 
-func (l customLog) Infof(format string, v ...interface{}) {
+func (l *durationLogger) Infof(format string, v ...interface{}) {
 	if shouldLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
 	}
 }
 
-func (l customLog) Slow(v ...interface{}) {
+func (l *durationLogger) Slow(v ...interface{}) {
 	if shouldLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprint(v...))
 	}
 }
 
-func (l customLog) Slowf(format string, v ...interface{}) {
+func (l *durationLogger) Slowf(format string, v ...interface{}) {
 	if shouldLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
 	}
 }
 
-func (l customLog) write(writer io.Writer, level, content string) {
+func (l *durationLogger) WithDuration(duration time.Duration) Logger {
+	l.Duration = timex.ReprOfDuration(duration)
+	return l
+}
+
+func (l *durationLogger) write(writer io.Writer, level, content string) {
 	l.Timestamp = getTimestamp()
 	l.Level = level
 	l.Content = content
-	outputJson(writer, logEntry(l))
+	outputJson(writer, logEntry(*l))
 }

core/logx/durationlogger_test.go

@@ -0,0 +1,52 @@
+package logx
+
+import (
+	"log"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWithDurationError(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Error("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
+func TestWithDurationErrorf(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Errorf("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
+func TestWithDurationInfo(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Info("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
+func TestWithDurationInfof(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Infof("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
+func TestWithDurationSlow(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).Slow("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}
+
+func TestWithDurationSlowf(t *testing.T) {
+	var builder strings.Builder
+	log.SetOutput(&builder)
+	WithDuration(time.Second).WithDuration(time.Hour).Slowf("foo")
+	assert.True(t, strings.Contains(builder.String(), "duration"), builder.String())
+}

core/logx/logs.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/tal-tech/go-zero/core/iox"
 	"github.com/tal-tech/go-zero/core/sysx"
@@ -96,6 +97,7 @@ type (
 		Infof(string, ...interface{})
 		Slow(...interface{})
 		Slowf(string, ...interface{})
+		WithDuration(time.Duration) Logger
 	}
 )
 

core/logx/logs_test.go

@@ -6,8 +6,10 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"os"
 	"runtime"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -21,10 +23,13 @@ var (
 )
 
 type mockWriter struct {
+	lock    sync.Mutex
 	builder strings.Builder
 }
 
 func (mw *mockWriter) Write(data []byte) (int, error) {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
 	return mw.builder.Write(data)
 }
 
@@ -32,12 +37,22 @@ func (mw *mockWriter) Close() error {
 	return nil
 }
 
+func (mw *mockWriter) Contains(text string) bool {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	return strings.Contains(mw.builder.String(), text)
+}
+
 func (mw *mockWriter) Reset() {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
 	mw.builder.Reset()
 }
 
-func (mw *mockWriter) Contains(text string) bool {
-	return strings.Contains(mw.builder.String(), text)
+func (mw *mockWriter) String() string {
+	mw.lock.Lock()
+	defer mw.lock.Unlock()
+	return mw.builder.String()
 }
 
 func TestFileLineFileMode(t *testing.T) {
@@ -85,6 +100,46 @@ func TestStructedLogSlow(t *testing.T) {
 	})
 }
 
+func TestStructedLogSlowf(t *testing.T) {
+	doTestStructedLog(t, levelSlow, func(writer io.WriteCloser) {
+		slowLog = writer
+	}, func(v ...interface{}) {
+		Slowf(fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogStat(t *testing.T) {
+	doTestStructedLog(t, levelStat, func(writer io.WriteCloser) {
+		statLog = writer
+	}, func(v ...interface{}) {
+		Stat(v...)
+	})
+}
+
+func TestStructedLogStatf(t *testing.T) {
+	doTestStructedLog(t, levelStat, func(writer io.WriteCloser) {
+		statLog = writer
+	}, func(v ...interface{}) {
+		Statf(fmt.Sprint(v...))
+	})
+}
+
+func TestStructedLogSevere(t *testing.T) {
+	doTestStructedLog(t, levelSevere, func(writer io.WriteCloser) {
+		severeLog = writer
+	}, func(v ...interface{}) {
+		Severe(v...)
+	})
+}
+
+func TestStructedLogSeveref(t *testing.T) {
+	doTestStructedLog(t, levelSevere, func(writer io.WriteCloser) {
+		severeLog = writer
+	}, func(v ...interface{}) {
+		Severef(fmt.Sprint(v...))
+	})
+}
+
 func TestStructedLogWithDuration(t *testing.T) {
 	const message = "hello there"
 	writer := new(mockWriter)
@@ -135,6 +190,66 @@ func TestMustNil(t *testing.T) {
 	Must(nil)
 }
 
+func TestSetup(t *testing.T) {
+	MustSetup(LogConf{
+		ServiceName: "any",
+		Mode:        "console",
+	})
+	MustSetup(LogConf{
+		ServiceName: "any",
+		Mode:        "file",
+		Path:        os.TempDir(),
+	})
+	MustSetup(LogConf{
+		ServiceName: "any",
+		Mode:        "volume",
+		Path:        os.TempDir(),
+	})
+	assert.NotNil(t, setupWithVolume(LogConf{}))
+	assert.NotNil(t, setupWithFiles(LogConf{}))
+	assert.Nil(t, setupWithFiles(LogConf{
+		ServiceName: "any",
+		Path:        os.TempDir(),
+		Compress:    true,
+		KeepDays:    1,
+	}))
+	setupLogLevel(LogConf{
+		Level: levelInfo,
+	})
+	setupLogLevel(LogConf{
+		Level: levelError,
+	})
+	setupLogLevel(LogConf{
+		Level: levelSevere,
+	})
+	_, err := createOutput("")
+	assert.NotNil(t, err)
+	Disable()
+}
+
+func TestDisable(t *testing.T) {
+	Disable()
+	WithKeepDays(1)
+	WithGzip()
+	assert.Nil(t, Close())
+	writeConsole = false
+	assert.Nil(t, Close())
+}
+
+func TestWithGzip(t *testing.T) {
+	fn := WithGzip()
+	var opt logOptions
+	fn(&opt)
+	assert.True(t, opt.gzipEnabled)
+}
+
+func TestWithKeepDays(t *testing.T) {
+	fn := WithKeepDays(1)
+	var opt logOptions
+	fn(&opt)
+	assert.Equal(t, 1, opt.keepDays)
+}
+
 func BenchmarkCopyByteSliceAppend(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		var buf []byte
@@ -232,7 +347,7 @@ func doTestStructedLog(t *testing.T, level string, setup func(writer io.WriteClo
 		t.Error(err)
 	}
 	assert.Equal(t, level, entry.Level)
-	assert.Equal(t, message, entry.Content)
+	assert.True(t, strings.Contains(entry.Content, message))
 }
 
 func testSetLevelTwiceWithMode(t *testing.T, mode string) {
@@ -252,4 +367,10 @@ func testSetLevelTwiceWithMode(t *testing.T, mode string) {
 	atomic.StoreUint32(&initialized, 1)
 	Info(message)
 	assert.Equal(t, 0, writer.builder.Len())
+	Infof(message)
+	assert.Equal(t, 0, writer.builder.Len())
+	ErrorStack(message)
+	assert.Equal(t, 0, writer.builder.Len())
+	ErrorStackf(message)
+	assert.Equal(t, 0, writer.builder.Len())
 }

core/logx/rotatelogger.go

@@ -192,14 +192,16 @@ func (l *RotateLogger) init() error {
 }
 
 func (l *RotateLogger) maybeCompressFile(file string) {
-	if l.compress {
-		defer func() {
-			if r := recover(); r != nil {
-				ErrorStack(r)
-			}
-		}()
-		compressLogFile(file)
+	if !l.compress {
+		return
 	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			ErrorStack(r)
+		}
+	}()
+	compressLogFile(file)
 }
 
 func (l *RotateLogger) maybeDeleteOutdatedFiles() {

core/logx/rotatelogger_test.go

@@ -0,0 +1,119 @@
+package logx
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/fs"
+)
+
+func TestDailyRotateRuleMarkRotated(t *testing.T) {
+	var rule DailyRotateRule
+	rule.MarkRotated()
+	assert.Equal(t, getNowDate(), rule.rotatedTime)
+}
+
+func TestDailyRotateRuleOutdatedFiles(t *testing.T) {
+	var rule DailyRotateRule
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.days = 1
+	assert.Empty(t, rule.OutdatedFiles())
+	rule.gzip = true
+	assert.Empty(t, rule.OutdatedFiles())
+}
+
+func TestDailyRotateRuleShallRotate(t *testing.T) {
+	var rule DailyRotateRule
+	rule.rotatedTime = time.Now().Add(time.Hour * 24).Format(dateFormat)
+	assert.True(t, rule.ShallRotate())
+}
+
+func TestRotateLoggerClose(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(DailyRotateRule), false)
+	assert.Nil(t, err)
+	assert.Nil(t, logger.Close())
+}
+
+func TestRotateLoggerGetBackupFilename(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(DailyRotateRule), false)
+	assert.Nil(t, err)
+	assert.True(t, len(logger.getBackupFilename()) > 0)
+	logger.backup = ""
+	assert.True(t, len(logger.getBackupFilename()) > 0)
+}
+
+func TestRotateLoggerMayCompressFile(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer os.Remove(filename)
+	}
+	logger, err := NewLogger(filename, new(DailyRotateRule), false)
+	assert.Nil(t, err)
+	logger.maybeCompressFile(filename)
+	_, err = os.Stat(filename)
+	assert.Nil(t, err)
+}
+
+func TestRotateLoggerMayCompressFileTrue(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	logger, err := NewLogger(filename, new(DailyRotateRule), true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer func() {
+			os.Remove(filename)
+			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+		}()
+	}
+	logger.maybeCompressFile(filename)
+	_, err = os.Stat(filename)
+	assert.NotNil(t, err)
+}
+
+func TestRotateLoggerRotate(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	logger, err := NewLogger(filename, new(DailyRotateRule), true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer func() {
+			os.Remove(filename)
+			os.Remove(logger.getBackupFilename())
+			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+		}()
+	}
+	err = logger.rotate()
+	assert.Nil(t, err)
+}
+
+func TestRotateLoggerWrite(t *testing.T) {
+	filename, err := fs.TempFilenameWithText("foo")
+	assert.Nil(t, err)
+	rule := new(DailyRotateRule)
+	logger, err := NewLogger(filename, rule, true)
+	assert.Nil(t, err)
+	if len(filename) > 0 {
+		defer func() {
+			os.Remove(filename)
+			os.Remove(logger.getBackupFilename())
+			os.Remove(filepath.Base(logger.getBackupFilename()) + ".gz")
+		}()
+	}
+	logger.write([]byte(`foo`))
+	rule.rotatedTime = time.Now().Add(-time.Hour * 24).Format(dateFormat)
+	logger.write([]byte(`bar`))
+}

core/logx/syslog_test.go

@@ -33,10 +33,10 @@ func captureOutput(f func()) string {
 	writer := new(mockWriter)
 	infoLog = writer
 
-	prevLevel := logLevel
-	logLevel = InfoLevel
+	prevLevel := atomic.LoadUint32(&logLevel)
+	SetLevel(InfoLevel)
 	f()
-	logLevel = prevLevel
+	SetLevel(prevLevel)
 
 	return writer.builder.String()
 }

core/logx/tracelog_test.go

@@ -1,49 +0,0 @@
-package logx
-
-import (
-	"context"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/tal-tech/go-zero/core/trace/tracespec"
-)
-
-const (
-	mockTraceId = "mock-trace-id"
-	mockSpanId  = "mock-span-id"
-)
-
-var mock tracespec.Trace = new(mockTrace)
-
-func TestTraceLog(t *testing.T) {
-	var buf strings.Builder
-	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
-	WithContext(ctx).(tracingEntry).write(&buf, levelInfo, testlog)
-	assert.True(t, strings.Contains(buf.String(), mockTraceId))
-	assert.True(t, strings.Contains(buf.String(), mockSpanId))
-}
-
-type mockTrace struct{}
-
-func (t mockTrace) TraceId() string {
-	return mockTraceId
-}
-
-func (t mockTrace) SpanId() string {
-	return mockSpanId
-}
-
-func (t mockTrace) Finish() {
-}
-
-func (t mockTrace) Fork(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Follow(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
-	return nil, nil
-}
-
-func (t mockTrace) Visit(fn func(key string, val string) bool) {
-}

core/logx/tracelogger.go

@@ -4,54 +4,61 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"time"
 
+	"github.com/tal-tech/go-zero/core/timex"
 	"github.com/tal-tech/go-zero/core/trace/tracespec"
 )
 
-type tracingEntry struct {
+type traceLogger struct {
 	logEntry
-	Trace string          `json:"trace,omitempty"`
-	Span  string          `json:"span,omitempty"`
-	ctx   context.Context `json:"-"`
+	Trace string `json:"trace,omitempty"`
+	Span  string `json:"span,omitempty"`
+	ctx   context.Context
 }
 
-func (l tracingEntry) Error(v ...interface{}) {
+func (l *traceLogger) Error(v ...interface{}) {
 	if shouldLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), customCallerDepth))
+		l.write(errorLog, levelError, formatWithCaller(fmt.Sprint(v...), durationCallerDepth))
 	}
 }
 
-func (l tracingEntry) Errorf(format string, v ...interface{}) {
+func (l *traceLogger) Errorf(format string, v ...interface{}) {
 	if shouldLog(ErrorLevel) {
-		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), customCallerDepth))
+		l.write(errorLog, levelError, formatWithCaller(fmt.Sprintf(format, v...), durationCallerDepth))
 	}
 }
 
-func (l tracingEntry) Info(v ...interface{}) {
+func (l *traceLogger) Info(v ...interface{}) {
 	if shouldLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprint(v...))
 	}
 }
 
-func (l tracingEntry) Infof(format string, v ...interface{}) {
+func (l *traceLogger) Infof(format string, v ...interface{}) {
 	if shouldLog(InfoLevel) {
 		l.write(infoLog, levelInfo, fmt.Sprintf(format, v...))
 	}
 }
 
-func (l tracingEntry) Slow(v ...interface{}) {
+func (l *traceLogger) Slow(v ...interface{}) {
 	if shouldLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprint(v...))
 	}
 }
 
-func (l tracingEntry) Slowf(format string, v ...interface{}) {
+func (l *traceLogger) Slowf(format string, v ...interface{}) {
 	if shouldLog(ErrorLevel) {
 		l.write(slowLog, levelSlow, fmt.Sprintf(format, v...))
 	}
 }
 
-func (l tracingEntry) write(writer io.Writer, level, content string) {
+func (l *traceLogger) WithDuration(duration time.Duration) Logger {
+	l.Duration = timex.ReprOfDuration(duration)
+	return l
+}
+
+func (l *traceLogger) write(writer io.Writer, level, content string) {
 	l.Timestamp = getTimestamp()
 	l.Level = level
 	l.Content = content
@@ -61,7 +68,7 @@ func (l tracingEntry) write(writer io.Writer, level, content string) {
 }
 
 func WithContext(ctx context.Context) Logger {
-	return tracingEntry{
+	return &traceLogger{
 		ctx: ctx,
 	}
 }

core/logx/tracelogger_test.go

@@ -0,0 +1,115 @@
+package logx
+
+import (
+	"context"
+	"log"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/trace/tracespec"
+)
+
+const (
+	mockTraceId = "mock-trace-id"
+	mockSpanId  = "mock-span-id"
+)
+
+var mock tracespec.Trace = new(mockTrace)
+
+func TestTraceLog(t *testing.T) {
+	var buf mockWriter
+	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	WithContext(ctx).(*traceLogger).write(&buf, levelInfo, testlog)
+	assert.True(t, strings.Contains(buf.String(), mockTraceId))
+	assert.True(t, strings.Contains(buf.String(), mockSpanId))
+}
+
+func TestTraceError(t *testing.T) {
+	var buf mockWriter
+	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	l := WithContext(ctx).(*traceLogger)
+	SetLevel(InfoLevel)
+	atomic.StoreUint32(&initialized, 1)
+	errorLog = newLogWriter(log.New(&buf, "", flags))
+	l.WithDuration(time.Second).Error(testlog)
+	assert.True(t, strings.Contains(buf.String(), mockTraceId))
+	assert.True(t, strings.Contains(buf.String(), mockSpanId))
+	buf.Reset()
+	l.WithDuration(time.Second).Errorf(testlog)
+	assert.True(t, strings.Contains(buf.String(), mockTraceId))
+	assert.True(t, strings.Contains(buf.String(), mockSpanId))
+}
+
+func TestTraceInfo(t *testing.T) {
+	var buf mockWriter
+	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	l := WithContext(ctx).(*traceLogger)
+	SetLevel(InfoLevel)
+	atomic.StoreUint32(&initialized, 1)
+	infoLog = newLogWriter(log.New(&buf, "", flags))
+	l.WithDuration(time.Second).Info(testlog)
+	assert.True(t, strings.Contains(buf.String(), mockTraceId))
+	assert.True(t, strings.Contains(buf.String(), mockSpanId))
+	buf.Reset()
+	l.WithDuration(time.Second).Infof(testlog)
+	assert.True(t, strings.Contains(buf.String(), mockTraceId))
+	assert.True(t, strings.Contains(buf.String(), mockSpanId))
+}
+
+func TestTraceSlow(t *testing.T) {
+	var buf mockWriter
+	ctx := context.WithValue(context.Background(), tracespec.TracingKey, mock)
+	l := WithContext(ctx).(*traceLogger)
+	SetLevel(InfoLevel)
+	atomic.StoreUint32(&initialized, 1)
+	slowLog = newLogWriter(log.New(&buf, "", flags))
+	l.WithDuration(time.Second).Slow(testlog)
+	assert.True(t, strings.Contains(buf.String(), mockTraceId))
+	assert.True(t, strings.Contains(buf.String(), mockSpanId))
+	buf.Reset()
+	l.WithDuration(time.Second).Slowf(testlog)
+	assert.True(t, strings.Contains(buf.String(), mockTraceId))
+	assert.True(t, strings.Contains(buf.String(), mockSpanId))
+}
+
+func TestTraceWithoutContext(t *testing.T) {
+	var buf mockWriter
+	l := WithContext(context.Background()).(*traceLogger)
+	SetLevel(InfoLevel)
+	atomic.StoreUint32(&initialized, 1)
+	infoLog = newLogWriter(log.New(&buf, "", flags))
+	l.WithDuration(time.Second).Info(testlog)
+	assert.False(t, strings.Contains(buf.String(), mockTraceId))
+	assert.False(t, strings.Contains(buf.String(), mockSpanId))
+	buf.Reset()
+	l.WithDuration(time.Second).Infof(testlog)
+	assert.False(t, strings.Contains(buf.String(), mockTraceId))
+	assert.False(t, strings.Contains(buf.String(), mockSpanId))
+}
+
+type mockTrace struct{}
+
+func (t mockTrace) TraceId() string {
+	return mockTraceId
+}
+
+func (t mockTrace) SpanId() string {
+	return mockSpanId
+}
+
+func (t mockTrace) Finish() {
+}
+
+func (t mockTrace) Fork(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
+	return nil, nil
+}
+
+func (t mockTrace) Follow(ctx context.Context, serviceName, operationName string) (context.Context, tracespec.Trace) {
+	return nil, nil
+}
+
+func (t mockTrace) Visit(fn func(key string, val string) bool) {
+}

core/mapping/fieldoptions_test.go

@@ -0,0 +1,31 @@
+package mapping
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type Bar struct {
+	Val string `json:"val"`
+}
+
+func TestFieldOptionOptionalDep(t *testing.T) {
+	var bar Bar
+	rt := reflect.TypeOf(bar)
+	for i := 0; i < rt.NumField(); i++ {
+		field := rt.Field(i)
+		val, opt, err := parseKeyAndOptions(jsonTagKey, field)
+		assert.Equal(t, "val", val)
+		assert.Nil(t, opt)
+		assert.Nil(t, err)
+	}
+
+	// check nil working
+	var o *fieldOptions
+	check := func(o *fieldOptions) {
+		assert.Equal(t, 0, len(o.optionalDep()))
+	}
+	check(o)
+}

core/mapping/unmarshaler.go

@@ -23,6 +23,7 @@ const (
 var (
 	errTypeMismatch     = errors.New("type mismatch")
 	errValueNotSettable = errors.New("value is not settable")
+	errValueNotStruct   = errors.New("value type is not struct")
 	keyUnmarshaler      = NewUnmarshaler(defaultKeyName)
 	cacheKeys           atomic.Value
 	cacheKeysLock       sync.Mutex
@@ -80,6 +81,10 @@ func (u *Unmarshaler) unmarshalWithFullName(m Valuer, v interface{}, fullName st
 	}
 
 	rte := reflect.TypeOf(v).Elem()
+	if rte.Kind() != reflect.Struct {
+		return errValueNotStruct
+	}
+
 	rve := rv.Elem()
 	numFields := rte.NumField()
 	for i := 0; i < numFields; i++ {
@@ -345,7 +350,7 @@ func (u *Unmarshaler) processNamedFieldWithValue(field reflect.StructField, valu
 			options := opts.options()
 			if len(options) > 0 {
 				if !stringx.Contains(options, mapValue.(string)) {
-					return fmt.Errorf(`error: value "%s" for field "%s" is not defined in opts "%v"`,
+					return fmt.Errorf(`error: value "%s" for field "%s" is not defined in options "%v"`,
 						mapValue, key, options)
 				}
 			}

core/mapping/unmarshaler_test.go

@@ -14,6 +14,13 @@ import (
 // so we only can test to 62 bits.
 const maxUintBitsToTest = 62
 
+func TestUnmarshalWithFullNameNotStruct(t *testing.T) {
+	var s map[string]interface{}
+	content := []byte(`{"name":"xiaoming"}`)
+	err := UnmarshalJsonBytes(content, &s)
+	assert.Equal(t, errValueNotStruct, err)
+}
+
 func TestUnmarshalWithoutTagName(t *testing.T) {
 	type inner struct {
 		Optional bool `key:",optional"`
@@ -2380,6 +2387,13 @@ func TestUnmarshalNestedMapSimpleTypeMatch(t *testing.T) {
 	assert.Equal(t, "1", c.Anything["id"])
 }
 
+func TestUnmarshalValuer(t *testing.T) {
+	unmarshaler := NewUnmarshaler(jsonTagKey)
+	var foo string
+	err := unmarshaler.UnmarshalValuer(nil, foo)
+	assert.NotNil(t, err)
+}
+
 func BenchmarkUnmarshalString(b *testing.B) {
 	type inner struct {
 		Value string `key:"value"`

core/mr/mapreduce.go

@@ -79,7 +79,7 @@ func Map(generate GenerateFunc, mapper MapFunc, opts ...Option) chan interface{}
 	collector := make(chan interface{}, options.workers)
 	done := syncx.NewDoneChan()
 
-	go mapDispatcher(mapper, source, collector, done.Done(), options.workers)
+	go executeMappers(mapper, source, collector, done.Done(), options.workers)
 
 	return collector
 }
@@ -124,8 +124,12 @@ func MapReduceWithSource(source <-chan interface{}, mapper MapperFunc, reducer R
 			}
 		}()
 		reducer(collector, writer, cancel)
+		drain(collector)
 	}()
-	go mapperDispatcher(mapper, source, collector, done.Done(), cancel, options.workers)
+
+	go executeMappers(func(item interface{}, w Writer) {
+		mapper(item, w, cancel)
+	}, source, collector, done.Done(), options.workers)
 
 	value, ok := <-output
 	if err := retErr.Load(); err != nil {
@@ -140,6 +144,7 @@ func MapReduceWithSource(source <-chan interface{}, mapper MapperFunc, reducer R
 func MapReduceVoid(generator GenerateFunc, mapper MapperFunc, reducer VoidReducerFunc, opts ...Option) error {
 	_, err := MapReduce(generator, mapper, func(input <-chan interface{}, writer Writer, cancel func(error)) {
 		reducer(input, cancel)
+		drain(input)
 		// We need to write a placeholder to let MapReduce to continue on reducer done,
 		// otherwise, all goroutines are waiting. The placeholder will be discarded by MapReduce.
 		writer.Write(lang.Placeholder)
@@ -224,20 +229,6 @@ func executeMappers(mapper MapFunc, input <-chan interface{}, collector chan<- i
 	}
 }
 
-func mapDispatcher(mapper MapFunc, input <-chan interface{}, collector chan<- interface{},
-	done <-chan lang.PlaceholderType, workers int) {
-	executeMappers(func(item interface{}, writer Writer) {
-		mapper(item, writer)
-	}, input, collector, done, workers)
-}
-
-func mapperDispatcher(mapper MapperFunc, input <-chan interface{}, collector chan<- interface{},
-	done <-chan lang.PlaceholderType, cancel func(error), workers int) {
-	executeMappers(func(item interface{}, writer Writer) {
-		mapper(item, writer, cancel)
-	}, input, collector, done, workers)
-}
-
 func newOptions() *mapReduceOptions {
 	return &mapReduceOptions{
 		workers: defaultWorkers,

core/proc/goroutines_test.go

@@ -0,0 +1,16 @@
+package proc
+
+import (
+	"log"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDumpGoroutines(t *testing.T) {
+	var buf strings.Builder
+	log.SetOutput(&buf)
+	dumpGoroutines()
+	assert.True(t, strings.Contains(buf.String(), ".dump"))
+}

core/proc/profile_test.go

@@ -0,0 +1,21 @@
+package proc
+
+import (
+	"log"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProfile(t *testing.T) {
+	var buf strings.Builder
+	log.SetOutput(&buf)
+	profiler := StartProfile()
+	// start again should not work
+	assert.NotNil(t, StartProfile())
+	profiler.Stop()
+	// stop twice
+	profiler.Stop()
+	assert.True(t, strings.Contains(buf.String(), ".pprof"))
+}

core/proc/shutdown.go

@@ -32,7 +32,7 @@ func AddWrapUpListener(fn func()) (waitForCalled func()) {
 	return wrapUpListeners.addListener(fn)
 }
 
-func SetTimeoutToForceQuit(duration time.Duration) {
+func SetTimeToForceQuit(duration time.Duration) {
 	delayTimeBeforeForceQuit = duration
 }
 

core/stat/alert_test.go

@@ -0,0 +1,22 @@
+// +build linux
+
+package stat
+
+import (
+	"strconv"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReport(t *testing.T) {
+	var count int32
+	SetReporter(func(s string) {
+		atomic.AddInt32(&count, 1)
+	})
+	for i := 0; i < 10; i++ {
+		Report(strconv.Itoa(i))
+	}
+	assert.Equal(t, int32(1), count)
+}

core/stat/internal/cpu_linux_test.go

@@ -1,6 +1,14 @@
 package internal
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRefreshCpu(t *testing.T) {
+	assert.True(t, RefreshCpu() >= 0)
+}
 
 func BenchmarkRefreshCpu(b *testing.B) {
 	for i := 0; i < b.N; i++ {

core/stat/metrics_test.go

@@ -0,0 +1,37 @@
+package stat
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMetrics(t *testing.T) {
+	counts := []int{1, 5, 10, 100, 1000, 1000}
+	for _, count := range counts {
+		m := NewMetrics("foo")
+		m.SetName("bar")
+		for i := 0; i < count; i++ {
+			m.Add(Task{
+				Duration:    time.Millisecond * time.Duration(i),
+				Description: strconv.Itoa(i),
+			})
+		}
+		m.AddDrop()
+		var writer mockedWriter
+		SetReportWriter(&writer)
+		m.executor.Flush()
+		assert.Equal(t, "bar", writer.report.Name)
+	}
+}
+
+type mockedWriter struct {
+	report *StatReport
+}
+
+func (m *mockedWriter) Write(report *StatReport) error {
+	m.report = report
+	return nil
+}

core/stat/remotewriter_test.go

@@ -0,0 +1,30 @@
+package stat
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/h2non/gock.v1"
+)
+
+func TestRemoteWriter(t *testing.T) {
+	defer gock.Off()
+
+	gock.New("http://foo.com").Reply(200).BodyString("foo")
+	writer := NewRemoteWriter("http://foo.com")
+	err := writer.Write(&StatReport{
+		Name: "bar",
+	})
+	assert.Nil(t, err)
+}
+
+func TestRemoteWriterFail(t *testing.T) {
+	defer gock.Off()
+
+	gock.New("http://foo.com").Reply(503).BodyString("foo")
+	writer := NewRemoteWriter("http://foo.com")
+	err := writer.Write(&StatReport{
+		Name: "bar",
+	})
+	assert.NotNil(t, err)
+}

core/stores/cache/cache.go

@@ -1,4 +1,4 @@
-package internal
+package cache
 
 import (
 	"fmt"

core/stores/cache/cache_test.go

@@ -1,4 +1,4 @@
-package internal
+package cache
 
 import (
 	"encoding/json"
@@ -111,6 +111,45 @@ func TestCache_SetDel(t *testing.T) {
 		assert.Nil(t, c.GetCache(fmt.Sprintf("key/%d", i), &v))
 		assert.Equal(t, i, v)
 	}
+	assert.Nil(t, c.DelCache())
+	for i := 0; i < total; i++ {
+		assert.Nil(t, c.DelCache(fmt.Sprintf("key/%d", i)))
+	}
+	for i := 0; i < total; i++ {
+		var v int
+		assert.Equal(t, errPlaceholder, c.GetCache(fmt.Sprintf("key/%d", i), &v))
+		assert.Equal(t, 0, v)
+	}
+}
+
+func TestCache_OneNode(t *testing.T) {
+	const total = 1000
+	r := miniredis.NewMiniRedis()
+	assert.Nil(t, r.Start())
+	defer r.Close()
+	conf := ClusterConf{
+		{
+			RedisConf: redis.RedisConf{
+				Host: r.Addr(),
+				Type: redis.NodeType,
+			},
+			Weight: 100,
+		},
+	}
+	c := NewCache(conf, syncx.NewSharedCalls(), NewCacheStat("mock"), errPlaceholder)
+	for i := 0; i < total; i++ {
+		if i%2 == 0 {
+			assert.Nil(t, c.SetCache(fmt.Sprintf("key/%d", i), i))
+		} else {
+			assert.Nil(t, c.SetCacheWithExpire(fmt.Sprintf("key/%d", i), i, 0))
+		}
+	}
+	for i := 0; i < total; i++ {
+		var v int
+		assert.Nil(t, c.GetCache(fmt.Sprintf("key/%d", i), &v))
+		assert.Equal(t, i, v)
+	}
+	assert.Nil(t, c.DelCache())
 	for i := 0; i < total; i++ {
 		assert.Nil(t, c.DelCache(fmt.Sprintf("key/%d", i)))
 	}
@@ -188,6 +227,25 @@ func TestCache_Balance(t *testing.T) {
 	assert.Equal(t, total/10, count)
 }
 
+func TestCacheNoNode(t *testing.T) {
+	dispatcher := hash.NewConsistentHash()
+	c := cacheCluster{
+		dispatcher:  dispatcher,
+		errNotFound: errPlaceholder,
+	}
+	assert.NotNil(t, c.DelCache("foo"))
+	assert.NotNil(t, c.DelCache("foo", "bar", "any"))
+	assert.NotNil(t, c.GetCache("foo", nil))
+	assert.NotNil(t, c.SetCache("foo", nil))
+	assert.NotNil(t, c.SetCacheWithExpire("foo", nil, time.Second))
+	assert.NotNil(t, c.Take(nil, "foo", func(v interface{}) error {
+		return nil
+	}))
+	assert.NotNil(t, c.TakeWithExpire(nil, "foo", func(v interface{}, duration time.Duration) error {
+		return nil
+	}))
+}
+
 func calcEntropy(m map[int]int, total int) float64 {
 	var entropy float64
 

core/stores/cache/cacheconf.go

@@ -1,5 +1,3 @@
 package cache
 
-import "github.com/tal-tech/go-zero/core/stores/internal"
-
-type CacheConf = internal.ClusterConf
+type CacheConf = ClusterConf

core/stores/cache/cachenode.go

@@ -1,13 +1,13 @@
-package internal
+package cache
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"math/rand"
 	"sync"
 	"time"
 
+	"github.com/tal-tech/go-zero/core/jsonx"
 	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/mathx"
 	"github.com/tal-tech/go-zero/core/stat"
@@ -79,7 +79,7 @@ func (c cacheNode) SetCache(key string, v interface{}) error {
 }
 
 func (c cacheNode) SetCacheWithExpire(key string, v interface{}, expire time.Duration) error {
-	data, err := json.Marshal(v)
+	data, err := jsonx.Marshal(v)
 	if err != nil {
 		return err
 	}
@@ -168,7 +168,7 @@ func (c cacheNode) doTake(v interface{}, key string, query func(v interface{}) e
 			}
 		}
 
-		return json.Marshal(v)
+		return jsonx.Marshal(v)
 	})
 	if err != nil {
 		return err
@@ -181,11 +181,11 @@ func (c cacheNode) doTake(v interface{}, key string, query func(v interface{}) e
 		c.stat.IncrementHit()
 	}
 
-	return json.Unmarshal(val.([]byte), v)
+	return jsonx.Unmarshal(val.([]byte), v)
 }
 
 func (c cacheNode) processCache(key string, data string, v interface{}) error {
-	err := json.Unmarshal([]byte(data), v)
+	err := jsonx.Unmarshal([]byte(data), v)
 	if err == nil {
 		return nil
 	}

core/stores/cache/cachenode_test.go

@@ -0,0 +1,208 @@
+package cache
+
+import (
+	"errors"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis"
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/core/mathx"
+	"github.com/tal-tech/go-zero/core/stat"
+	"github.com/tal-tech/go-zero/core/stores/redis"
+	"github.com/tal-tech/go-zero/core/syncx"
+)
+
+var errTestNotFound = errors.New("not found")
+
+func init() {
+	logx.Disable()
+	stat.SetReporter(nil)
+}
+
+func TestCacheNode_DelCache(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+	defer s.Close()
+
+	cn := cacheNode{
+		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewCacheStat("any"),
+		errNotFound:    errTestNotFound,
+	}
+	assert.Nil(t, cn.DelCache())
+	assert.Nil(t, cn.DelCache([]string{}...))
+	assert.Nil(t, cn.DelCache(make([]string, 0)...))
+	cn.SetCache("first", "one")
+	assert.Nil(t, cn.DelCache("first"))
+	cn.SetCache("first", "one")
+	cn.SetCache("second", "two")
+	assert.Nil(t, cn.DelCache("first", "second"))
+}
+
+func TestCacheNode_InvalidCache(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+	defer s.Close()
+
+	cn := cacheNode{
+		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewCacheStat("any"),
+		errNotFound:    errTestNotFound,
+	}
+	s.Set("any", "value")
+	var str string
+	assert.NotNil(t, cn.GetCache("any", &str))
+	assert.Equal(t, "", str)
+	_, err = s.Get("any")
+	assert.Equal(t, miniredis.ErrKeyNotFound, err)
+}
+
+func TestCacheNode_Take(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+	defer s.Close()
+
+	cn := cacheNode{
+		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		barrier:        syncx.NewSharedCalls(),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewCacheStat("any"),
+		errNotFound:    errTestNotFound,
+	}
+	var str string
+	err = cn.Take(&str, "any", func(v interface{}) error {
+		*v.(*string) = "value"
+		return nil
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, "value", str)
+	assert.Nil(t, cn.GetCache("any", &str))
+	val, err := s.Get("any")
+	assert.Nil(t, err)
+	assert.Equal(t, `"value"`, val)
+}
+
+func TestCacheNode_TakeNotFound(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+	defer s.Close()
+
+	cn := cacheNode{
+		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		barrier:        syncx.NewSharedCalls(),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewCacheStat("any"),
+		errNotFound:    errTestNotFound,
+	}
+	var str string
+	err = cn.Take(&str, "any", func(v interface{}) error {
+		return errTestNotFound
+	})
+	assert.Equal(t, errTestNotFound, err)
+	assert.Equal(t, errTestNotFound, cn.GetCache("any", &str))
+	val, err := s.Get("any")
+	assert.Nil(t, err)
+	assert.Equal(t, `*`, val)
+
+	s.Set("any", "*")
+	err = cn.Take(&str, "any", func(v interface{}) error {
+		return nil
+	})
+	assert.Equal(t, errTestNotFound, err)
+	assert.Equal(t, errTestNotFound, cn.GetCache("any", &str))
+
+	s.Del("any")
+	var errDummy = errors.New("dummy")
+	err = cn.Take(&str, "any", func(v interface{}) error {
+		return errDummy
+	})
+	assert.Equal(t, errDummy, err)
+}
+
+func TestCacheNode_TakeWithExpire(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+	defer s.Close()
+
+	cn := cacheNode{
+		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		barrier:        syncx.NewSharedCalls(),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewCacheStat("any"),
+		errNotFound:    errors.New("any"),
+	}
+	var str string
+	err = cn.TakeWithExpire(&str, "any", func(v interface{}, expire time.Duration) error {
+		*v.(*string) = "value"
+		return nil
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, "value", str)
+	assert.Nil(t, cn.GetCache("any", &str))
+	val, err := s.Get("any")
+	assert.Nil(t, err)
+	assert.Equal(t, `"value"`, val)
+}
+
+func TestCacheNode_String(t *testing.T) {
+	s, err := miniredis.Run()
+	assert.Nil(t, err)
+	defer s.Close()
+
+	cn := cacheNode{
+		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		barrier:        syncx.NewSharedCalls(),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewCacheStat("any"),
+		errNotFound:    errors.New("any"),
+	}
+	assert.Equal(t, s.Addr(), cn.String())
+}
+
+func TestCacheValueWithBigInt(t *testing.T) {
+	s, err := miniredis.Run()
+	if err != nil {
+		t.Error(err)
+	}
+	defer s.Close()
+
+	cn := cacheNode{
+		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
+		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
+		barrier:        syncx.NewSharedCalls(),
+		lock:           new(sync.Mutex),
+		unstableExpiry: mathx.NewUnstable(expiryDeviation),
+		stat:           NewCacheStat("any"),
+		errNotFound:    errors.New("any"),
+	}
+
+	const (
+		key         = "key"
+		value int64 = 323427211229009810
+	)
+
+	assert.Nil(t, cn.SetCache(key, value))
+	var val interface{}
+	assert.Nil(t, cn.GetCache(key, &val))
+	assert.Equal(t, strconv.FormatInt(value, 10), fmt.Sprintf("%v", val))
+}

core/stores/cache/cacheopt.go

@@ -1,21 +1,45 @@
 package cache
 
-import (
-	"time"
+import "time"
 
-	"github.com/tal-tech/go-zero/core/stores/internal"
+const (
+	defaultExpiry         = time.Hour * 24 * 7
+	defaultNotFoundExpiry = time.Minute
 )
 
-type Option = internal.Option
+type (
+	Options struct {
+		Expiry         time.Duration
+		NotFoundExpiry time.Duration
+	}
+
+	Option func(o *Options)
+)
+
+func newOptions(opts ...Option) Options {
+	var o Options
+	for _, opt := range opts {
+		opt(&o)
+	}
+
+	if o.Expiry <= 0 {
+		o.Expiry = defaultExpiry
+	}
+	if o.NotFoundExpiry <= 0 {
+		o.NotFoundExpiry = defaultNotFoundExpiry
+	}
+
+	return o
+}
 
 func WithExpiry(expiry time.Duration) Option {
-	return func(o *internal.Options) {
+	return func(o *Options) {
 		o.Expiry = expiry
 	}
 }
 
 func WithNotFoundExpiry(expiry time.Duration) Option {
-	return func(o *internal.Options) {
+	return func(o *Options) {
 		o.NotFoundExpiry = expiry
 	}
 }

core/stores/cache/cachestat.go

@@ -1,4 +1,4 @@
-package internal
+package cache
 
 import (
 	"sync/atomic"

core/stores/cache/cleaner.go

@@ -1,4 +1,4 @@
-package internal
+package cache
 
 import (
 	"fmt"

core/stores/cache/cleaner_test.go

@@ -0,0 +1,56 @@
+package cache
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNextDelay(t *testing.T) {
+	tests := []struct {
+		name   string
+		input  time.Duration
+		output time.Duration
+		ok     bool
+	}{
+		{
+			name:   "second",
+			input:  time.Second,
+			output: time.Second * 5,
+			ok:     true,
+		},
+		{
+			name:   "5 seconds",
+			input:  time.Second * 5,
+			output: time.Minute,
+			ok:     true,
+		},
+		{
+			name:   "minute",
+			input:  time.Minute,
+			output: time.Minute * 5,
+			ok:     true,
+		},
+		{
+			name:   "5 minutes",
+			input:  time.Minute * 5,
+			output: time.Hour,
+			ok:     true,
+		},
+		{
+			name:   "hour",
+			input:  time.Hour,
+			output: 0,
+			ok:     false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			next, ok := nextDelay(test.input)
+			assert.Equal(t, test.ok, ok)
+			assert.Equal(t, test.output, next)
+		})
+	}
+}

core/stores/cache/config.go

@@ -1,4 +1,4 @@
-package internal
+package cache
 
 import "github.com/tal-tech/go-zero/core/stores/redis"
 

core/stores/cache/util.go

@@ -1,4 +1,4 @@
-package internal
+package cache
 
 import "strings"
 

core/stores/cache/util_test.go

@@ -0,0 +1,26 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFormatKeys(t *testing.T) {
+	assert.Equal(t, "a,b", formatKeys([]string{"a", "b"}))
+}
+
+func TestTotalWeights(t *testing.T) {
+	val := TotalWeights([]NodeConf{
+		{
+			Weight: -1,
+		},
+		{
+			Weight: 0,
+		},
+		{
+			Weight: 1,
+		},
+	})
+	assert.Equal(t, 1, val)
+}

core/stores/internal/cachenode_test.go

@@ -1,65 +0,0 @@
-package internal
-
-import (
-	"errors"
-	"math/rand"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/alicebob/miniredis"
-	"github.com/stretchr/testify/assert"
-	"github.com/tal-tech/go-zero/core/logx"
-	"github.com/tal-tech/go-zero/core/mathx"
-	"github.com/tal-tech/go-zero/core/stat"
-	"github.com/tal-tech/go-zero/core/stores/redis"
-)
-
-func init() {
-	logx.Disable()
-	stat.SetReporter(nil)
-}
-
-func TestCacheNode_DelCache(t *testing.T) {
-	s, err := miniredis.Run()
-	assert.Nil(t, err)
-	defer s.Close()
-
-	cn := cacheNode{
-		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
-		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		lock:           new(sync.Mutex),
-		unstableExpiry: mathx.NewUnstable(expiryDeviation),
-		stat:           NewCacheStat("any"),
-		errNotFound:    errors.New("any"),
-	}
-	assert.Nil(t, cn.DelCache())
-	assert.Nil(t, cn.DelCache([]string{}...))
-	assert.Nil(t, cn.DelCache(make([]string, 0)...))
-	cn.SetCache("first", "one")
-	assert.Nil(t, cn.DelCache("first"))
-	cn.SetCache("first", "one")
-	cn.SetCache("second", "two")
-	assert.Nil(t, cn.DelCache("first", "second"))
-}
-
-func TestCacheNode_InvalidCache(t *testing.T) {
-	s, err := miniredis.Run()
-	assert.Nil(t, err)
-	defer s.Close()
-
-	cn := cacheNode{
-		rds:            redis.NewRedis(s.Addr(), redis.NodeType),
-		r:              rand.New(rand.NewSource(time.Now().UnixNano())),
-		lock:           new(sync.Mutex),
-		unstableExpiry: mathx.NewUnstable(expiryDeviation),
-		stat:           NewCacheStat("any"),
-		errNotFound:    errors.New("any"),
-	}
-	s.Set("any", "value")
-	var str string
-	assert.NotNil(t, cn.GetCache("any", &str))
-	assert.Equal(t, "", str)
-	_, err = s.Get("any")
-	assert.Equal(t, miniredis.ErrKeyNotFound, err)
-}

core/stores/internal/cacheopt.go

@@ -1,33 +0,0 @@
-package internal
-
-import "time"
-
-const (
-	defaultExpiry         = time.Hour * 24 * 7
-	defaultNotFoundExpiry = time.Minute
-)
-
-type (
-	Options struct {
-		Expiry         time.Duration
-		NotFoundExpiry time.Duration
-	}
-
-	Option func(o *Options)
-)
-
-func newOptions(opts ...Option) Options {
-	var o Options
-	for _, opt := range opts {
-		opt(&o)
-	}
-
-	if o.Expiry <= 0 {
-		o.Expiry = defaultExpiry
-	}
-	if o.NotFoundExpiry <= 0 {
-		o.NotFoundExpiry = defaultNotFoundExpiry
-	}
-
-	return o
-}

core/stores/kv/config.go

@@ -1,5 +1,7 @@
 package kv
 
-import "github.com/tal-tech/go-zero/core/stores/internal"
+import (
+	"github.com/tal-tech/go-zero/core/stores/cache"
+)
 
-type KvConf = internal.ClusterConf
+type KvConf = cache.ClusterConf

core/stores/kv/store.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/tal-tech/go-zero/core/errorx"
 	"github.com/tal-tech/go-zero/core/hash"
-	"github.com/tal-tech/go-zero/core/stores/internal"
+	"github.com/tal-tech/go-zero/core/stores/cache"
 	"github.com/tal-tech/go-zero/core/stores/redis"
 )
 
@@ -81,7 +81,7 @@ type (
 )
 
 func NewStore(c KvConf) Store {
-	if len(c) == 0 || internal.TotalWeights(c) <= 0 {
+	if len(c) == 0 || cache.TotalWeights(c) <= 0 {
 		log.Fatal("no cache nodes")
 	}
 

core/stores/kv/store_test.go

@@ -6,7 +6,8 @@ import (
 
 	"github.com/alicebob/miniredis"
 	"github.com/stretchr/testify/assert"
-	"github.com/tal-tech/go-zero/core/stores/internal"
+	"github.com/tal-tech/go-zero/core/hash"
+	"github.com/tal-tech/go-zero/core/stores/cache"
 	"github.com/tal-tech/go-zero/core/stores/redis"
 	"github.com/tal-tech/go-zero/core/stringx"
 )
@@ -15,6 +16,10 @@ var s1, _ = miniredis.Run()
 var s2, _ = miniredis.Run()
 
 func TestRedis_Exists(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Exists("foo")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		ok, err := client.Exists("a")
 		assert.Nil(t, err)
@@ -27,6 +32,10 @@ func TestRedis_Exists(t *testing.T) {
 }
 
 func TestRedis_Eval(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Eval(`redis.call("EXISTS", KEYS[1])`, "key1")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		_, err := client.Eval(`redis.call("EXISTS", KEYS[1])`, "notexist")
 		assert.Equal(t, redis.Nil, err)
@@ -41,6 +50,12 @@ func TestRedis_Eval(t *testing.T) {
 }
 
 func TestRedis_Hgetall(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	err := store.Hset("a", "aa", "aaa")
+	assert.NotNil(t, err)
+	_, err = store.Hgetall("a")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
@@ -54,6 +69,10 @@ func TestRedis_Hgetall(t *testing.T) {
 }
 
 func TestRedis_Hvals(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Hvals("a")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
@@ -64,6 +83,10 @@ func TestRedis_Hvals(t *testing.T) {
 }
 
 func TestRedis_Hsetnx(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Hsetnx("a", "dd", "ddd")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
@@ -80,6 +103,12 @@ func TestRedis_Hsetnx(t *testing.T) {
 }
 
 func TestRedis_HdelHlen(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Hdel("a", "aa")
+	assert.NotNil(t, err)
+	_, err = store.Hlen("a")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
@@ -96,6 +125,10 @@ func TestRedis_HdelHlen(t *testing.T) {
 }
 
 func TestRedis_HIncrBy(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Hincrby("key", "field", 3)
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		val, err := client.Hincrby("key", "field", 2)
 		assert.Nil(t, err)
@@ -107,6 +140,10 @@ func TestRedis_HIncrBy(t *testing.T) {
 }
 
 func TestRedis_Hkeys(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Hkeys("a")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
@@ -117,6 +154,10 @@ func TestRedis_Hkeys(t *testing.T) {
 }
 
 func TestRedis_Hmget(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Hmget("a", "aa", "bb")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
@@ -130,6 +171,12 @@ func TestRedis_Hmget(t *testing.T) {
 }
 
 func TestRedis_Hmset(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	err := store.Hmset("a", map[string]string{
+		"aa": "aaa",
+	})
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		assert.Nil(t, client.Hmset("a", map[string]string{
 			"aa": "aaa",
@@ -142,6 +189,10 @@ func TestRedis_Hmset(t *testing.T) {
 }
 
 func TestRedis_Incr(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Incr("a")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		val, err := client.Incr("a")
 		assert.Nil(t, err)
@@ -153,6 +204,10 @@ func TestRedis_Incr(t *testing.T) {
 }
 
 func TestRedis_IncrBy(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Incrby("a", 2)
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		val, err := client.Incrby("a", 2)
 		assert.Nil(t, err)
@@ -164,6 +219,20 @@ func TestRedis_IncrBy(t *testing.T) {
 }
 
 func TestRedis_List(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Lpush("key", "value1", "value2")
+	assert.NotNil(t, err)
+	_, err = store.Rpush("key", "value3", "value4")
+	assert.NotNil(t, err)
+	_, err = store.Llen("key")
+	assert.NotNil(t, err)
+	_, err = store.Lrange("key", 0, 10)
+	assert.NotNil(t, err)
+	_, err = store.Lpop("key")
+	assert.NotNil(t, err)
+	_, err = store.Lrem("key", 0, "val")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		val, err := client.Lpush("key", "value1", "value2")
 		assert.Nil(t, err)
@@ -202,6 +271,14 @@ func TestRedis_List(t *testing.T) {
 }
 
 func TestRedis_Persist(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Persist("key")
+	assert.NotNil(t, err)
+	err = store.Expire("key", 5)
+	assert.NotNil(t, err)
+	err = store.Expireat("key", time.Now().Unix()+5)
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		ok, err := client.Persist("key")
 		assert.Nil(t, err)
@@ -225,8 +302,16 @@ func TestRedis_Persist(t *testing.T) {
 }
 
 func TestRedis_Sscan(t *testing.T) {
+	key := "list"
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Sadd(key, nil)
+	assert.NotNil(t, err)
+	_, _, err = store.Sscan(key, 0, "", 100)
+	assert.NotNil(t, err)
+	_, err = store.Del(key)
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
-		key := "list"
 		var list []string
 		for i := 0; i < 1550; i++ {
 			list = append(list, stringx.Randn(i))
@@ -254,6 +339,20 @@ func TestRedis_Sscan(t *testing.T) {
 }
 
 func TestRedis_Set(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Scard("key")
+	assert.NotNil(t, err)
+	_, err = store.Sismember("key", 2)
+	assert.NotNil(t, err)
+	_, err = store.Srem("key", 3, 4)
+	assert.NotNil(t, err)
+	_, err = store.Smembers("key")
+	assert.NotNil(t, err)
+	_, err = store.Srandmember("key", 1)
+	assert.NotNil(t, err)
+	_, err = store.Spop("key")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		num, err := client.Sadd("key", 1, 2, 3, 4)
 		assert.Nil(t, err)
@@ -290,6 +389,14 @@ func TestRedis_Set(t *testing.T) {
 }
 
 func TestRedis_SetGetDel(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	err := store.Set("hello", "world")
+	assert.NotNil(t, err)
+	_, err = store.Get("hello")
+	assert.NotNil(t, err)
+	_, err = store.Del("hello")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		err := client.Set("hello", "world")
 		assert.Nil(t, err)
@@ -303,6 +410,16 @@ func TestRedis_SetGetDel(t *testing.T) {
 }
 
 func TestRedis_SetExNx(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	err := store.Setex("hello", "world", 5)
+	assert.NotNil(t, err)
+	_, err = store.Setnx("newhello", "newworld")
+	assert.NotNil(t, err)
+	_, err = store.Ttl("hello")
+	assert.NotNil(t, err)
+	_, err = store.SetnxEx("newhello", "newworld", 5)
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		err := client.Setex("hello", "world", 5)
 		assert.Nil(t, err)
@@ -337,6 +454,16 @@ func TestRedis_SetExNx(t *testing.T) {
 }
 
 func TestRedis_SetGetDelHashField(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	err := store.Hset("key", "field", "value")
+	assert.NotNil(t, err)
+	_, err = store.Hget("key", "field")
+	assert.NotNil(t, err)
+	_, err = store.Hexists("key", "field")
+	assert.NotNil(t, err)
+	_, err = store.Hdel("key", "field")
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		err := client.Hset("key", "field", "value")
 		assert.Nil(t, err)
@@ -356,6 +483,48 @@ func TestRedis_SetGetDelHashField(t *testing.T) {
 }
 
 func TestRedis_SortedSet(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Zadd("key", 1, "value1")
+	assert.NotNil(t, err)
+	_, err = store.Zscore("key", "value1")
+	assert.NotNil(t, err)
+	_, err = store.Zcount("key", 6, 7)
+	assert.NotNil(t, err)
+	_, err = store.Zincrby("key", 3, "value1")
+	assert.NotNil(t, err)
+	_, err = store.Zrank("key", "value2")
+	assert.NotNil(t, err)
+	_, err = store.Zrem("key", "value2", "value3")
+	assert.NotNil(t, err)
+	_, err = store.Zremrangebyscore("key", 6, 7)
+	assert.NotNil(t, err)
+	_, err = store.Zremrangebyrank("key", 1, 2)
+	assert.NotNil(t, err)
+	_, err = store.Zcard("key")
+	assert.NotNil(t, err)
+	_, err = store.Zrange("key", 0, -1)
+	assert.NotNil(t, err)
+	_, err = store.Zrevrange("key", 0, -1)
+	assert.NotNil(t, err)
+	_, err = store.ZrangeWithScores("key", 0, -1)
+	assert.NotNil(t, err)
+	_, err = store.ZrangebyscoreWithScores("key", 5, 8)
+	assert.NotNil(t, err)
+	_, err = store.ZrangebyscoreWithScoresAndLimit("key", 5, 8, 1, 1)
+	assert.NotNil(t, err)
+	_, err = store.ZrevrangebyscoreWithScores("key", 5, 8)
+	assert.NotNil(t, err)
+	_, err = store.ZrevrangebyscoreWithScoresAndLimit("key", 5, 8, 1, 1)
+	assert.NotNil(t, err)
+	_, err = store.Zadds("key", redis.Pair{
+		Key:   "value2",
+		Score: 6,
+	}, redis.Pair{
+		Key:   "value3",
+		Score: 7,
+	})
+	assert.NotNil(t, err)
+
 	runOnCluster(t, func(client Store) {
 		ok, err := client.Zadd("key", 1, "value1")
 		assert.Nil(t, err)
@@ -471,6 +640,30 @@ func TestRedis_SortedSet(t *testing.T) {
 				Score: 5,
 			},
 		}, pairs)
+		val, err = client.Zadds("key", redis.Pair{
+			Key:   "value2",
+			Score: 6,
+		}, redis.Pair{
+			Key:   "value3",
+			Score: 7,
+		})
+		assert.Nil(t, err)
+		assert.Equal(t, int64(2), val)
+	})
+}
+
+func TestRedis_HyperLogLog(t *testing.T) {
+	store := clusterStore{dispatcher: hash.NewConsistentHash()}
+	_, err := store.Pfadd("key")
+	assert.NotNil(t, err)
+	_, err = store.Pfcount("key")
+	assert.NotNil(t, err)
+
+	runOnCluster(t, func(cluster Store) {
+		_, err := cluster.Pfadd("key")
+		assert.NotNil(t, err)
+		_, err = cluster.Pfcount("key")
+		assert.NotNil(t, err)
 	})
 }
 
@@ -478,7 +671,7 @@ func runOnCluster(t *testing.T, fn func(cluster Store)) {
 	s1.FlushAll()
 	s2.FlushAll()
 
-	store := NewStore([]internal.NodeConf{
+	store := NewStore([]cache.NodeConf{
 		{
 			RedisConf: redis.RedisConf{
 				Host: s1.Addr(),

core/stores/mongo/collection.go

@@ -7,6 +7,7 @@ import (
 	"github.com/globalsign/mgo"
 	"github.com/tal-tech/go-zero/core/breaker"
 	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/core/stores/mongo/internal"
 	"github.com/tal-tech/go-zero/core/timex"
 )
 
@@ -29,8 +30,9 @@ type (
 	}
 
 	decoratedCollection struct {
-		*mgo.Collection
-		brk breaker.Breaker
+		name       string
+		collection internal.MgoCollection
+		brk        breaker.Breaker
 	}
 
 	keepablePromise struct {
@@ -41,7 +43,8 @@ type (
 
 func newCollection(collection *mgo.Collection) Collection {
 	return &decoratedCollection{
-		Collection: collection,
+		name:       collection.FullName,
+		collection: collection,
 		brk:        breaker.NewBreaker(),
 	}
 }
@@ -54,7 +57,7 @@ func (c *decoratedCollection) Find(query interface{}) Query {
 
 	startTime := timex.Now()
 	return promisedQuery{
-		Query: c.Collection.Find(query),
+		Query: c.collection.Find(query),
 		promise: keepablePromise{
 			promise: promise,
 			log: func(err error) {
@@ -73,7 +76,7 @@ func (c *decoratedCollection) FindId(id interface{}) Query {
 
 	startTime := timex.Now()
 	return promisedQuery{
-		Query: c.Collection.FindId(id),
+		Query: c.collection.FindId(id),
 		promise: keepablePromise{
 			promise: promise,
 			log: func(err error) {
@@ -92,7 +95,7 @@ func (c *decoratedCollection) Insert(docs ...interface{}) (err error) {
 			c.logDuration("insert", duration, err, docs...)
 		}()
 
-		return c.Collection.Insert(docs...)
+		return c.collection.Insert(docs...)
 	}, acceptable)
 }
 
@@ -104,7 +107,7 @@ func (c *decoratedCollection) Pipe(pipeline interface{}) Pipe {
 
 	startTime := timex.Now()
 	return promisedPipe{
-		Pipe: c.Collection.Pipe(pipeline),
+		Pipe: c.collection.Pipe(pipeline),
 		promise: keepablePromise{
 			promise: promise,
 			log: func(err error) {
@@ -123,7 +126,7 @@ func (c *decoratedCollection) Remove(selector interface{}) (err error) {
 			c.logDuration("remove", duration, err, selector)
 		}()
 
-		return c.Collection.Remove(selector)
+		return c.collection.Remove(selector)
 	}, acceptable)
 }
 
@@ -135,7 +138,7 @@ func (c *decoratedCollection) RemoveAll(selector interface{}) (info *mgo.ChangeI
 			c.logDuration("removeAll", duration, err, selector)
 		}()
 
-		info, err = c.Collection.RemoveAll(selector)
+		info, err = c.collection.RemoveAll(selector)
 		return err
 	}, acceptable)
 
@@ -150,7 +153,7 @@ func (c *decoratedCollection) RemoveId(id interface{}) (err error) {
 			c.logDuration("removeId", duration, err, id)
 		}()
 
-		return c.Collection.RemoveId(id)
+		return c.collection.RemoveId(id)
 	}, acceptable)
 }
 
@@ -162,7 +165,7 @@ func (c *decoratedCollection) Update(selector, update interface{}) (err error) {
 			c.logDuration("update", duration, err, selector, update)
 		}()
 
-		return c.Collection.Update(selector, update)
+		return c.collection.Update(selector, update)
 	}, acceptable)
 }
 
@@ -174,7 +177,7 @@ func (c *decoratedCollection) UpdateId(id, update interface{}) (err error) {
 			c.logDuration("updateId", duration, err, id, update)
 		}()
 
-		return c.Collection.UpdateId(id, update)
+		return c.collection.UpdateId(id, update)
 	}, acceptable)
 }
 
@@ -186,7 +189,7 @@ func (c *decoratedCollection) Upsert(selector, update interface{}) (info *mgo.Ch
 			c.logDuration("upsert", duration, err, selector, update)
 		}()
 
-		info, err = c.Collection.Upsert(selector, update)
+		info, err = c.collection.Upsert(selector, update)
 		return err
 	}, acceptable)
 
@@ -200,17 +203,17 @@ func (c *decoratedCollection) logDuration(method string, duration time.Duration,
 	} else if err != nil {
 		if duration > slowThreshold {
 			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - fail(%s) - %s",
-				c.FullName, method, err.Error(), string(content))
+				c.name, method, err.Error(), string(content))
 		} else {
 			logx.WithDuration(duration).Infof("mongo(%s) - %s - fail(%s) - %s",
-				c.FullName, method, err.Error(), string(content))
+				c.name, method, err.Error(), string(content))
 		}
 	} else {
 		if duration > slowThreshold {
 			logx.WithDuration(duration).Slowf("[MONGO] mongo(%s) - slowcall - %s - ok - %s",
-				c.FullName, method, string(content))
+				c.name, method, string(content))
 		} else {
-			logx.WithDuration(duration).Infof("mongo(%s) - %s - ok - %s", c.FullName, method, string(content))
+			logx.WithDuration(duration).Infof("mongo(%s) - %s - ok - %s", c.name, method, string(content))
 		}
 	}
 }

core/stores/mongo/collection_test.go

@@ -5,10 +5,20 @@ import (
 	"testing"
 
 	"github.com/globalsign/mgo"
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/breaker"
+	"github.com/tal-tech/go-zero/core/logx"
+	"github.com/tal-tech/go-zero/core/stores/mongo/internal"
 	"github.com/tal-tech/go-zero/core/stringx"
 )
 
+var errDummy = errors.New("dummy")
+
+func init() {
+	logx.Disable()
+}
+
 func TestKeepPromise_accept(t *testing.T) {
 	p := new(mockPromise)
 	kp := keepablePromise{
@@ -56,6 +66,206 @@ func TestKeepPromise_keep(t *testing.T) {
 	}
 }
 
+func TestNewCollection(t *testing.T) {
+	col := newCollection(&mgo.Collection{
+		Database: nil,
+		Name:     "foo",
+		FullName: "bar",
+	})
+	assert.Equal(t, "bar", col.(*decoratedCollection).name)
+}
+
+func TestCollectionFind(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	var query mgo.Query
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().Find(gomock.Any()).Return(&query)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	actual := c.Find(nil)
+	switch v := actual.(type) {
+	case promisedQuery:
+		assert.Equal(t, &query, v.Query)
+		assert.Equal(t, errDummy, v.promise.keep(errDummy))
+	default:
+		t.Fail()
+	}
+	c.brk = new(dropBreaker)
+	actual = c.Find(nil)
+	assert.Equal(t, rejectedQuery{}, actual)
+}
+
+func TestCollectionFindId(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	var query mgo.Query
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().FindId(gomock.Any()).Return(&query)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	actual := c.FindId(nil)
+	switch v := actual.(type) {
+	case promisedQuery:
+		assert.Equal(t, &query, v.Query)
+		assert.Equal(t, errDummy, v.promise.keep(errDummy))
+	default:
+		t.Fail()
+	}
+	c.brk = new(dropBreaker)
+	actual = c.FindId(nil)
+	assert.Equal(t, rejectedQuery{}, actual)
+}
+
+func TestCollectionInsert(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().Insert(nil, nil).Return(errDummy)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	err := c.Insert(nil, nil)
+	assert.Equal(t, errDummy, err)
+	c.brk = new(dropBreaker)
+	err = c.Insert(nil, nil)
+	assert.Equal(t, errDummy, err)
+}
+
+func TestCollectionPipe(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	var pipe mgo.Pipe
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().Pipe(gomock.Any()).Return(&pipe)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	actual := c.Pipe(nil)
+	switch v := actual.(type) {
+	case promisedPipe:
+		assert.Equal(t, &pipe, v.Pipe)
+		assert.Equal(t, errDummy, v.promise.keep(errDummy))
+	default:
+		t.Fail()
+	}
+	c.brk = new(dropBreaker)
+	actual = c.Pipe(nil)
+	assert.Equal(t, rejectedPipe{}, actual)
+}
+
+func TestCollectionRemove(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().Remove(gomock.Any()).Return(errDummy)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	err := c.Remove(nil)
+	assert.Equal(t, errDummy, err)
+	c.brk = new(dropBreaker)
+	err = c.Remove(nil)
+	assert.Equal(t, errDummy, err)
+}
+
+func TestCollectionRemoveAll(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().RemoveAll(gomock.Any()).Return(nil, errDummy)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	_, err := c.RemoveAll(nil)
+	assert.Equal(t, errDummy, err)
+	c.brk = new(dropBreaker)
+	_, err = c.RemoveAll(nil)
+	assert.Equal(t, errDummy, err)
+}
+
+func TestCollectionRemoveId(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().RemoveId(gomock.Any()).Return(errDummy)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	err := c.RemoveId(nil)
+	assert.Equal(t, errDummy, err)
+	c.brk = new(dropBreaker)
+	err = c.RemoveId(nil)
+	assert.Equal(t, errDummy, err)
+}
+
+func TestCollectionUpdate(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().Update(gomock.Any(), gomock.Any()).Return(errDummy)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	err := c.Update(nil, nil)
+	assert.Equal(t, errDummy, err)
+	c.brk = new(dropBreaker)
+	err = c.Update(nil, nil)
+	assert.Equal(t, errDummy, err)
+}
+
+func TestCollectionUpdateId(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().UpdateId(gomock.Any(), gomock.Any()).Return(errDummy)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	err := c.UpdateId(nil, nil)
+	assert.Equal(t, errDummy, err)
+	c.brk = new(dropBreaker)
+	err = c.UpdateId(nil, nil)
+	assert.Equal(t, errDummy, err)
+}
+
+func TestCollectionUpsert(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	col := internal.NewMockMgoCollection(ctrl)
+	col.EXPECT().Upsert(gomock.Any(), gomock.Any()).Return(nil, errDummy)
+	c := decoratedCollection{
+		collection: col,
+		brk:        breaker.NewBreaker(),
+	}
+	_, err := c.Upsert(nil, nil)
+	assert.Equal(t, errDummy, err)
+	c.brk = new(dropBreaker)
+	_, err = c.Upsert(nil, nil)
+	assert.Equal(t, errDummy, err)
+}
+
 type mockPromise struct {
 	accepted bool
 	reason   string
@@ -68,3 +278,31 @@ func (p *mockPromise) Accept() {
 func (p *mockPromise) Reject(reason string) {
 	p.reason = reason
 }
+
+type dropBreaker struct {
+}
+
+func (d *dropBreaker) Name() string {
+	return "dummy"
+}
+
+func (d *dropBreaker) Allow() (breaker.Promise, error) {
+	return nil, errDummy
+}
+
+func (d *dropBreaker) Do(req func() error) error {
+	return nil
+}
+
+func (d *dropBreaker) DoWithAcceptable(req func() error, acceptable breaker.Acceptable) error {
+	return errDummy
+}
+
+func (d *dropBreaker) DoWithFallback(req func() error, fallback func(err error) error) error {
+	return nil
+}
+
+func (d *dropBreaker) DoWithFallbackAcceptable(req func() error, fallback func(err error) error,
+	acceptable breaker.Acceptable) error {
+	return nil
+}

core/stores/mongo/internal/collection.go

@@ -0,0 +1,17 @@
+//go:generate mockgen -package internal -destination collection_mock.go -source collection.go
+package internal
+
+import "github.com/globalsign/mgo"
+
+type MgoCollection interface {
+	Find(query interface{}) *mgo.Query
+	FindId(id interface{}) *mgo.Query
+	Insert(docs ...interface{}) error
+	Pipe(pipeline interface{}) *mgo.Pipe
+	Remove(selector interface{}) error
+	RemoveAll(selector interface{}) (*mgo.ChangeInfo, error)
+	RemoveId(id interface{}) error
+	Update(selector, update interface{}) error
+	UpdateId(id, update interface{}) error
+	Upsert(selector, update interface{}) (*mgo.ChangeInfo, error)
+}

core/stores/mongo/internal/collection_mock.go

@@ -0,0 +1,180 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: collection.go
+
+// Package internal is a generated GoMock package.
+package internal
+
+import (
+	mgo "github.com/globalsign/mgo"
+	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
+)
+
+// MockMgoCollection is a mock of MgoCollection interface
+type MockMgoCollection struct {
+	ctrl     *gomock.Controller
+	recorder *MockMgoCollectionMockRecorder
+}
+
+// MockMgoCollectionMockRecorder is the mock recorder for MockMgoCollection
+type MockMgoCollectionMockRecorder struct {
+	mock *MockMgoCollection
+}
+
+// NewMockMgoCollection creates a new mock instance
+func NewMockMgoCollection(ctrl *gomock.Controller) *MockMgoCollection {
+	mock := &MockMgoCollection{ctrl: ctrl}
+	mock.recorder = &MockMgoCollectionMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockMgoCollection) EXPECT() *MockMgoCollectionMockRecorder {
+	return m.recorder
+}
+
+// Find mocks base method
+func (m *MockMgoCollection) Find(query interface{}) *mgo.Query {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Find", query)
+	ret0, _ := ret[0].(*mgo.Query)
+	return ret0
+}
+
+// Find indicates an expected call of Find
+func (mr *MockMgoCollectionMockRecorder) Find(query interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockMgoCollection)(nil).Find), query)
+}
+
+// FindId mocks base method
+func (m *MockMgoCollection) FindId(id interface{}) *mgo.Query {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "FindId", id)
+	ret0, _ := ret[0].(*mgo.Query)
+	return ret0
+}
+
+// FindId indicates an expected call of FindId
+func (mr *MockMgoCollectionMockRecorder) FindId(id interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindId", reflect.TypeOf((*MockMgoCollection)(nil).FindId), id)
+}
+
+// Insert mocks base method
+func (m *MockMgoCollection) Insert(docs ...interface{}) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{}
+	for _, a := range docs {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Insert", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Insert indicates an expected call of Insert
+func (mr *MockMgoCollectionMockRecorder) Insert(docs ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockMgoCollection)(nil).Insert), docs...)
+}
+
+// Pipe mocks base method
+func (m *MockMgoCollection) Pipe(pipeline interface{}) *mgo.Pipe {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Pipe", pipeline)
+	ret0, _ := ret[0].(*mgo.Pipe)
+	return ret0
+}
+
+// Pipe indicates an expected call of Pipe
+func (mr *MockMgoCollectionMockRecorder) Pipe(pipeline interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pipe", reflect.TypeOf((*MockMgoCollection)(nil).Pipe), pipeline)
+}
+
+// Remove mocks base method
+func (m *MockMgoCollection) Remove(selector interface{}) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Remove", selector)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Remove indicates an expected call of Remove
+func (mr *MockMgoCollectionMockRecorder) Remove(selector interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockMgoCollection)(nil).Remove), selector)
+}
+
+// RemoveAll mocks base method
+func (m *MockMgoCollection) RemoveAll(selector interface{}) (*mgo.ChangeInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveAll", selector)
+	ret0, _ := ret[0].(*mgo.ChangeInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// RemoveAll indicates an expected call of RemoveAll
+func (mr *MockMgoCollectionMockRecorder) RemoveAll(selector interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveAll", reflect.TypeOf((*MockMgoCollection)(nil).RemoveAll), selector)
+}
+
+// RemoveId mocks base method
+func (m *MockMgoCollection) RemoveId(id interface{}) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "RemoveId", id)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// RemoveId indicates an expected call of RemoveId
+func (mr *MockMgoCollectionMockRecorder) RemoveId(id interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveId", reflect.TypeOf((*MockMgoCollection)(nil).RemoveId), id)
+}
+
+// Update mocks base method
+func (m *MockMgoCollection) Update(selector, update interface{}) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Update", selector, update)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Update indicates an expected call of Update
+func (mr *MockMgoCollectionMockRecorder) Update(selector, update interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockMgoCollection)(nil).Update), selector, update)
+}
+
+// UpdateId mocks base method
+func (m *MockMgoCollection) UpdateId(id, update interface{}) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateId", id, update)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpdateId indicates an expected call of UpdateId
+func (mr *MockMgoCollectionMockRecorder) UpdateId(id, update interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateId", reflect.TypeOf((*MockMgoCollection)(nil).UpdateId), id, update)
+}
+
+// Upsert mocks base method
+func (m *MockMgoCollection) Upsert(selector, update interface{}) (*mgo.ChangeInfo, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Upsert", selector, update)
+	ret0, _ := ret[0].(*mgo.ChangeInfo)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Upsert indicates an expected call of Upsert
+func (mr *MockMgoCollectionMockRecorder) Upsert(selector, update interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockMgoCollection)(nil).Upsert), selector, update)
+}

core/stores/mongo/model.go

@@ -22,8 +22,8 @@ type (
 	}
 )
 
-func MustNewModel(url, database, collection string, opts ...Option) *Model {
-	model, err := NewModel(url, database, collection, opts...)
+func MustNewModel(url, collection string, opts ...Option) *Model {
+	model, err := NewModel(url, collection, opts...)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -31,15 +31,16 @@ func MustNewModel(url, database, collection string, opts ...Option) *Model {
 	return model
 }
 
-func NewModel(url, database, collection string, opts ...Option) (*Model, error) {
+func NewModel(url, collection string, opts ...Option) (*Model, error) {
 	session, err := getConcurrentSession(url)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Model{
-		session:    session,
-		db:         session.DB(database),
+		session: session,
+		// If name is empty, the database name provided in the dialed URL is used instead
+		db:         session.DB(""),
 		collection: collection,
 		opts:       opts,
 	}, nil

core/stores/mongoc/cachedcollection.go

@@ -2,7 +2,7 @@ package mongoc
 
 import (
 	"github.com/globalsign/mgo"
-	"github.com/tal-tech/go-zero/core/stores/internal"
+	"github.com/tal-tech/go-zero/core/stores/cache"
 	"github.com/tal-tech/go-zero/core/stores/mongo"
 	"github.com/tal-tech/go-zero/core/syncx"
 )
@@ -12,7 +12,7 @@ var (
 
 	// can't use one SharedCalls per conn, because multiple conns may share the same cache key.
 	sharedCalls = syncx.NewSharedCalls()
-	stats       = internal.NewCacheStat("mongoc")
+	stats       = cache.NewCacheStat("mongoc")
 )
 
 type (
@@ -20,11 +20,11 @@ type (
 
 	cachedCollection struct {
 		collection mongo.Collection
-		cache      internal.Cache
+		cache      cache.Cache
 	}
 )
 
-func newCollection(collection mongo.Collection, c internal.Cache) *cachedCollection {
+func newCollection(collection mongo.Collection, c cache.Cache) *cachedCollection {
 	return &cachedCollection{
 		collection: collection,
 		cache:      c,

core/stores/mongoc/cachedcollection_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/globalsign/mgo/bson"
 	"github.com/stretchr/testify/assert"
 	"github.com/tal-tech/go-zero/core/stat"
-	"github.com/tal-tech/go-zero/core/stores/internal"
+	"github.com/tal-tech/go-zero/core/stores/cache"
 	"github.com/tal-tech/go-zero/core/stores/mongo"
 	"github.com/tal-tech/go-zero/core/stores/redis"
 )
@@ -33,7 +33,7 @@ func TestStat(t *testing.T) {
 	}
 
 	r := redis.NewRedis(s.Addr(), redis.NodeType)
-	cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
+	cach := cache.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
 	c := newCollection(dummyConn{}, cach)
 
 	for i := 0; i < 10; i++ {
@@ -56,7 +56,7 @@ func TestStatCacheFails(t *testing.T) {
 	defer log.SetOutput(os.Stdout)
 
 	r := redis.NewRedis("localhost:59999", redis.NodeType)
-	cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
+	cach := cache.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
 	c := newCollection(dummyConn{}, cach)
 
 	for i := 0; i < 20; i++ {
@@ -79,7 +79,7 @@ func TestStatDbFails(t *testing.T) {
 	}
 
 	r := redis.NewRedis(s.Addr(), redis.NodeType)
-	cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
+	cach := cache.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
 	c := newCollection(dummyConn{}, cach)
 
 	for i := 0; i < 20; i++ {
@@ -103,7 +103,7 @@ func TestStatFromMemory(t *testing.T) {
 	}
 
 	r := redis.NewRedis(s.Addr(), redis.NodeType)
-	cach := internal.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
+	cach := cache.NewCacheNode(r, sharedCalls, stats, mgo.ErrNotFound)
 	c := newCollection(dummyConn{}, cach)
 
 	var all sync.WaitGroup

core/stores/mongoc/cachedmodel.go

@@ -5,19 +5,18 @@ import (
 
 	"github.com/globalsign/mgo"
 	"github.com/tal-tech/go-zero/core/stores/cache"
-	"github.com/tal-tech/go-zero/core/stores/internal"
 	"github.com/tal-tech/go-zero/core/stores/mongo"
 	"github.com/tal-tech/go-zero/core/stores/redis"
 )
 
 type Model struct {
 	*mongo.Model
-	cache              internal.Cache
+	cache              cache.Cache
 	generateCollection func(*mgo.Session) *cachedCollection
 }
 
-func MustNewNodeModel(url, database, collection string, rds *redis.Redis, opts ...cache.Option) *Model {
-	model, err := NewNodeModel(url, database, collection, rds, opts...)
+func MustNewNodeModel(url, collection string, rds *redis.Redis, opts ...cache.Option) *Model {
+	model, err := NewNodeModel(url, collection, rds, opts...)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -25,8 +24,8 @@ func MustNewNodeModel(url, database, collection string, rds *redis.Redis, opts .
 	return model
 }
 
-func MustNewModel(url, database, collection string, c cache.CacheConf, opts ...cache.Option) *Model {
-	model, err := NewModel(url, database, collection, c, opts...)
+func MustNewModel(url, collection string, c cache.CacheConf, opts ...cache.Option) *Model {
+	model, err := NewModel(url, collection, c, opts...)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -34,16 +33,16 @@ func MustNewModel(url, database, collection string, c cache.CacheConf, opts ...c
 	return model
 }
 
-func NewNodeModel(url, database, collection string, rds *redis.Redis, opts ...cache.Option) (*Model, error) {
-	c := internal.NewCacheNode(rds, sharedCalls, stats, mgo.ErrNotFound, opts...)
-	return createModel(url, database, collection, c, func(collection mongo.Collection) *cachedCollection {
+func NewNodeModel(url, collection string, rds *redis.Redis, opts ...cache.Option) (*Model, error) {
+	c := cache.NewCacheNode(rds, sharedCalls, stats, mgo.ErrNotFound, opts...)
+	return createModel(url, collection, c, func(collection mongo.Collection) *cachedCollection {
 		return newCollection(collection, c)
 	})
 }
 
-func NewModel(url, database, collection string, conf cache.CacheConf, opts ...cache.Option) (*Model, error) {
-	c := internal.NewCache(conf, sharedCalls, stats, mgo.ErrNotFound, opts...)
-	return createModel(url, database, collection, c, func(collection mongo.Collection) *cachedCollection {
+func NewModel(url, collection string, conf cache.CacheConf, opts ...cache.Option) (*Model, error) {
+	c := cache.NewCache(conf, sharedCalls, stats, mgo.ErrNotFound, opts...)
+	return createModel(url, collection, c, func(collection mongo.Collection) *cachedCollection {
 		return newCollection(collection, c)
 	})
 }
@@ -224,9 +223,9 @@ func (mm *Model) pipe(fn func(c *cachedCollection) mongo.Pipe) (mongo.Pipe, erro
 	return fn(mm.GetCollection(session)), nil
 }
 
-func createModel(url, database, collection string, c internal.Cache,
+func createModel(url, collection string, c cache.Cache,
 	create func(mongo.Collection) *cachedCollection) (*Model, error) {
-	model, err := mongo.NewModel(url, database, collection)
+	model, err := mongo.NewModel(url, collection)
 	if err != nil {
 		return nil, err
 	}

core/stores/redis/conf_test.go

@@ -0,0 +1,110 @@
+package redis
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tal-tech/go-zero/core/stringx"
+)
+
+func TestRedisConf(t *testing.T) {
+	tests := []struct {
+		name string
+		RedisConf
+		ok bool
+	}{
+		{
+			name: "missing host",
+			RedisConf: RedisConf{
+				Host: "",
+				Type: NodeType,
+				Pass: "",
+			},
+			ok: false,
+		},
+		{
+			name: "missing type",
+			RedisConf: RedisConf{
+				Host: "localhost:6379",
+				Type: "",
+				Pass: "",
+			},
+			ok: false,
+		},
+		{
+			name: "ok",
+			RedisConf: RedisConf{
+				Host: "localhost:6379",
+				Type: NodeType,
+				Pass: "",
+			},
+			ok: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(stringx.RandId(), func(t *testing.T) {
+			if test.ok {
+				assert.Nil(t, test.RedisConf.Validate())
+				assert.NotNil(t, test.RedisConf.NewRedis())
+			} else {
+				assert.NotNil(t, test.RedisConf.Validate())
+			}
+		})
+	}
+}
+
+func TestRedisKeyConf(t *testing.T) {
+	tests := []struct {
+		name string
+		RedisKeyConf
+		ok bool
+	}{
+		{
+			name: "missing host",
+			RedisKeyConf: RedisKeyConf{
+				RedisConf: RedisConf{
+					Host: "",
+					Type: NodeType,
+					Pass: "",
+				},
+				Key: "foo",
+			},
+			ok: false,
+		},
+		{
+			name: "missing key",
+			RedisKeyConf: RedisKeyConf{
+				RedisConf: RedisConf{
+					Host: "localhost:6379",
+					Type: NodeType,
+					Pass: "",
+				},
+				Key: "",
+			},
+			ok: false,
+		},
+		{
+			name: "ok",
+			RedisKeyConf: RedisKeyConf{
+				RedisConf: RedisConf{
+					Host: "localhost:6379",
+					Type: NodeType,
+					Pass: "",
+				},
+				Key: "foo",
+			},
+			ok: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if test.ok {
+				assert.Nil(t, test.RedisKeyConf.Validate())
+			} else {
+				assert.NotNil(t, test.RedisKeyConf.Validate())
+			}
+		})
+	}
+}

core/stores/redis/redis_test.go

@@ -7,11 +7,14 @@ import (
 	"time"
 
 	"github.com/alicebob/miniredis"
+	red "github.com/go-redis/redis"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestRedis_Exists(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		_, err := NewRedis(client.Addr, "").Exists("a")
+		assert.NotNil(t, err)
 		ok, err := client.Exists("a")
 		assert.Nil(t, err)
 		assert.False(t, ok)
@@ -24,7 +27,9 @@ func TestRedis_Exists(t *testing.T) {
 
 func TestRedis_Eval(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
-		_, err := client.Eval(`redis.call("EXISTS", KEYS[1])`, []string{"notexist"})
+		_, err := NewRedis(client.Addr, "").Eval(`redis.call("EXISTS", KEYS[1])`, []string{"notexist"})
+		assert.NotNil(t, err)
+		_, err = client.Eval(`redis.call("EXISTS", KEYS[1])`, []string{"notexist"})
 		assert.Equal(t, Nil, err)
 		err = client.Set("key1", "value1")
 		assert.Nil(t, err)
@@ -40,6 +45,8 @@ func TestRedis_Hgetall(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
+		_, err := NewRedis(client.Addr, "").Hgetall("a")
+		assert.NotNil(t, err)
 		vals, err := client.Hgetall("a")
 		assert.Nil(t, err)
 		assert.EqualValues(t, map[string]string{
@@ -51,8 +58,11 @@ func TestRedis_Hgetall(t *testing.T) {
 
 func TestRedis_Hvals(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		assert.NotNil(t, NewRedis(client.Addr, "").Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
+		_, err := NewRedis(client.Addr, "").Hvals("a")
+		assert.NotNil(t, err)
 		vals, err := client.Hvals("a")
 		assert.Nil(t, err)
 		assert.ElementsMatch(t, []string{"aaa", "bbb"}, vals)
@@ -63,6 +73,8 @@ func TestRedis_Hsetnx(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
+		_, err := NewRedis(client.Addr, "").Hsetnx("a", "bb", "ccc")
+		assert.NotNil(t, err)
 		ok, err := client.Hsetnx("a", "bb", "ccc")
 		assert.Nil(t, err)
 		assert.False(t, ok)
@@ -79,6 +91,8 @@ func TestRedis_HdelHlen(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
+		_, err := NewRedis(client.Addr, "").Hlen("a")
+		assert.NotNil(t, err)
 		num, err := client.Hlen("a")
 		assert.Nil(t, err)
 		assert.Equal(t, 2, num)
@@ -93,6 +107,8 @@ func TestRedis_HdelHlen(t *testing.T) {
 
 func TestRedis_HIncrBy(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		_, err := NewRedis(client.Addr, "").Hincrby("key", "field", 2)
+		assert.NotNil(t, err)
 		val, err := client.Hincrby("key", "field", 2)
 		assert.Nil(t, err)
 		assert.Equal(t, 2, val)
@@ -106,6 +122,8 @@ func TestRedis_Hkeys(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
+		_, err := NewRedis(client.Addr, "").Hkeys("a")
+		assert.NotNil(t, err)
 		vals, err := client.Hkeys("a")
 		assert.Nil(t, err)
 		assert.ElementsMatch(t, []string{"aa", "bb"}, vals)
@@ -116,6 +134,8 @@ func TestRedis_Hmget(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		assert.Nil(t, client.Hset("a", "aa", "aaa"))
 		assert.Nil(t, client.Hset("a", "bb", "bbb"))
+		_, err := NewRedis(client.Addr, "").Hmget("a", "aa", "bb")
+		assert.NotNil(t, err)
 		vals, err := client.Hmget("a", "aa", "bb")
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"aaa", "bbb"}, vals)
@@ -127,6 +147,7 @@ func TestRedis_Hmget(t *testing.T) {
 
 func TestRedis_Hmset(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		assert.NotNil(t, NewRedis(client.Addr, "").Hmset("a", nil))
 		assert.Nil(t, client.Hmset("a", map[string]string{
 			"aa": "aaa",
 			"bb": "bbb",
@@ -139,6 +160,8 @@ func TestRedis_Hmset(t *testing.T) {
 
 func TestRedis_Incr(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		_, err := NewRedis(client.Addr, "").Incr("a")
+		assert.NotNil(t, err)
 		val, err := client.Incr("a")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(1), val)
@@ -150,6 +173,8 @@ func TestRedis_Incr(t *testing.T) {
 
 func TestRedis_IncrBy(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		_, err := NewRedis(client.Addr, "").Incrby("a", 2)
+		assert.NotNil(t, err)
 		val, err := client.Incrby("a", 2)
 		assert.Nil(t, err)
 		assert.Equal(t, int64(2), val)
@@ -165,26 +190,49 @@ func TestRedis_Keys(t *testing.T) {
 		assert.Nil(t, err)
 		err = client.Set("key2", "value2")
 		assert.Nil(t, err)
+		_, err = NewRedis(client.Addr, "").Keys("*")
+		assert.NotNil(t, err)
 		keys, err := client.Keys("*")
 		assert.Nil(t, err)
 		assert.ElementsMatch(t, []string{"key1", "key2"}, keys)
 	})
 }
 
+func TestRedis_HyperLogLog(t *testing.T) {
+	runOnRedis(t, func(client *Redis) {
+		client.Ping()
+		r := NewRedis(client.Addr, "")
+		_, err := r.Pfadd("key1")
+		assert.NotNil(t, err)
+		_, err = r.Pfcount("*")
+		assert.NotNil(t, err)
+		err = r.Pfmerge("*")
+		assert.NotNil(t, err)
+	})
+}
+
 func TestRedis_List(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		_, err := NewRedis(client.Addr, "").Lpush("key", "value1", "value2")
+		assert.NotNil(t, err)
 		val, err := client.Lpush("key", "value1", "value2")
 		assert.Nil(t, err)
 		assert.Equal(t, 2, val)
+		_, err = NewRedis(client.Addr, "").Rpush("key", "value3", "value4")
+		assert.NotNil(t, err)
 		val, err = client.Rpush("key", "value3", "value4")
 		assert.Nil(t, err)
 		assert.Equal(t, 4, val)
+		_, err = NewRedis(client.Addr, "").Llen("key")
+		assert.NotNil(t, err)
 		val, err = client.Llen("key")
 		assert.Nil(t, err)
 		assert.Equal(t, 4, val)
 		vals, err := client.Lrange("key", 0, 10)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"value2", "value1", "value3", "value4"}, vals)
+		_, err = NewRedis(client.Addr, "").Lpop("key")
+		assert.NotNil(t, err)
 		v, err := client.Lpop("key")
 		assert.Nil(t, err)
 		assert.Equal(t, "value2", v)
@@ -194,9 +242,13 @@ func TestRedis_List(t *testing.T) {
 		val, err = client.Rpush("key", "value3", "value3")
 		assert.Nil(t, err)
 		assert.Equal(t, 7, val)
+		_, err = NewRedis(client.Addr, "").Lrem("key", 2, "value1")
+		assert.NotNil(t, err)
 		n, err := client.Lrem("key", 2, "value1")
 		assert.Nil(t, err)
 		assert.Equal(t, 2, n)
+		_, err = NewRedis(client.Addr, "").Lrange("key", 0, 10)
+		assert.NotNil(t, err)
 		vals, err = client.Lrange("key", 0, 10)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"value2", "value3", "value4", "value3", "value3"}, vals)
@@ -215,6 +267,8 @@ func TestRedis_Mget(t *testing.T) {
 		assert.Nil(t, err)
 		err = client.Set("key2", "value2")
 		assert.Nil(t, err)
+		_, err = NewRedis(client.Addr, "").Mget("key1", "key0", "key2", "key3")
+		assert.NotNil(t, err)
 		vals, err := client.Mget("key1", "key0", "key2", "key3")
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"value1", "", "value2", ""}, vals)
@@ -223,7 +277,9 @@ func TestRedis_Mget(t *testing.T) {
 
 func TestRedis_SetBit(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
-		err := client.SetBit("key", 1, 1)
+		err := NewRedis(client.Addr, "").SetBit("key", 1, 1)
+		assert.NotNil(t, err)
+		err = client.SetBit("key", 1, 1)
 		assert.Nil(t, err)
 	})
 }
@@ -232,6 +288,8 @@ func TestRedis_GetBit(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		err := client.SetBit("key", 2, 1)
 		assert.Nil(t, err)
+		_, err = NewRedis(client.Addr, "").GetBit("key", 2)
+		assert.NotNil(t, err)
 		val, err := client.GetBit("key", 2)
 		assert.Nil(t, err)
 		assert.Equal(t, 1, val)
@@ -240,6 +298,8 @@ func TestRedis_GetBit(t *testing.T) {
 
 func TestRedis_Persist(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		_, err := NewRedis(client.Addr, "").Persist("key")
+		assert.NotNil(t, err)
 		ok, err := client.Persist("key")
 		assert.Nil(t, err)
 		assert.False(t, ok)
@@ -248,11 +308,15 @@ func TestRedis_Persist(t *testing.T) {
 		ok, err = client.Persist("key")
 		assert.Nil(t, err)
 		assert.False(t, ok)
+		err = NewRedis(client.Addr, "").Expire("key", 5)
+		assert.NotNil(t, err)
 		err = client.Expire("key", 5)
 		assert.Nil(t, err)
 		ok, err = client.Persist("key")
 		assert.Nil(t, err)
 		assert.True(t, ok)
+		err = NewRedis(client.Addr, "").Expireat("key", time.Now().Unix()+5)
+		assert.NotNil(t, err)
 		err = client.Expireat("key", time.Now().Unix()+5)
 		assert.Nil(t, err)
 		ok, err = client.Persist("key")
@@ -274,6 +338,8 @@ func TestRedis_Scan(t *testing.T) {
 		assert.Nil(t, err)
 		err = client.Set("key2", "value2")
 		assert.Nil(t, err)
+		_, _, err = NewRedis(client.Addr, "").Scan(0, "*", 100)
+		assert.NotNil(t, err)
 		keys, _, err := client.Scan(0, "*", 100)
 		assert.Nil(t, err)
 		assert.ElementsMatch(t, []string{"key1", "key2"}, keys)
@@ -294,6 +360,8 @@ func TestRedis_Sscan(t *testing.T) {
 		var cursor uint64 = 0
 		sum := 0
 		for {
+			_, _, err := NewRedis(client.Addr, "").Sscan(key, cursor, "", 100)
+			assert.NotNil(t, err)
 			keys, next, err := client.Sscan(key, cursor, "", 100)
 			assert.Nil(t, err)
 			sum += len(keys)
@@ -304,6 +372,8 @@ func TestRedis_Sscan(t *testing.T) {
 		}
 
 		assert.Equal(t, sum, 1550)
+		_, err = NewRedis(client.Addr, "").Del(key)
+		assert.NotNil(t, err)
 		_, err = client.Del(key)
 		assert.Nil(t, err)
 	})
@@ -311,46 +381,72 @@ func TestRedis_Sscan(t *testing.T) {
 
 func TestRedis_Set(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		_, err := NewRedis(client.Addr, "").Sadd("key", 1, 2, 3, 4)
+		assert.NotNil(t, err)
 		num, err := client.Sadd("key", 1, 2, 3, 4)
 		assert.Nil(t, err)
 		assert.Equal(t, 4, num)
+		_, err = NewRedis(client.Addr, "").Scard("key")
+		assert.NotNil(t, err)
 		val, err := client.Scard("key")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(4), val)
+		_, err = NewRedis(client.Addr, "").Sismember("key", 2)
+		assert.NotNil(t, err)
 		ok, err := client.Sismember("key", 2)
 		assert.Nil(t, err)
 		assert.True(t, ok)
+		_, err = NewRedis(client.Addr, "").Srem("key", 3, 4)
+		assert.NotNil(t, err)
 		num, err = client.Srem("key", 3, 4)
 		assert.Nil(t, err)
 		assert.Equal(t, 2, num)
+		_, err = NewRedis(client.Addr, "").Smembers("key")
+		assert.NotNil(t, err)
 		vals, err := client.Smembers("key")
 		assert.Nil(t, err)
 		assert.ElementsMatch(t, []string{"1", "2"}, vals)
+		_, err = NewRedis(client.Addr, "").Srandmember("key", 1)
+		assert.NotNil(t, err)
 		members, err := client.Srandmember("key", 1)
 		assert.Nil(t, err)
 		assert.Len(t, members, 1)
 		assert.Contains(t, []string{"1", "2"}, members[0])
+		_, err = NewRedis(client.Addr, "").Spop("key")
+		assert.NotNil(t, err)
 		member, err := client.Spop("key")
 		assert.Nil(t, err)
 		assert.Contains(t, []string{"1", "2"}, member)
+		_, err = NewRedis(client.Addr, "").Smembers("key")
+		assert.NotNil(t, err)
 		vals, err = client.Smembers("key")
 		assert.Nil(t, err)
 		assert.NotContains(t, vals, member)
+		_, err = NewRedis(client.Addr, "").Sadd("key1", 1, 2, 3, 4)
+		assert.NotNil(t, err)
 		num, err = client.Sadd("key1", 1, 2, 3, 4)
 		assert.Nil(t, err)
 		assert.Equal(t, 4, num)
 		num, err = client.Sadd("key2", 2, 3, 4, 5)
 		assert.Nil(t, err)
 		assert.Equal(t, 4, num)
+		_, err = NewRedis(client.Addr, "").Sunion("key1", "key2")
+		assert.NotNil(t, err)
 		vals, err = client.Sunion("key1", "key2")
 		assert.Nil(t, err)
 		assert.ElementsMatch(t, []string{"1", "2", "3", "4", "5"}, vals)
+		_, err = NewRedis(client.Addr, "").Sunionstore("key3", "key1", "key2")
+		assert.NotNil(t, err)
 		num, err = client.Sunionstore("key3", "key1", "key2")
 		assert.Nil(t, err)
 		assert.Equal(t, 5, num)
+		_, err = NewRedis(client.Addr, "").Sdiff("key1", "key2")
+		assert.NotNil(t, err)
 		vals, err = client.Sdiff("key1", "key2")
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"1"}, vals)
+		_, err = NewRedis(client.Addr, "").Sdiffstore("key4", "key1", "key2")
+		assert.NotNil(t, err)
 		num, err = client.Sdiffstore("key4", "key1", "key2")
 		assert.Nil(t, err)
 		assert.Equal(t, 1, num)
@@ -359,8 +455,12 @@ func TestRedis_Set(t *testing.T) {
 
 func TestRedis_SetGetDel(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
-		err := client.Set("hello", "world")
+		err := NewRedis(client.Addr, "").Set("hello", "world")
+		assert.NotNil(t, err)
+		err = client.Set("hello", "world")
 		assert.Nil(t, err)
+		_, err = NewRedis(client.Addr, "").Get("hello")
+		assert.NotNil(t, err)
 		val, err := client.Get("hello")
 		assert.Nil(t, err)
 		assert.Equal(t, "world", val)
@@ -372,8 +472,12 @@ func TestRedis_SetGetDel(t *testing.T) {
 
 func TestRedis_SetExNx(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
-		err := client.Setex("hello", "world", 5)
+		err := NewRedis(client.Addr, "").Setex("hello", "world", 5)
+		assert.NotNil(t, err)
+		err = client.Setex("hello", "world", 5)
 		assert.Nil(t, err)
+		_, err = NewRedis(client.Addr, "").Setnx("hello", "newworld")
+		assert.NotNil(t, err)
 		ok, err := client.Setnx("hello", "newworld")
 		assert.Nil(t, err)
 		assert.False(t, ok)
@@ -389,6 +493,8 @@ func TestRedis_SetExNx(t *testing.T) {
 		ttl, err := client.Ttl("hello")
 		assert.Nil(t, err)
 		assert.True(t, ttl > 0)
+		_, err = NewRedis(client.Addr, "").SetnxEx("newhello", "newworld", 5)
+		assert.NotNil(t, err)
 		ok, err = client.SetnxEx("newhello", "newworld", 5)
 		assert.Nil(t, err)
 		assert.False(t, ok)
@@ -408,12 +514,18 @@ func TestRedis_SetGetDelHashField(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
 		err := client.Hset("key", "field", "value")
 		assert.Nil(t, err)
+		_, err = NewRedis(client.Addr, "").Hget("key", "field")
+		assert.NotNil(t, err)
 		val, err := client.Hget("key", "field")
 		assert.Nil(t, err)
 		assert.Equal(t, "value", val)
+		_, err = NewRedis(client.Addr, "").Hexists("key", "field")
+		assert.NotNil(t, err)
 		ok, err := client.Hexists("key", "field")
 		assert.Nil(t, err)
 		assert.True(t, ok)
+		_, err = NewRedis(client.Addr, "").Hdel("key", "field")
+		assert.NotNil(t, err)
 		ret, err := client.Hdel("key", "field")
 		assert.Nil(t, err)
 		assert.True(t, ret)
@@ -434,23 +546,50 @@ func TestRedis_SortedSet(t *testing.T) {
 		val, err := client.Zscore("key", "value1")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(2), val)
+		_, err = NewRedis(client.Addr, "").Zincrby("key", 3, "value1")
+		assert.NotNil(t, err)
 		val, err = client.Zincrby("key", 3, "value1")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(5), val)
+		_, err = NewRedis(client.Addr, "").Zscore("key", "value1")
+		assert.NotNil(t, err)
 		val, err = client.Zscore("key", "value1")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(5), val)
-		ok, err = client.Zadd("key", 6, "value2")
+		val, err = NewRedis(client.Addr, "").Zadds("key")
+		assert.NotNil(t, err)
+		val, err = client.Zadds("key", Pair{
+			Key:   "value2",
+			Score: 6,
+		}, Pair{
+			Key:   "value3",
+			Score: 7,
+		})
 		assert.Nil(t, err)
-		assert.True(t, ok)
-		ok, err = client.Zadd("key", 7, "value3")
+		assert.Equal(t, int64(2), val)
+		pairs, err := NewRedis(client.Addr, "").ZRevRangeWithScores("key", 1, 3)
+		assert.NotNil(t, err)
+		pairs, err = client.ZRevRangeWithScores("key", 1, 3)
 		assert.Nil(t, err)
-		assert.True(t, ok)
+		assert.EqualValues(t, []Pair{
+			{
+				Key:   "value2",
+				Score: 6,
+			},
+			{
+				Key:   "value1",
+				Score: 5,
+			},
+		}, pairs)
 		rank, err := client.Zrank("key", "value2")
 		assert.Nil(t, err)
 		assert.Equal(t, int64(1), rank)
+		_, err = NewRedis(client.Addr, "").Zrank("key", "value4")
+		assert.NotNil(t, err)
 		_, err = client.Zrank("key", "value4")
 		assert.Equal(t, Nil, err)
+		_, err = NewRedis(client.Addr, "").Zrem("key", "value2", "value3")
+		assert.NotNil(t, err)
 		num, err := client.Zrem("key", "value2", "value3")
 		assert.Nil(t, err)
 		assert.Equal(t, 2, num)
@@ -463,31 +602,47 @@ func TestRedis_SortedSet(t *testing.T) {
 		ok, err = client.Zadd("key", 8, "value4")
 		assert.Nil(t, err)
 		assert.True(t, ok)
+		_, err = NewRedis(client.Addr, "").Zremrangebyscore("key", 6, 7)
+		assert.NotNil(t, err)
 		num, err = client.Zremrangebyscore("key", 6, 7)
 		assert.Nil(t, err)
 		assert.Equal(t, 2, num)
 		ok, err = client.Zadd("key", 6, "value2")
 		assert.Nil(t, err)
 		assert.True(t, ok)
+		_, err = NewRedis(client.Addr, "").Zadd("key", 7, "value3")
+		assert.NotNil(t, err)
 		ok, err = client.Zadd("key", 7, "value3")
 		assert.Nil(t, err)
 		assert.True(t, ok)
+		_, err = NewRedis(client.Addr, "").Zcount("key", 6, 7)
+		assert.NotNil(t, err)
 		num, err = client.Zcount("key", 6, 7)
 		assert.Nil(t, err)
 		assert.Equal(t, 2, num)
+		_, err = NewRedis(client.Addr, "").Zremrangebyrank("key", 1, 2)
+		assert.NotNil(t, err)
 		num, err = client.Zremrangebyrank("key", 1, 2)
 		assert.Nil(t, err)
 		assert.Equal(t, 2, num)
+		_, err = NewRedis(client.Addr, "").Zcard("key")
+		assert.NotNil(t, err)
 		card, err := client.Zcard("key")
 		assert.Nil(t, err)
 		assert.Equal(t, 2, card)
+		_, err = NewRedis(client.Addr, "").Zrange("key", 0, -1)
+		assert.NotNil(t, err)
 		vals, err := client.Zrange("key", 0, -1)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"value1", "value4"}, vals)
+		_, err = NewRedis(client.Addr, "").Zrevrange("key", 0, -1)
+		assert.NotNil(t, err)
 		vals, err = client.Zrevrange("key", 0, -1)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []string{"value4", "value1"}, vals)
-		pairs, err := client.ZrangeWithScores("key", 0, -1)
+		_, err = NewRedis(client.Addr, "").ZrangeWithScores("key", 0, -1)
+		assert.NotNil(t, err)
+		pairs, err = client.ZrangeWithScores("key", 0, -1)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []Pair{
 			{
@@ -499,6 +654,8 @@ func TestRedis_SortedSet(t *testing.T) {
 				Score: 8,
 			},
 		}, pairs)
+		_, err = NewRedis(client.Addr, "").ZrangebyscoreWithScores("key", 5, 8)
+		assert.NotNil(t, err)
 		pairs, err = client.ZrangebyscoreWithScores("key", 5, 8)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []Pair{
@@ -511,6 +668,9 @@ func TestRedis_SortedSet(t *testing.T) {
 				Score: 8,
 			},
 		}, pairs)
+		_, err = NewRedis(client.Addr, "").ZrangebyscoreWithScoresAndLimit(
+			"key", 5, 8, 1, 1)
+		assert.NotNil(t, err)
 		pairs, err = client.ZrangebyscoreWithScoresAndLimit("key", 5, 8, 1, 1)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []Pair{
@@ -519,6 +679,11 @@ func TestRedis_SortedSet(t *testing.T) {
 				Score: 8,
 			},
 		}, pairs)
+		pairs, err = client.ZrangebyscoreWithScoresAndLimit("key", 5, 8, 1, 0)
+		assert.Nil(t, err)
+		assert.Equal(t, 0, len(pairs))
+		_, err = NewRedis(client.Addr, "").ZrevrangebyscoreWithScores("key", 5, 8)
+		assert.NotNil(t, err)
 		pairs, err = client.ZrevrangebyscoreWithScores("key", 5, 8)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []Pair{
@@ -531,6 +696,9 @@ func TestRedis_SortedSet(t *testing.T) {
 				Score: 5,
 			},
 		}, pairs)
+		_, err = NewRedis(client.Addr, "").ZrevrangebyscoreWithScoresAndLimit(
+			"key", 5, 8, 1, 1)
+		assert.NotNil(t, err)
 		pairs, err = client.ZrevrangebyscoreWithScoresAndLimit("key", 5, 8, 1, 1)
 		assert.Nil(t, err)
 		assert.EqualValues(t, []Pair{
@@ -539,11 +707,17 @@ func TestRedis_SortedSet(t *testing.T) {
 				Score: 5,
 			},
 		}, pairs)
+		pairs, err = client.ZrevrangebyscoreWithScoresAndLimit("key", 5, 8, 1, 0)
+		assert.Nil(t, err)
+		assert.Equal(t, 0, len(pairs))
 	})
 }
 
 func TestRedis_Pipelined(t *testing.T) {
 	runOnRedis(t, func(client *Redis) {
+		assert.NotNil(t, NewRedis(client.Addr, "").Pipelined(func(pipeliner Pipeliner) error {
+			return nil
+		}))
 		err := client.Pipelined(
 			func(pipe Pipeliner) error {
 				pipe.Incr("pipelined_counter")
@@ -553,6 +727,8 @@ func TestRedis_Pipelined(t *testing.T) {
 			},
 		)
 		assert.Nil(t, err)
+		_, err = NewRedis(client.Addr, "").Ttl("pipelined_counter")
+		assert.NotNil(t, err)
 		ttl, err := client.Ttl("pipelined_counter")
 		assert.Nil(t, err)
 		assert.Equal(t, 3600, ttl)
@@ -565,6 +741,76 @@ func TestRedis_Pipelined(t *testing.T) {
 	})
 }
 
+func TestRedisString(t *testing.T) {
+	runOnRedis(t, func(client *Redis) {
+		client.Ping()
+		_, err := getRedis(NewRedis(client.Addr, ClusterType))
+		assert.Nil(t, err)
+		assert.Equal(t, client.Addr, client.String())
+		assert.NotNil(t, NewRedis(client.Addr, "").Ping())
+	})
+}
+
+func TestRedisScriptLoad(t *testing.T) {
+	runOnRedis(t, func(client *Redis) {
+		client.Ping()
+		_, err := NewRedis(client.Addr, "").scriptLoad("foo")
+		assert.NotNil(t, err)
+		_, err = client.scriptLoad("foo")
+		assert.NotNil(t, err)
+	})
+}
+
+func TestRedisToPairs(t *testing.T) {
+	pairs := toPairs([]red.Z{
+		{
+			Member: 1,
+			Score:  1,
+		},
+		{
+			Member: 2,
+			Score:  2,
+		},
+	})
+	assert.EqualValues(t, []Pair{
+		{
+			Key:   "1",
+			Score: 1,
+		},
+		{
+			Key:   "2",
+			Score: 2,
+		},
+	}, pairs)
+}
+
+func TestRedisToStrings(t *testing.T) {
+	vals := toStrings([]interface{}{1, 2})
+	assert.EqualValues(t, []string{"1", "2"}, vals)
+}
+
+func TestRedisBlpop(t *testing.T) {
+	runOnRedis(t, func(client *Redis) {
+		client.Ping()
+		var node mockedNode
+		_, err := client.Blpop(nil, "foo")
+		assert.NotNil(t, err)
+		_, err = client.Blpop(node, "foo")
+		assert.NotNil(t, err)
+	})
+}
+
+func TestRedisBlpopEx(t *testing.T) {
+	runOnRedis(t, func(client *Redis) {
+		client.Ping()
+		var node mockedNode
+		_, _, err := client.BlpopEx(nil, "foo")
+		assert.NotNil(t, err)
+		_, _, err = client.BlpopEx(node, "foo")
+		assert.NotNil(t, err)
+	})
+}
+
 func runOnRedis(t *testing.T, fn func(client *Redis)) {
 	s, err := miniredis.Run()
 	assert.Nil(t, err)
@@ -576,8 +822,18 @@ func runOnRedis(t *testing.T, fn func(client *Redis)) {
 			t.Error(err)
 		}
 
-		client.Close()
+		if client != nil {
+			client.Close()
+		}
 	}()
 
 	fn(NewRedis(s.Addr(), NodeType))
 }
+
+type mockedNode struct {
+	RedisNode
+}
+
+func (n mockedNode) BLPop(timeout time.Duration, keys ...string) *red.StringSliceCmd {
+	return red.NewStringSliceCmd("foo", "bar")
+}

core/stores/sqlc/cachedsql.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/tal-tech/go-zero/core/stores/cache"
-	"github.com/tal-tech/go-zero/core/stores/internal"
 	"github.com/tal-tech/go-zero/core/stores/redis"
 	"github.com/tal-tech/go-zero/core/stores/sqlx"
 	"github.com/tal-tech/go-zero/core/syncx"
@@ -19,7 +18,7 @@ var (
 
 	// can't use one SharedCalls per conn, because multiple conns may share the same cache key.
 	exclusiveCalls = syncx.NewSharedCalls()
-	stats          = internal.NewCacheStat("sqlc")
+	stats          = cache.NewCacheStat("sqlc")
 )
 
 type (
@@ -30,21 +29,21 @@ type (
 
 	CachedConn struct {
 		db    sqlx.SqlConn
-		cache internal.Cache
+		cache cache.Cache
 	}
 )
 
 func NewNodeConn(db sqlx.SqlConn, rds *redis.Redis, opts ...cache.Option) CachedConn {
 	return CachedConn{
 		db:    db,
-		cache: internal.NewCacheNode(rds, exclusiveCalls, stats, sql.ErrNoRows, opts...),
+		cache: cache.NewCacheNode(rds, exclusiveCalls, stats, sql.ErrNoRows, opts...),
 	}
 }
 
 func NewConn(db sqlx.SqlConn, c cache.CacheConf, opts ...cache.Option) CachedConn {
 	return CachedConn{
 		db:    db,
-		cache: internal.NewCache(c, exclusiveCalls, stats, sql.ErrNoRows, opts...),
+		cache: cache.NewCache(c, exclusiveCalls, stats, sql.ErrNoRows, opts...),
 	}
 }
 
@@ -83,6 +82,7 @@ func (cc CachedConn) QueryRowIndex(v interface{}, key string, keyer func(primary
 	indexQuery IndexQueryFn, primaryQuery PrimaryQueryFn) error {
 	var primaryKey interface{}
 	var found bool
+
 	if err := cc.cache.TakeWithExpire(&primaryKey, key, func(val interface{}, expire time.Duration) (err error) {
 		primaryKey, err = indexQuery(cc.db, v)
 		if err != nil {

core/stores/sqlc/cachedsql_test.go

@@ -79,9 +79,29 @@ func TestCachedConn_QueryRowIndex_NoCache(t *testing.T) {
 	}
 
 	r := redis.NewRedis(s.Addr(), redis.NodeType)
-	c := NewNodeConn(dummySqlConn{}, r, cache.WithExpiry(time.Second*10))
+	c := NewConn(dummySqlConn{}, cache.CacheConf{
+		{
+			RedisConf: redis.RedisConf{
+				Host: s.Addr(),
+				Type: redis.NodeType,
+			},
+			Weight: 100,
+		},
+	}, cache.WithExpiry(time.Second*10))
 
 	var str string
+	err = c.QueryRowIndex(&str, "index", func(s interface{}) string {
+		return fmt.Sprintf("%s/1234", s)
+	}, func(conn sqlx.SqlConn, v interface{}) (interface{}, error) {
+		*v.(*string) = "zero"
+		return "primary", errors.New("foo")
+	}, func(conn sqlx.SqlConn, v, pri interface{}) error {
+		assert.Equal(t, "primary", pri)
+		*v.(*string) = "xin"
+		return nil
+	})
+	assert.NotNil(t, err)
+
 	err = c.QueryRowIndex(&str, "index", func(s interface{}) string {
 		return fmt.Sprintf("%s/1234", s)
 	}, func(conn sqlx.SqlConn, v interface{}) (interface{}, error) {
@@ -135,6 +155,103 @@ func TestCachedConn_QueryRowIndex_HasCache(t *testing.T) {
 	assert.Equal(t, `"xin"`, val)
 }
 
+func TestCachedConn_QueryRowIndex_HasCache_IntPrimary(t *testing.T) {
+	const (
+		primaryInt8   int8   = 100
+		primaryInt16  int16  = 10000
+		primaryInt32  int32  = 10000000
+		primaryInt64  int64  = 10000000
+		primaryUint8  uint8  = 100
+		primaryUint16 uint16 = 10000
+		primaryUint32 uint32 = 10000000
+		primaryUint64 uint64 = 10000000
+	)
+	tests := []struct {
+		name         string
+		primary      interface{}
+		primaryCache string
+	}{
+		{
+			name:         "int8 primary",
+			primary:      primaryInt8,
+			primaryCache: fmt.Sprint(primaryInt8),
+		},
+		{
+			name:         "int16 primary",
+			primary:      primaryInt16,
+			primaryCache: fmt.Sprint(primaryInt16),
+		},
+		{
+			name:         "int32 primary",
+			primary:      primaryInt32,
+			primaryCache: fmt.Sprint(primaryInt32),
+		},
+		{
+			name:         "int64 primary",
+			primary:      primaryInt64,
+			primaryCache: fmt.Sprint(primaryInt64),
+		},
+		{
+			name:         "uint8 primary",
+			primary:      primaryUint8,
+			primaryCache: fmt.Sprint(primaryUint8),
+		},
+		{
+			name:         "uint16 primary",
+			primary:      primaryUint16,
+			primaryCache: fmt.Sprint(primaryUint16),
+		},
+		{
+			name:         "uint32 primary",
+			primary:      primaryUint32,
+			primaryCache: fmt.Sprint(primaryUint32),
+		},
+		{
+			name:         "uint64 primary",
+			primary:      primaryUint64,
+			primaryCache: fmt.Sprint(primaryUint64),
+		},
+	}
+
+	s, err := miniredis.Run()
+	if err != nil {
+		t.Error(err)
+	}
+	defer s.Close()
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			resetStats()
+			s.FlushAll()
+
+			r := redis.NewRedis(s.Addr(), redis.NodeType)
+			c := NewNodeConn(dummySqlConn{}, r, cache.WithExpiry(time.Second*10),
+				cache.WithNotFoundExpiry(time.Second))
+
+			var str string
+			r.Set("index", test.primaryCache)
+			err = c.QueryRowIndex(&str, "index", func(s interface{}) string {
+				return fmt.Sprintf("%v/1234", s)
+			}, func(conn sqlx.SqlConn, v interface{}) (interface{}, error) {
+				assert.Fail(t, "should not go here")
+				return test.primary, nil
+			}, func(conn sqlx.SqlConn, v, primary interface{}) error {
+				*v.(*string) = "xin"
+				assert.Equal(t, primary, primary)
+				return nil
+			})
+			assert.Nil(t, err)
+			assert.Equal(t, "xin", str)
+			val, err := r.Get("index")
+			assert.Nil(t, err)
+			assert.Equal(t, test.primaryCache, val)
+			val, err = r.Get(test.primaryCache + "/1234")
+			assert.Nil(t, err)
+			assert.Equal(t, `"xin"`, val)
+		})
+	}
+}
+
 func TestCachedConn_QueryRowIndex_HasWrongCache(t *testing.T) {
 	caches := map[string]string{
 		"index":        "primary",
@@ -148,6 +265,8 @@ func TestCachedConn_QueryRowIndex_HasWrongCache(t *testing.T) {
 			if err != nil {
 				t.Error(err)
 			}
+			s.FlushAll()
+			defer s.Close()
 
 			r := redis.NewRedis(s.Addr(), redis.NodeType)
 			c := NewNodeConn(dummySqlConn{}, r, cache.WithExpiry(time.Second*10),
@@ -401,6 +520,10 @@ func TestCachedConnExecDropCache(t *testing.T) {
 	assert.True(t, conn.execValue)
 	_, err = s.Get(key)
 	assert.Exactly(t, miniredis.ErrKeyNotFound, err)
+	_, err = c.Exec(func(conn sqlx.SqlConn) (result sql.Result, e error) {
+		return nil, errors.New("foo")
+	}, key)
+	assert.NotNil(t, err)
 }
 
 func TestCachedConnExecDropCacheFailed(t *testing.T) {
@@ -446,6 +569,31 @@ func TestCachedConnTransact(t *testing.T) {
 	assert.True(t, conn.transactValue)
 }
 
+func TestQueryRowNoCache(t *testing.T) {
+	s, err := miniredis.Run()
+	if err != nil {
+		t.Error(err)
+	}
+
+	const (
+		key   = "user"
+		value = "any"
+	)
+	var user string
+	var ran bool
+	r := redis.NewRedis(s.Addr(), redis.NodeType)
+	conn := dummySqlConn{queryRow: func(v interface{}, q string, args ...interface{}) error {
+		user = value
+		ran = true
+		return nil
+	}}
+	c := NewNodeConn(&conn, r, cache.WithExpiry(time.Second*30))
+	err = c.QueryRowNoCache(&user, key)
+	assert.Nil(t, err)
+	assert.Equal(t, value, user)
+	assert.True(t, ran)
+}
+
 func resetStats() {
 	atomic.StoreUint64(&stats.Total, 0)
 	atomic.StoreUint64(&stats.Hit, 0)
@@ -454,6 +602,7 @@ func resetStats() {
 }
 
 type dummySqlConn struct {
+	queryRow func(interface{}, string, ...interface{}) error
 }
 
 func (d dummySqlConn) Exec(query string, args ...interface{}) (sql.Result, error) {
@@ -465,6 +614,9 @@ func (d dummySqlConn) Prepare(query string) (sqlx.StmtSession, error) {
 }
 
 func (d dummySqlConn) QueryRow(v interface{}, query string, args ...interface{}) error {
+	if d.queryRow != nil {
+		return d.queryRow(v, query, args...)
+	}
 	return nil
 }
 

core/stores/sqlx/bulkinserter_test.go

@@ -2,6 +2,7 @@ package sqlx
 
 import (
 	"database/sql"
+	"errors"
 	"strconv"
 	"testing"
 
@@ -11,14 +12,15 @@ import (
 )
 
 type mockedConn struct {
-	query string
-	args  []interface{}
+	query   string
+	args    []interface{}
+	execErr error
 }
 
 func (c *mockedConn) Exec(query string, args ...interface{}) (sql.Result, error) {
 	c.query = query
 	c.args = args
-	return nil, nil
+	return nil, c.execErr
 }
 
 func (c *mockedConn) Prepare(query string) (StmtSession, error) {
@@ -68,9 +70,12 @@ func TestBulkInserterSuffix(t *testing.T) {
 		inserter, err := NewBulkInserter(&conn, `INSERT INTO classroom_dau(classroom, user, count) VALUES`+
 			`(?, ?, ?) ON DUPLICATE KEY UPDATE is_overtime=VALUES(is_overtime)`)
 		assert.Nil(t, err)
+		assert.Nil(t, inserter.UpdateStmt(`INSERT INTO classroom_dau(classroom, user, count) VALUES`+
+			`(?, ?, ?) ON DUPLICATE KEY UPDATE is_overtime=VALUES(is_overtime)`))
 		for i := 0; i < 5; i++ {
 			assert.Nil(t, inserter.Insert("class_"+strconv.Itoa(i), "user_"+strconv.Itoa(i), i))
 		}
+		inserter.SetResultHandler(func(result sql.Result, err error) {})
 		inserter.Flush()
 		assert.Equal(t, `INSERT INTO classroom_dau(classroom, user, count) VALUES `+
 			`('class_0', 'user_0', 0), ('class_1', 'user_1', 1), ('class_2', 'user_2', 2), `+
@@ -80,6 +85,33 @@ func TestBulkInserterSuffix(t *testing.T) {
 	})
 }
 
+func TestBulkInserterBadStatement(t *testing.T) {
+	runSqlTest(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
+		var conn mockedConn
+		_, err := NewBulkInserter(&conn, "foo")
+		assert.NotNil(t, err)
+	})
+}
+
+func TestBulkInserter_Update(t *testing.T) {
+	conn := mockedConn{
+		execErr: errors.New("foo"),
+	}
+	_, err := NewBulkInserter(&conn, `INSERT INTO classroom_dau(classroom, user, count) VALUES()`)
+	assert.NotNil(t, err)
+	_, err = NewBulkInserter(&conn, `INSERT INTO classroom_dau(classroom, user, count) VALUES(?)`)
+	assert.NotNil(t, err)
+	inserter, err := NewBulkInserter(&conn, `INSERT INTO classroom_dau(classroom, user, count) VALUES(?, ?, ?)`)
+	assert.Nil(t, err)
+	inserter.inserter.Execute([]string{"bar"})
+	inserter.SetResultHandler(func(result sql.Result, err error) {
+	})
+	inserter.UpdateOrDelete(func() {})
+	inserter.inserter.Execute([]string(nil))
+	assert.NotNil(t, inserter.UpdateStmt("foo"))
+	assert.NotNil(t, inserter.Insert("foo", "bar"))
+}
+
 func runSqlTest(t *testing.T, fn func(db *sql.DB, mock sqlmock.Sqlmock)) {
 	logx.Disable()
 

core/stores/sqlx/orm_test.go

@@ -2,6 +2,7 @@ package sqlx
 
 import (
 	"database/sql"
+	"errors"
 	"testing"
 
 	"github.com/DATA-DOG/go-sqlmock"
@@ -22,6 +23,18 @@ func TestUnmarshalRowBool(t *testing.T) {
 	})
 }
 
+func TestUnmarshalRowBoolNotSettable(t *testing.T) {
+	runOrmTest(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
+		rs := sqlmock.NewRows([]string{"value"}).FromCSVString("1")
+		mock.ExpectQuery("select (.+) from users where user=?").WithArgs("anyone").WillReturnRows(rs)
+
+		var value bool
+		assert.NotNil(t, query(db, func(rows *sql.Rows) error {
+			return unmarshalRow(value, rows, true)
+		}, "select value from users where user=?", "anyone"))
+	})
+}
+
 func TestUnmarshalRowInt(t *testing.T) {
 	runOrmTest(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
 		rs := sqlmock.NewRows([]string{"value"}).FromCSVString("2")
@@ -228,6 +241,22 @@ func TestUnmarshalRowStructWithTags(t *testing.T) {
 	})
 }
 
+func TestUnmarshalRowStructWithTagsWrongColumns(t *testing.T) {
+	var value = new(struct {
+		Age  *int   `db:"age"`
+		Name string `db:"name"`
+	})
+
+	runOrmTest(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
+		rs := sqlmock.NewRows([]string{"name"}).FromCSVString("liao")
+		mock.ExpectQuery("select (.+) from users where user=?").WithArgs("anyone").WillReturnRows(rs)
+
+		assert.NotNil(t, query(db, func(rows *sql.Rows) error {
+			return unmarshalRow(value, rows, true)
+		}, "select name, age from users where user=?", "anyone"))
+	})
+}
+
 func TestUnmarshalRowsBool(t *testing.T) {
 	runOrmTest(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
 		var expect = []bool{true, false}
@@ -955,6 +984,62 @@ func TestCommonSqlConn_QueryRowOptional(t *testing.T) {
 	})
 }
 
+func TestUnmarshalRowError(t *testing.T) {
+	tests := []struct {
+		name     string
+		colErr   error
+		scanErr  error
+		err      error
+		next     int
+		validate func(err error)
+	}{
+		{
+			name: "with error",
+			err:  errors.New("foo"),
+			validate: func(err error) {
+				assert.NotNil(t, err)
+			},
+		},
+		{
+			name: "without next",
+			validate: func(err error) {
+				assert.Equal(t, ErrNotFound, err)
+			},
+		},
+		{
+			name:    "with error",
+			scanErr: errors.New("foo"),
+			next:    1,
+			validate: func(err error) {
+				assert.Equal(t, ErrNotFound, err)
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			runOrmTest(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
+				rs := sqlmock.NewRows([]string{"age"}).FromCSVString("5")
+				mock.ExpectQuery("select (.+) from users where user=?").WithArgs(
+					"anyone").WillReturnRows(rs)
+
+				var r struct {
+					User string `db:"user"`
+					Age  int    `db:"age"`
+				}
+				test.validate(query(db, func(rows *sql.Rows) error {
+					scanner := mockedScanner{
+						colErr:  test.colErr,
+						scanErr: test.scanErr,
+						err:     test.err,
+					}
+					return unmarshalRow(&r, &scanner, false)
+				}, "select age from users where user=?", "anyone"))
+			})
+		})
+	}
+}
+
 func runOrmTest(t *testing.T, fn func(db *sql.DB, mock sqlmock.Sqlmock)) {
 	logx.Disable()
 
@@ -970,3 +1055,30 @@ func runOrmTest(t *testing.T, fn func(db *sql.DB, mock sqlmock.Sqlmock)) {
 		t.Errorf("there were unfulfilled expectations: %s", err)
 	}
 }
+
+type mockedScanner struct {
+	colErr  error
+	scanErr error
+	err     error
+	next    int
+}
+
+func (m *mockedScanner) Columns() ([]string, error) {
+	return nil, m.colErr
+}
+
+func (m *mockedScanner) Err() error {
+	return m.err
+}
+
+func (m *mockedScanner) Next() bool {
+	if m.next > 0 {
+		m.next--
+		return true
+	}
+	return false
+}
+
+func (m *mockedScanner) Scan(v ...interface{}) error {
+	return m.scanErr
+}

core/syncx/limit.go

@@ -6,18 +6,22 @@ import (
 	"github.com/tal-tech/go-zero/core/lang"
 )
 
-var ErrReturn = errors.New("discarding limited token, resource pool is full, someone returned multiple times")
+// ErrLimitReturn indicates that the more than borrowed elements were returned.
+var ErrLimitReturn = errors.New("discarding limited token, resource pool is full, someone returned multiple times")
 
+// Limit controls the concurrent requests.
 type Limit struct {
 	pool chan lang.PlaceholderType
 }
 
+// NewLimit creates a Limit that can borrow n elements from it concurrently.
 func NewLimit(n int) Limit {
 	return Limit{
 		pool: make(chan lang.PlaceholderType, n),
 	}
 }
 
+// Borrow borrows an element from Limit in blocking mode.
 func (l Limit) Borrow() {
 	l.pool <- lang.Placeholder
 }
@@ -28,10 +32,12 @@ func (l Limit) Return() error {
 	case <-l.pool:
 		return nil
 	default:
-		return ErrReturn
+		return ErrLimitReturn
 	}
 }
 
+// TryBorrow tries to borrow an element from Limit, in non-blocking mode.
+// If success, true returned, false for otherwise.
 func (l Limit) TryBorrow() bool {
 	select {
 	case l.pool <- lang.Placeholder:

core/syncx/limit_test.go

@@ -13,5 +13,5 @@ func TestLimit(t *testing.T) {
 	assert.False(t, limit.TryBorrow())
 	assert.Nil(t, limit.Return())
 	assert.Nil(t, limit.Return())
-	assert.Equal(t, ErrReturn, limit.Return())
+	assert.Equal(t, ErrLimitReturn, limit.Return())
 }

core/syncx/sharedcalls.go

@@ -33,35 +33,42 @@ func NewSharedCalls() SharedCalls {
 }
 
 func (g *sharedGroup) Do(key string, fn func() (interface{}, error)) (interface{}, error) {
-	g.lock.Lock()
-	if c, ok := g.calls[key]; ok {
-		g.lock.Unlock()
-		c.wg.Wait()
+	c, done := g.createCall(key, fn)
+	if done {
 		return c.val, c.err
 	}
 
-	c := g.makeCall(key, fn)
+	g.makeCall(c, key, fn)
 	return c.val, c.err
 }
 
 func (g *sharedGroup) DoEx(key string, fn func() (interface{}, error)) (val interface{}, fresh bool, err error) {
+	c, done := g.createCall(key, fn)
+	if done {
+		return c.val, false, c.err
+	}
+
+	g.makeCall(c, key, fn)
+	return c.val, true, c.err
+}
+
+func (g *sharedGroup) createCall(key string, fn func() (interface{}, error)) (c *call, done bool) {
 	g.lock.Lock()
 	if c, ok := g.calls[key]; ok {
 		g.lock.Unlock()
 		c.wg.Wait()
-		return c.val, false, c.err
+		return c, true
 	}
 
-	c := g.makeCall(key, fn)
-	return c.val, true, c.err
-}
-
-func (g *sharedGroup) makeCall(key string, fn func() (interface{}, error)) *call {
-	c := new(call)
+	c = new(call)
 	c.wg.Add(1)
 	g.calls[key] = c
 	g.lock.Unlock()
 
+	return c, false
+}
+
+func (g *sharedGroup) makeCall(c *call, key string, fn func() (interface{}, error)) {
 	defer func() {
 		// delete key first, done later. can't reverse the order, because if reverse,
 		// another Do call might wg.Wait() without get notified with wg.Done()
@@ -72,5 +79,4 @@ func (g *sharedGroup) makeCall(key string, fn func() (interface{}, error)) *call
 	}()
 
 	c.val, c.err = fn()
-	return c
 }

core/syncx/sharedcalls_test.go

@@ -68,6 +68,38 @@ func TestExclusiveCallDoDupSuppress(t *testing.T) {
 	}
 }
 
+func TestExclusiveCallDoDiffDupSuppress(t *testing.T) {
+	g := NewSharedCalls()
+	broadcast := make(chan struct{})
+	var calls int32
+	tests := []string{"e", "a", "e", "a", "b", "c", "b", "a", "c", "d", "b", "c", "d"}
+
+	var wg sync.WaitGroup
+	for _, key := range tests {
+		wg.Add(1)
+		go func(k string) {
+			<-broadcast // get all goroutines ready
+			_, err := g.Do(k, func() (interface{}, error) {
+				atomic.AddInt32(&calls, 1)
+				time.Sleep(10 * time.Millisecond)
+				return nil, nil
+			})
+			if err != nil {
+				t.Errorf("Do error: %v", err)
+			}
+			wg.Done()
+		}(key)
+	}
+
+	time.Sleep(100 * time.Millisecond) // let goroutines above block
+	close(broadcast)
+	wg.Wait()
+
+	if got := atomic.LoadInt32(&calls); got != 5 {	// five letters
+		t.Errorf("number of calls = %d; want 5", got)
+	}
+}
+
 func TestExclusiveCallDoExDupSuppress(t *testing.T) {
 	g := NewSharedCalls()
 	c := make(chan string)

core/syncx/timeoutlimit_test.go

@@ -29,5 +29,5 @@ func TestTimeoutLimit(t *testing.T) {
 	assert.Equal(t, ErrTimeout, limit.Borrow(time.Millisecond*100))
 	assert.Nil(t, limit.Return())
 	assert.Nil(t, limit.Return())
-	assert.Equal(t, ErrReturn, limit.Return())
+	assert.Equal(t, ErrLimitReturn, limit.Return())
 }

core/trace/tracespec/keys.go

@@ -0,0 +1,12 @@
+package tracespec
+
+// TracingKey is tracing key for context
+var TracingKey = contextKey("X-Trace")
+
+// contextKey a type for context key
+type contextKey string
+
+// Printing a context will reveal a fair amount of information about it.
+func (c contextKey) String() string {
+	return "trace/tracespec context key " + string(c)
+}

core/trace/tracespec/vars.go

@@ -1,3 +0,0 @@
-package tracespec
-
-const TracingKey = "X-Trace"

doc/bookstore-en.md

@@ -1,8 +1,8 @@
-English | [简体中文](bookstore.md)
-
 # Rapid development of microservices - multiple RPCs
 
-## 0. Why building microservices are so difficult?
+English | [简体中文](bookstore.md)
+
+## 0. Why building microservices are so difficult
 
 To build a well working microservice, we need lots of knowledges from different aspects.
 
@@ -25,7 +25,7 @@ As well, we always adhere to the idea that **prefer tools over conventions and d
 
 Let’s take the shorturl microservice as a quick example to demonstrate how to quickly create microservices by using [go-zero](https://github.com/tal-tech/go-zero). After finishing this tutorial, you’ll find that it’s so easy to write microservices!
 
-## 1. What is a bookstore service?
+## 1. What is a bookstore service
 
 For simplicity, the bookstore service only contains two functionalities, adding books and quering prices.
 
@@ -57,13 +57,19 @@ And now, let’s walk through the complete flow of quickly create a microservice
 
 * install etcd, mysql, redis
 
+* install protoc-gen-go
+
+  ```shell
+  go get -u github.com/golang/protobuf/protoc-gen-go
+  ```
+
 * install goctl
 
   ```shell
   GO111MODULE=on go get -u github.com/tal-tech/go-zero/tools/goctl
   ```
 
-* create the working dir bookstore
+* create the working dir `bookstore` and `bookstore/api`
 
 * in `bookstore` dir, execute `go mod init bookstore` to initialize `go.mod``
 
@@ -71,7 +77,7 @@ And now, let’s walk through the complete flow of quickly create a microservice
 
 * use goctl to generate `api/bookstore.api`
 
-  ```
+  ```Plain Text
   goctl api -o bookstore.api
   ```
 
@@ -128,7 +134,7 @@ And now, let’s walk through the complete flow of quickly create a microservice
 
   the generated file structure looks like:
 
-  ```
+  ```Plain Text
   api
   ├── bookstore.api                  // api definition
   ├── bookstore.go                   // main entrance
@@ -185,16 +191,18 @@ And now, let’s walk through the complete flow of quickly create a microservice
 
 ## 6. Write code for add rpc service
 
+- under directory `bookstore` create dir `rpc`
+
 * under directory `rpc/add` create `add.proto` file
 
   ```shell
-goctl rpc template -o add.proto
+  goctl rpc template -o add.proto
   ```
   
   edit the file and make the code looks like:
 
   ```protobuf
-syntax = "proto3";
+  syntax = "proto3";
   
   package add;
   
@@ -220,7 +228,7 @@ syntax = "proto3";
 
   the generated file structure looks like:
 
-  ```
+  ```Plain Text
   rpc/add
   ├── add.go                      // rpc main entrance
   ├── add.proto                   // rpc definition
@@ -243,7 +251,6 @@ syntax = "proto3";
       └── add.pb.go
   ```
 
-
 just run it, looks like:
 
 ```shell
@@ -258,13 +265,13 @@ you can change the listening port in file `etc/add.yaml`.
 * under directory `rpc/check` create `check.proto` file
 
   ```shell
-goctl rpc template -o check.proto
+  goctl rpc template -o check.proto
   ```
   
   edit the file and make the code looks like:
 
   ```protobuf
-syntax = "proto3";
+  syntax = "proto3";
   
   package check;
   
@@ -290,7 +297,7 @@ syntax = "proto3";
 
   the generated file structure looks like:
 
-  ```
+  ```Plain Text
   rpc/check
   ├── check.go                    // rpc main entrance
   ├── check.proto                 // rpc definition
@@ -315,7 +322,7 @@ syntax = "proto3";
 
   you can change the listening port in `etc/check.yaml`.
 
-  we need to change the port in `etc/check.yaml` to `8081`, because `8080 ` is used by `add` service.
+  we need to change the port in `etc/check.yaml` to `8081`, because `8080` is used by `add` service.
 
   just run it, looks like:
   
@@ -348,8 +355,8 @@ syntax = "proto3";
   ```go
   type Config struct {
       rest.RestConf
-      Add   rpcx.RpcClientConf     // manual code
-      Check rpcx.RpcClientConf     // manual code
+      Add   zrpc.RpcClientConf     // manual code
+      Check zrpc.RpcClientConf     // manual code
   }
   ```
 
@@ -365,8 +372,8 @@ syntax = "proto3";
   func NewServiceContext(c config.Config) *ServiceContext {
       return &ServiceContext{
           Config:  c,
-          Adder:   adder.NewAdder(rpcx.MustNewClient(c.Add)),         // manual code
-          Checker: checker.NewChecker(rpcx.MustNewClient(c.Check)),   // manual code
+          Adder:   adder.NewAdder(zrpc.MustNewClient(c.Add)),         // manual code
+          Checker: checker.NewChecker(zrpc.MustNewClient(c.Check)),   // manual code
       }
   }
   ```
@@ -444,7 +451,7 @@ Till now, we’ve done the modification of API Gateway. All the manually added c
   source book.sql;
   ```
 
-* under the directory `rpc/model execute the following command to genrate CRUD+cache code, `-c` means using `redis cache`
+* under the directory `rpc/model` execute the following command to genrate CRUD+cache code, `-c` means using `redis cache`
 
   ```shell
   goctl model mysql ddl -c -src book.sql -dir .
@@ -454,7 +461,7 @@ Till now, we’ve done the modification of API Gateway. All the manually added c
 
   the generated file structure looks like:
 
-  ```
+  ```Plain Text
   rpc/model
   ├── bookstore.sql
   ├── bookstoremodel.go     // CRUD+cache code
@@ -478,7 +485,7 @@ Till now, we’ve done the modification of API Gateway. All the manually added c
 
   ```go
   type Config struct {
-      rpcx.RpcServerConf
+      zrpc.RpcServerConf
       DataSource string             // manual code
       Table      string             // manual code
       Cache      cache.CacheConf    // manual code

doc/bookstore.md

@@ -1,8 +1,8 @@
-[English](bookstore-en.md) | 简体中文
-
 # 快速构建微服务-多RPC版
 
-## 0. 为什么说做好微服务很难？
+[English](bookstore-en.md) | 简体中文
+
+## 0. 为什么说做好微服务很难
 
 要想做好微服务，我们需要理解和掌握的知识点非常多，从几个维度上来说：
 
@@ -57,13 +57,19 @@
 
 * 安装etcd, mysql, redis
 
+* 安装`protoc-gen-go`
+
+  ```shell
+  go get -u github.com/golang/protobuf/protoc-gen-go
+  ```
+
 * 安装goctl工具
 
   ```shell
   GO111MODULE=on GOPROXY=https://goproxy.cn/,direct go get -u github.com/tal-tech/go-zero/tools/goctl
   ```
 
-* 创建工作目录`bookstore`
+* 创建工作目录 `bookstore` 和 `bookstore/api`
 
 * 在`bookstore`目录下执行`go mod init bookstore`初始化`go.mod`
 
@@ -71,7 +77,7 @@
 
 * 在`bookstore/api`目录下通过goctl生成`api/bookstore.api`：
 
-  ```
+  ```bash
   goctl api -o bookstore.api
   ```
 
@@ -128,7 +134,7 @@
 
   生成的文件结构如下：
 
-  ```
+  ```Plain Text
   api
   ├── bookstore.api                  // api定义
   ├── bookstore.go                   // main入口定义
@@ -185,6 +191,8 @@
 
 ## 6. 编写add rpc服务
 
+- 在 `bookstore` 下创建 `rpc` 目录
+
 * 在`rpc/add`目录下编写`add.proto`文件
 
   可以通过命令生成proto文件模板
@@ -222,7 +230,7 @@
 
   文件结构如下：
 
-  ```
+  ```Plain Text
   rpc/add
   ├── add.go                      // rpc服务main函数
   ├── add.proto                   // rpc接口定义
@@ -245,7 +253,6 @@
       └── add.pb.go
   ```
 
-
 直接可以运行，如下：
 
 ```shell
@@ -294,7 +301,7 @@
 
   文件结构如下：
 
-  ```
+  ```Plain Text
   rpc/check
   ├── check.go                    // rpc服务main函数
   ├── check.proto                 // rpc接口定义
@@ -350,8 +357,8 @@
   ```go
   type Config struct {
       rest.RestConf
-      Add   rpcx.RpcClientConf     // 手动代码
-      Check rpcx.RpcClientConf     // 手动代码
+      Add   zrpc.RpcClientConf     // 手动代码
+      Check zrpc.RpcClientConf     // 手动代码
   }
   ```
 
@@ -367,8 +374,8 @@
   func NewServiceContext(c config.Config) *ServiceContext {
       return &ServiceContext{
           Config:  c,
-          Adder:   adder.NewAdder(rpcx.MustNewClient(c.Add)),         // 手动代码
-          Checker: checker.NewChecker(rpcx.MustNewClient(c.Check)),   // 手动代码
+          Adder:   adder.NewAdder(zrpc.MustNewClient(c.Add)),         // 手动代码
+          Checker: checker.NewChecker(zrpc.MustNewClient(c.Check)),   // 手动代码
       }
   }
   ```
@@ -454,7 +461,7 @@
 
   生成后的文件结构如下：
 
-  ```
+  ```Plain Text
   rpc/model
   ├── bookstore.sql
   ├── bookstoremodel.go     // CRUD+cache代码
@@ -478,7 +485,7 @@
 
   ```go
   type Config struct {
-      rpcx.RpcServerConf
+      zrpc.RpcServerConf
       DataSource string             // 手动代码
       Table      string             // 手动代码
       Cache      cache.CacheConf    // 手动代码
@@ -541,7 +548,7 @@
   }
   ```
   
-  至此代码修改完成，凡事手动修改的代码我加了标注
+  至此代码修改完成，凡是手动修改的代码我加了标注
 
 ## 11. 完整调用演示
 
@@ -615,4 +622,3 @@ go-zero不只是一个框架，更是一个建立在框架+工具基础上的，
 通过go-zero+goctl生成的代码，包含了微服务治理的各种组件，包括：并发控制、自适应熔断、自适应降载、自动缓存控制等，可以轻松部署以承载巨大访问量。
 
 有任何好的提升工程效率的想法，随时欢迎交流！👏
-

doc/breaker.md

@@ -1,4 +1,5 @@
 # 熔断机制设计
 
 ## 设计目的
+
 * 依赖的服务出现大规模故障时，调用方应该尽可能少调用，降低故障服务的压力，使之尽快恢复服务

doc/collection.md

@@ -0,0 +1,111 @@
+# 通过 collection.Cache 进行缓存
+
+go-zero微服务框架中提供了许多开箱即用的工具，好的工具不仅能提升服务的性能而且还能提升代码的鲁棒性避免出错，实现代码风格的统一方便他人阅读等等，本系列文章将分别介绍go-zero框架中工具的使用及其实现原理  
+
+## 进程内缓存工具[collection.Cache](https://github.com/tal-tech/go-zero/tree/master/core/collection/cache.go)
+
+在做服务器开发的时候，相信都会遇到使用缓存的情况，go-zero 提供的简单的缓存封装 **collection.Cache**，简单使用方式如下
+
+```go
+// 初始化 cache，其中 WithLimit 可以指定最大缓存的数量
+c, err := collection.NewCache(time.Minute, collection.WithLimit(10000))
+if err != nil {
+  panic(err)
+}
+
+// 设置缓存
+c.Set("key", user)
+
+// 获取缓存，ok：是否存在
+v, ok := c.Get("key")
+
+// 删除缓存
+c.Del("key")
+
+// 获取缓存，如果 key 不存在的，则会调用 func 去生成缓存
+v, err := c.Take("key", func() (interface{}, error) {
+  return user, nil
+})
+```
+
+cache 实现的建的功能包括
+
+* 缓存自动失效，可以指定过期时间
+* 缓存大小限制，可以指定缓存个数
+* 缓存增删改
+* 缓存命中率统计
+* 并发安全
+* 缓存击穿
+
+实现原理：
+Cache 自动失效，是采用 TimingWheel(https://github.com/tal-tech/go-zero/blob/master/core/collection/timingwheel.go) 进行管理的
+
+``` go
+timingWheel, err := NewTimingWheel(time.Second, slots, func(k, v interface{}) {
+		key, ok := k.(string)
+		if !ok {
+			return
+		}
+
+		cache.Del(key)
+})
+```
+
+Cache 大小限制，是采用 LRU 淘汰策略，在新增缓存的时候会去检查是否已经超出过限制，具体代码在 keyLru 中实现
+
+``` go
+func (klru *keyLru) add(key string) {
+	if elem, ok := klru.elements[key]; ok {
+		klru.evicts.MoveToFront(elem)
+		return
+	}
+
+	// Add new item
+	elem := klru.evicts.PushFront(key)
+	klru.elements[key] = elem
+
+	// Verify size not exceeded
+	if klru.evicts.Len() > klru.limit {
+		klru.removeOldest()
+	}
+}
+```
+
+Cache 的命中率统计，是在代码中实现 cacheStat,在缓存命中丢失的时候自动统计，并且会定时打印使用的命中率, qps 等状态.
+
+打印的具体效果如下
+
+```go
+cache(proc) - qpm: 2, hit_ratio: 50.0%, elements: 0, hit: 1, miss: 1
+```
+
+缓存击穿包含是使用 syncx.SharedCalls(https://github.com/tal-tech/go-zero/blob/master/core/syncx/sharedcalls.go) 进行实现的，就是将同时请求同一个 key 的请求, 关于 sharedcalls 后续会继续补充。 相关具体实现是在:
+
+```go
+func (c *Cache) Take(key string, fetch func() (interface{}, error)) (interface{}, error) {
+	val, fresh, err := c.barrier.DoEx(key, func() (interface{}, error) {
+		v, e := fetch()
+		if e != nil {
+			return nil, e
+		}
+
+		c.Set(key, v)
+		return v, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if fresh {
+		c.stats.IncrementMiss()
+		return val, nil
+	} else {
+		// got the result from previous ongoing query
+		c.stats.IncrementHit()
+	}
+
+	return val, nil
+}
+```
+
+本文主要介绍了go-zero框架中的 Cache 工具，在实际的项目中非常实用。用好工具对于提升服务性能和开发效率都有很大的帮助，希望本篇文章能给大家带来一些收获。

doc/goctl-model-sql.md

@@ -0,0 +1,272 @@
+# Goctl Model
+
+goctl model 为go-zero下的工具模块中的组件之一，目前支持识别mysql ddl进行model层代码生成，通过命令行或者idea插件（即将支持）可以有选择地生成带redis cache或者不带redis cache的代码逻辑。
+
+## 快速开始
+
+* 通过ddl生成
+
+    ```shell script
+    goctl model mysql ddl -src="./sql/user.sql" -dir="./sql/model" -c=true
+    ```
+
+    执行上述命令后即可快速生成CURD代码。
+
+    ```Plain Text
+    model
+    │   ├── error.go
+    │   └── usermodel.go
+    ```
+
+* 通过datasource生成
+
+    ```shell script
+    goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="table1,table2"  -dir="./model"
+    ```
+
+* 生成代码示例
+  
+  ``` go
+    package model
+
+    import (
+    	"database/sql"
+    	"fmt"
+    	"strings"
+    	"time"
+
+    	"github.com/tal-tech/go-zero/core/stores/cache"
+    	"github.com/tal-tech/go-zero/core/stores/sqlc"
+    	"github.com/tal-tech/go-zero/core/stores/sqlx"
+    	"github.com/tal-tech/go-zero/core/stringx"
+    	"github.com/tal-tech/go-zero/tools/goctl/model/sql/builderx"
+    )
+
+    var (
+    	userFieldNames          = builderx.FieldNames(&User{})
+    	userRows                = strings.Join(userFieldNames, ",")
+    	userRowsExpectAutoSet   = strings.Join(stringx.Remove(userFieldNames, "id", "create_time", "update_time"), ",")
+    	userRowsWithPlaceHolder = strings.Join(stringx.Remove(userFieldNames, "id", "create_time", "update_time"), "=?,") + "=?"
+
+    	cacheUserMobilePrefix = "cache#User#mobile#"
+    	cacheUserIdPrefix     = "cache#User#id#"
+    	cacheUserNamePrefix   = "cache#User#name#"
+    )
+
+    type (
+    	UserModel struct {
+    		sqlc.CachedConn
+    		table string
+    	}
+
+    	User struct {
+    		Id         int64     `db:"id"`
+    		Name       string    `db:"name"`     // 用户名称
+    		Password   string    `db:"password"` // 用户密码
+    		Mobile     string    `db:"mobile"`   // 手机号
+    		Gender     string    `db:"gender"`   // 男｜女｜未公开
+    		Nickname   string    `db:"nickname"` // 用户昵称
+    		CreateTime time.Time `db:"create_time"`
+    		UpdateTime time.Time `db:"update_time"`
+    	}
+    )
+
+    func NewUserModel(conn sqlx.SqlConn, c cache.CacheConf, table string) *UserModel {
+    	return &UserModel{
+    		CachedConn: sqlc.NewConn(conn, c),
+    		table:      table,
+    	}
+    }
+
+    func (m *UserModel) Insert(data User) (sql.Result, error) {
+    	query := `insert into ` + m.table + `(` + userRowsExpectAutoSet + `) value (?, ?, ?, ?, ?)`
+    	return m.ExecNoCache(query, data.Name, data.Password, data.Mobile, data.Gender, data.Nickname)
+    }
+
+    func (m *UserModel) FindOne(id int64) (*User, error) {
+    	userIdKey := fmt.Sprintf("%s%v", cacheUserIdPrefix, id)
+    	var resp User
+    	err := m.QueryRow(&resp, userIdKey, func(conn sqlx.SqlConn, v interface{}) error {
+    		query := `select ` + userRows + ` from ` + m.table + ` where id = ? limit 1`
+    		return conn.QueryRow(v, query, id)
+    	})
+    	switch err {
+    	case nil:
+    		return &resp, nil
+    	case sqlc.ErrNotFound:
+    		return nil, ErrNotFound
+    	default:
+    		return nil, err
+    	}
+    }
+
+    func (m *UserModel) FindOneByName(name string) (*User, error) {
+    	userNameKey := fmt.Sprintf("%s%v", cacheUserNamePrefix, name)
+    	var resp User
+    	err := m.QueryRowIndex(&resp, userNameKey, func(primary interface{}) string {
+    		return fmt.Sprintf("%s%v", cacheUserIdPrefix, primary)
+    	}, func(conn sqlx.SqlConn, v interface{}) (i interface{}, e error) {
+    		query := `select ` + userRows + ` from ` + m.table + ` where name = ? limit 1`
+    		if err := conn.QueryRow(&resp, query, name); err != nil {
+    			return nil, err
+    		}
+    		return resp.Id, nil
+    	}, func(conn sqlx.SqlConn, v, primary interface{}) error {
+    		query := `select ` + userRows + ` from ` + m.table + ` where id = ? limit 1`
+    		return conn.QueryRow(v, query, primary)
+    	})
+    	switch err {
+    	case nil:
+    		return &resp, nil
+    	case sqlc.ErrNotFound:
+    		return nil, ErrNotFound
+    	default:
+    		return nil, err
+    	}
+    }
+
+    func (m *UserModel) FindOneByMobile(mobile string) (*User, error) {
+    	userMobileKey := fmt.Sprintf("%s%v", cacheUserMobilePrefix, mobile)
+    	var resp User
+    	err := m.QueryRowIndex(&resp, userMobileKey, func(primary interface{}) string {
+    		return fmt.Sprintf("%s%v", cacheUserIdPrefix, primary)
+    	}, func(conn sqlx.SqlConn, v interface{}) (i interface{}, e error) {
+    		query := `select ` + userRows + ` from ` + m.table + ` where mobile = ? limit 1`
+    		if err := conn.QueryRow(&resp, query, mobile); err != nil {
+    			return nil, err
+    		}
+    		return resp.Id, nil
+    	}, func(conn sqlx.SqlConn, v, primary interface{}) error {
+    		query := `select ` + userRows + ` from ` + m.table + ` where id = ? limit 1`
+    		return conn.QueryRow(v, query, primary)
+    	})
+    	switch err {
+    	case nil:
+    		return &resp, nil
+    	case sqlc.ErrNotFound:
+    		return nil, ErrNotFound
+    	default:
+    		return nil, err
+    	}
+    }
+
+    func (m *UserModel) Update(data User) error {
+    	userIdKey := fmt.Sprintf("%s%v", cacheUserIdPrefix, data.Id)
+    	_, err := m.Exec(func(conn sqlx.SqlConn) (result sql.Result, err error) {
+    		query := `update ` + m.table + ` set ` + userRowsWithPlaceHolder + ` where id = ?`
+    		return conn.Exec(query, data.Name, data.Password, data.Mobile, data.Gender, data.Nickname, data.Id)
+    	}, userIdKey)
+    	return err
+    }
+
+    func (m *UserModel) Delete(id int64) error {
+    	data, err := m.FindOne(id)
+    	if err != nil {
+    		return err
+    	}
+    	userIdKey := fmt.Sprintf("%s%v", cacheUserIdPrefix, id)
+    	userNameKey := fmt.Sprintf("%s%v", cacheUserNamePrefix, data.Name)
+    	userMobileKey := fmt.Sprintf("%s%v", cacheUserMobilePrefix, data.Mobile)
+    	_, err = m.Exec(func(conn sqlx.SqlConn) (result sql.Result, err error) {
+    		query := `delete from ` + m.table + ` where id = ?`
+    		return conn.Exec(query, id)
+    	}, userIdKey, userNameKey, userMobileKey)
+    	return err
+    }
+  ```
+
+### 用法
+
+```Plain Text
+goctl model mysql -h
+```
+
+```Plain Text
+NAME:
+   goctl model mysql - generate mysql model"
+
+USAGE:
+   goctl model mysql command [command options] [arguments...]
+
+COMMANDS:
+   ddl         generate mysql model from ddl"
+   datasource  generate model from datasource"
+
+OPTIONS:
+   --help, -h  show help
+```
+
+## 生成规则
+
+* 默认规则
+  
+  我们默认用户在建表时会创建createTime、updateTime字段(忽略大小写、下划线命名风格)且默认值均为`CURRENT_TIMESTAMP`，而updateTime支持`ON UPDATE CURRENT_TIMESTAMP`，对于这两个字段生成`insert`、`update`时会被移除，不在赋值范畴内，当然，如果你不需要这两个字段那也无大碍。
+* 带缓存模式
+  * ddl
+
+	```shell script
+	goctl model mysql -src={filename} -dir={dir} -cache=true
+	```
+
+  * datasource
+
+	```shell script
+	goctl model mysql datasource -url={datasource} -table={tables}  -dir={dir} -cache=true
+	```
+
+  目前仅支持redis缓存，如果选择带缓存模式，即生成的`FindOne(ByXxx)`&`Delete`代码会生成带缓存逻辑的代码，目前仅支持单索引字段（除全文索引外），对于联合索引我们默认认为不需要带缓存，且不属于通用型代码，因此没有放在代码生成行列，如example中user表中的`id`、`name`、`mobile`字段均属于单字段索引。
+
+* 不带缓存模式
+  
+  * ddl
+  
+      ```shell script
+      goctl model -src={filename} -dir={dir}
+      ```
+
+  * datasource
+  
+      ```shell script
+          goctl model mysql datasource -url={datasource} -table={tables}  -dir={dir}
+      ```
+
+  or
+  * ddl
+
+      ```shell script
+        goctl model -src={filename} -dir={dir} -cache=false
+      ```
+
+  * datasource
+  
+      ```shell script
+        goctl model mysql datasource -url={datasource} -table={tables}  -dir={dir} -cache=false
+      ```
+  
+  生成代码仅基本的CURD结构。
+
+## 缓存
+
+  对于缓存这一块我选择用一问一答的形式进行罗列。我想这样能够更清晰的描述model中缓存的功能。
+
+* 缓存会缓存哪些信息？
+
+  对于主键字段缓存，会缓存整个结构体信息，而对于单索引字段（除全文索引）则缓存主键字段值。
+
+* 数据有更新（`update`）操作会清空缓存吗？
+  
+  会，但仅清空主键缓存的信息，why？这里就不做详细赘述了。
+
+* 为什么不按照单索引字段生成`updateByXxx`和`deleteByXxx`的代码？
+  
+  理论上是没任何问题，但是我们认为，对于model层的数据操作均是以整个结构体为单位，包括查询，我不建议只查询某部分字段（不反对），否则我们的缓存就没有意义了。
+
+* 为什么不支持`findPageLimit`、`findAll`这么模式代码生层？
+  
+  目前，我认为除了基本的CURD外，其他的代码均属于<i>业务型</i>代码，这个我觉得开发人员根据业务需要进行编写更好。
+
+## QA
+
+* goctl model除了命令行模式，支持插件模式吗？
+
+  很快支持idea插件。

doc/goctl-rpc.md

@@ -0,0 +1,238 @@
+# Rpc Generation
+
+Goctl Rpc是`goctl`脚手架下的一个rpc服务代码生成模块，支持proto模板生成和rpc服务代码生成，通过此工具生成代码你只需要关注业务逻辑编写而不用去编写一些重复性的代码。这使得我们把精力重心放在业务上，从而加快了开发效率且降低了代码出错率。
+
+## 特性
+
+* 简单易用
+* 快速提升开发效率
+* 出错率低
+
+## 快速开始
+
+### 方式一：快速生成greet服务
+
+  通过命令 `goctl rpc new ${servieName}`生成
+
+  如生成greet rpc服务：
+
+  ```shell script
+  goctl rpc new greet
+  ```
+
+  执行后代码结构如下:
+
+  ```golang
+  └── greet
+    ├── etc
+    │   └── greet.yaml
+    ├── go.mod
+    ├── go.sum
+    ├── greet
+    │   ├── greet.go
+    │   ├── greet_mock.go
+    │   └── types.go
+    ├── greet.go
+    ├── greet.proto
+    ├── internal
+    │   ├── config
+    │   │   └── config.go
+    │   ├── logic
+    │   │   └── pinglogic.go
+    │   ├── server
+    │   │   └── greetserver.go
+    │   └── svc
+    │       └── servicecontext.go
+    └── pb
+        └── greet.pb.go
+  ```
+
+rpc一键生成常见问题解决见 <a href="#常见问题解决">常见问题解决</a>
+
+### 方式二：通过指定proto生成rpc服务
+
+* 生成proto模板
+
+  ```shell script
+  goctl rpc template -o=user.proto
+  ```
+
+  ```golang
+  syntax = "proto3";
+
+  package remote;
+
+  message Request {
+    // 用户名
+    string username = 1;
+    // 用户密码
+    string password = 2;
+  }
+
+  message Response {
+    // 用户名称
+    string name = 1;
+    // 用户性别
+    string gender = 2;
+  }
+
+  service User {
+    // 登录
+    rpc Login(Request)returns(Response);
+  }
+  ```
+
+* 生成rpc服务代码
+
+  ```shell script
+  goctl rpc proto -src=user.proto
+  ```
+
+  代码tree
+
+  ```Plain Text
+  user
+      ├── etc
+      │   └── user.json
+      ├── internal
+      │   ├── config
+      │   │   └── config.go
+      │   ├── handler
+      │   │   ├── loginhandler.go
+      │   ├── logic
+      │   │   └── loginlogic.go
+      │   └── svc
+      │       └── servicecontext.go
+      ├── pb
+      │   └── user.pb.go
+      ├── shared
+      │   ├── mockusermodel.go
+      │   ├── types.go
+      │   └── usermodel.go
+      ├── user.go
+      └── user.proto
+
+  ```
+
+## 准备工作
+
+* 安装了go环境
+* 安装了protoc&protoc-gen-go，并且已经设置环境变量
+* mockgen(可选,将移除)
+* 更多问题请见 <a href="#注意事项">注意事项</a>
+
+## 用法
+
+### rpc服务生成用法
+
+```shell script
+goctl rpc proto -h
+```
+
+```shell script
+NAME:
+   goctl rpc proto - generate rpc from proto
+
+USAGE:
+   goctl rpc proto [command options] [arguments...]
+
+OPTIONS:
+   --src value, -s value         the file path of the proto source file
+   --dir value, -d value         the target path of the code,default path is "${pwd}". [option]
+   --service value, --srv value  the name of rpc service. [option]
+   --shared[已废弃] value                the dir of the shared file,default path is "${pwd}/shared. [option]"
+   --idea                        whether the command execution environment is from idea plugin. [option]
+
+```
+
+### 参数说明
+
+* --src 必填，proto数据源，目前暂时支持单个proto文件生成，这里不支持（不建议）外部依赖
+* --dir 非必填，默认为proto文件所在目录，生成代码的目标目录
+* --service 服务名称，非必填，默认为proto文件所在目录名称，但是，如果proto所在目录为一下结构：
+
+    ```shell script
+    user
+        ├── cmd
+        │   └── rpc
+        │       └── user.proto
+    ```
+
+    则服务名称亦为user，而非proto所在文件夹名称了，这里推荐使用这种结构，可以方便在同一个服务名下建立不同类型的服务(api、rpc、mq等)，便于代码管理与维护。
+* --shared[⚠️已废弃] 非必填，默认为$dir(xxx.proto)/shared，rpc client逻辑代码存放目录。
+  
+  > 注意：这里的shared文件夹名称将会是代码中的package名称。
+
+* --idea 非必填，是否为idea插件中执行，保留字段，终端执行可以忽略
+
+## 开发人员需要做什么
+
+关注业务代码编写，将重复性、与业务无关的工作交给goctl，生成好rpc服务代码后，开饭人员仅需要修改
+
+* 服务中的配置文件编写(etc/xx.json、internal/config/config.go)
+* 服务中业务逻辑编写(internal/logic/xxlogic.go)
+* 服务中资源上下文的编写(internal/svc/servicecontext.go)
+
+## 扩展
+
+对于需要进行rpc mock的开发人员，在安装了`mockgen`工具的前提下可以在rpc的shared文件中生成好对应的mock文件。
+
+## 注意事项
+
+* `google.golang.org/grpc`需要降级到v1.26.0,且protoc-gen-go版本不能高于v1.3.2（see [https://github.com/grpc/grpc-go/issues/3347](https://github.com/grpc/grpc-go/issues/3347)）即
+
+  ```shell script
+  replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
+  ```
+
+* proto不支持暂多文件同时生成
+* proto不支持外部依赖包引入，message不支持inline
+* 目前main文件、shared文件、handler文件会被强制覆盖，而和开发人员手动需要编写的则不会覆盖生成，这一类在代码头部均有
+
+    ```shell script
+    // Code generated by goctl. DO NOT EDIT!
+    // Source: xxx.proto
+    ```
+  
+  的标识，请注意不要将也写业务性代码写在里面。
+
+## 常见问题解决(go mod工程)
+
+* 错误一:
+
+  ```golang
+  pb/xx.pb.go:220:7: undefined: grpc.ClientConnInterface
+  pb/xx.pb.go:224:11: undefined: grpc.SupportPackageIsVersion6
+  pb/xx.pb.go:234:5: undefined: grpc.ClientConnInterface
+  pb/xx.pb.go:237:24: undefined: grpc.ClientConnInterface
+  ```
+
+  解决方法：请将`protoc-gen-go`版本降至v1.3.2及一下
+
+* 错误二:
+
+  ```golang
+
+  # go.etcd.io/etcd/clientv3/balancer/picker
+  ../../../go/pkg/mod/go.etcd.io/etcd@v0.0.0-20200402134248-51bdeb39e698/clientv3/balancer/picker/err.go:25:9: cannot use &errPicker literal (type *errPicker) as type Picker in return argument:*errPicker does not implement Picker (wrong type for Pick method)
+    have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
+    want Pick(balancer.PickInfo) (balancer.PickResult, error)
+    ../../../go/pkg/mod/go.etcd.io/etcd@v0.0.0-20200402134248-51bdeb39e698/clientv3/balancer/picker/roundrobin_balanced.go:33:9: cannot use &rrBalanced literal (type *rrBalanced) as type Picker in return argument:
+    *rrBalanced does not implement Picker (wrong type for Pick method)
+		have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
+    want Pick(balancer.PickInfo) (balancer.PickResult, error)
+    #github.com/tal-tech/go-zero/zrpc/internal/balancer/p2c
+    ../../../go/pkg/mod/github.com/tal-tech/go-zero@v1.0.12/zrpc/internal/balancer/p2c/p2c.go:41:32: not enough arguments in call to base.NewBalancerBuilder
+	have (string, *p2cPickerBuilder)
+  want (string, base.PickerBuilder, base.Config)
+  ../../../go/pkg/mod/github.com/tal-tech/go-zero@v1.0.12/zrpc/internal/balancer/p2c/p2c.go:58:9: cannot use &p2cPicker literal (type *p2cPicker) as type balancer.Picker in return argument:
+	*p2cPicker does not implement balancer.Picker (wrong type for Pick method)
+		have Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error)
+		want Pick(balancer.PickInfo) (balancer.PickResult, error)
+  ```
+
+  解决方法：
+  
+    ```golang
+    replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
+    ```

doc/goctl.md

@@ -1,33 +1,40 @@
-# goctl使用说明
+# goctl使用
 
 ## goctl用途
+
 * 定义api请求
 * 根据定义的api自动生成golang(后端), java(iOS & Android), typescript(web & 晓程序)，dart(flutter)
 * 生成MySQL CURD+Cache
-*  生成MongoDB CURD+Cache
+* 生成MongoDB CURD+Cache
 
 ## goctl使用说明
+
+### 快速生成服务
+
+* api: goctl api new xxxx
+* rpc: goctl rpc new xxxx
+
 #### goctl参数说明
 
   `goctl api [go/java/ts] [-api user/user.api] [-dir ./src]`
 
   > api 后面接生成的语言，现支持go/java/typescript
-
+  >
   > -api 自定义api所在路径
-
+  >
   > -dir 自定义生成目录
 
 #### 保持goctl总是最新版
 
   第一次运行会在~/.goctl里增加下面两行：
 
-  ```
+  ```Plain Text
   url = http://47.97.184.41:7777/
   ```
 
 #### API 语法说明
 
-```
+``` golang
 info(
     title: doc title
     desc: >
@@ -88,20 +95,20 @@ service user-api {
     )
     @server(
         handler: GetUserHandler
-        folder: user
+        group: user
     )
     get /api/user/:name(getRequest) returns(getResponse)
 
     @server(
         handler: CreateUserHandler
-        folder: user
+        group: user
     )
     post /api/users/create(createRequest)
 }
 
 @server(
     jwt: Auth
-    folder: profile
+    group: profile
 )
 service user-api {
     @doc(summary: user title)
@@ -123,15 +130,18 @@ service user-api {
     )
     head /api/ping()
 }
+
 ```
+
 1. info部分：描述了api基本信息，比如Auth，api是哪个用途。
 2. type部分：type类型声明和golang语法兼容。
-3. service部分：service代表一组服务，一个服务可以由多组名称相同的service组成，可以针对每一组service配置jwt和auth认证，另外通过folder属性可以指定service生成所在子目录。
+3. service部分：service代表一组服务，一个服务可以由多组名称相同的service组成，可以针对每一组service配置jwt和auth认证，另外通过group属性可以指定service生成所在子目录。
    service里面包含api路由，比如上面第一组service的第一个路由，doc用来描述此路由的用途，GetProfileHandler表示处理这个路由的handler，
    `get /api/profile/:name(getRequest) returns(getResponse)` 中get代表api的请求方式（get/post/put/delete）, `/api/profile/:name` 描述了路由path，`:name`通过
    请求getRequest里面的属性赋值，getResponse为返回的结构体，这两个类型都定义在2描述的类型中。
 
 #### api vscode插件
+
 开发者可以在vscode中搜索goctl的api插件，它提供了api语法高亮，语法检测和格式化相关功能。
 
  1. 支持语法高亮和类型导航。
@@ -143,7 +153,7 @@ service user-api {
   命令如下：  
   `goctl api go -api user/user.api -dir user`
 
-  ```
+  ```Plain Text
 
 	.
     ├── internal
@@ -173,17 +183,20 @@ service user-api {
     └── user.go
 
   ```
+
   生成的代码可以直接跑，有几个地方需要改：
 
-  * 在`servicecontext.go`里面增加需要传递给logic的一些资源，比如mysql, redis，rpc等
-  * 在定义的get/post/put/delete等请求的handler和logic里增加处理业务逻辑的代码
+* 在`servicecontext.go`里面增加需要传递给logic的一些资源，比如mysql, redis，rpc等
+* 在定义的get/post/put/delete等请求的handler和logic里增加处理业务逻辑的代码
 
 #### 根据定义好的api文件生成java代码
+
 ```shell
 goctl api java -api user/user.api -dir ./src
 ```
 
 #### 根据定义好的api文件生成typescript代码
+
 ```shell
 goctl api ts -api user/user.api -dir ./src -webapi ***
 
@@ -191,6 +204,7 @@ ts需要指定webapi所在目录
 ```
 
 #### 根据定义好的api文件生成Dart代码
+
 ```shell
 goctl api dart -api user/user.api -dir ./src
 ```
@@ -198,32 +212,37 @@ goctl api dart -api user/user.api -dir ./src
 ## 根据mysql ddl或者datasource生成model文件
 
 ```shell script
-$ goctl model mysql -src={filename} -dir={dir} -cache={true|false}
+goctl model mysql -src={filename} -dir={dir} -cache={true|false}
 ```
+
 详情参考[model文档](https://github.com/tal-tech/go-zero/blob/master/tools/goctl/model/sql/README.MD)
 
 ## 根据定义好的简单go文件生成mongo代码文件(仅限golang使用)  
+
 ```shell
 goctl model mongo -src {{yourDir}}/xiao/service/xhb/user/model/usermodel.go -cache yes
-
--src需要提供简单的usermodel.go文件，里面只需要提供一个结构体即可
--cache 控制是否需要缓存 yes=需要 no=不需要
-src 示例代码如下
 ```
-  ```go
-package model
 
-type User struct {
-	Name string `o:"find,get,set" c:"姓名"`
-	Age int `o:"find,get,set" c:"年纪"`
-	School string `c:"学校"`
-}
+* src需要提供简单的usermodel.go文件，里面只需要提供一个结构体即可
+* cache 控制是否需要缓存 yes=需要 no=不需要
+
+src 示例代码如下
+
+  ```go
+    package model
+
+    type User struct {
+      Name string `o:"find,get,set" c:"姓名"`
+      Age int `o:"find,get,set" c:"年纪"`
+      School string `c:"学校"`
+    }
   ```
-     结构体中不需要提供Id,CreateTime,UpdateTime三个字段，会自动生成   
-     结构体中每个tag有两个可选标签 c 和 o  
-     c是改字段的注释  
-     o是改字段需要生产的操作函数 可以取得get,find,set 分别表示生成返回单个对象的查询方法，返回多个对象的查询方法，设置该字段方法  
-     生成的目标文件会覆盖该简单go文件  
+
+结构体中不需要提供Id,CreateTime,UpdateTime三个字段，会自动生成
+结构体中每个tag有两个可选标签 c 和 o
+c 是该字段的注释
+o 是该字段需要生产的操作函数 可以取得get,find,set 分别表示生成返回单个对象的查询方法，返回多个对象的查询方法，设置该字段方法
+生成的目标文件会覆盖该简单go文件
 
 ## goctl rpc生成（业务剥离中，暂未开放）
 
@@ -232,15 +251,15 @@ type User struct {
 
   参数说明：
 
-  - ${proto}: proto文件
-  - ${serviceName}: rpc服务名称
-  - ${projectName}: 所属项目，如xjy,xhb,crm,hera，具体查看help，主要为了根据不同项目服务往redis注册key，可选
-  - ${directory}: 输出目录
-  - ${shared}: shared文件生成目录，可选，默认为${pwd}/shared
+* ${proto}: proto文件
+* ${serviceName}: rpc服务名称
+* ${projectName}: 所属项目，如xjy,xhb,crm,hera，具体查看help，主要为了根据不同项目服务往redis注册key，可选
+* ${directory}: 输出目录
+* ${shared}: shared文件生成目录，可选，默认为${pwd}/shared
 
-  生成目录结构示例：
+生成目录结构示例：
 
-  ``` go
+  ```Plain Text
 	.
     ├── shared [示例目录，可自己指定，强制覆盖更新]
     │   └── contentservicemodel.go
@@ -276,4 +295,5 @@ type User struct {
     │   └── test.go [强制覆盖更新]
     └── test.proto
   ```
-  - 注意 ：目前rpc目录生成的proto文件暂不支持import外部proto文件   
+
+注意 ：目前rpc目录生成的proto文件暂不支持import外部proto文件