Skip to content

Commit e524822

Browse files
soxftsoxft
committed
⚗️ Perf: set key expire performance optimization
1 parent 369c9e3 commit e524822

File tree

6 files changed

+141
-41
lines changed

6 files changed

+141
-41
lines changed

‎core/count.go

Lines changed: 3 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,8 @@ import (
66
"github.com/soxft/busuanzi/process/redisutil"
77
"github.com/spf13/viper"
88
"strings"
9-
"time"
109
)
1110

12-
// index 数据类型 key
13-
// sitePv Str bsz:site_pv:md5(host)
14-
// siteUv HyperLogLog bsz:site_uv:md5(host)
15-
// pagePv zset bsz:page_pv:md5(host) / md5(path)
16-
// pageUv HyperLogLog bsz:site_uv:md5(host):md5(path)
17-
18-
type Counts struct {
19-
SitePv int64
20-
SiteUv int64
21-
PagePv int64
22-
PageUv int64
23-
}
24-
25-
type RKeys struct {
26-
SitePvKey string
27-
SiteUvKey string
28-
PagePvKey string
29-
PageUvKey string
30-
SiteUnique string
31-
PathUnique string
32-
}
33-
3411
// Count
3512
// @description return and count the number of users in the redis
3613
func Count(ctx context.Context, host string, path string, userIdentity string) Counts {
@@ -51,7 +28,7 @@ func Count(ctx context.Context, host string, path string, userIdentity string) C
5128
pageUv, _ := _redis.PFCount(ctx, rk.PageUvKey).Result()
5229

5330
// setExpire
54-
go setExpire(ctx, rk.SiteUvKey, rk.PageUvKey, rk.SitePvKey, rk.PagePvKey)
31+
go setExpire(rk.SiteUvKey, rk.PageUvKey, rk.SitePvKey, rk.PagePvKey)
5532

5633
return Counts{
5734
SitePv: sitePv,
@@ -77,7 +54,7 @@ func Put(ctx context.Context, host string, path string, userIdentity string) {
7754
_redis.PFAdd(ctx, rk.PageUvKey, userIdentity)
7855

7956
// setExpire
80-
go setExpire(ctx, rk.SiteUvKey, rk.PageUvKey, rk.SitePvKey, rk.PagePvKey)
57+
go setExpire(rk.SiteUvKey, rk.PageUvKey, rk.SitePvKey, rk.PagePvKey)
8158
return
8259
}
8360

@@ -96,7 +73,7 @@ func Get(ctx context.Context, host string, path string) Counts {
9673
pageUv, _ := _redis.PFCount(ctx, rk.PageUvKey).Result()
9774

9875
// setExpire
99-
go setExpire(ctx, rk.SiteUvKey, rk.PageUvKey, rk.SitePvKey, rk.PagePvKey)
76+
go setExpire(rk.SiteUvKey, rk.PageUvKey, rk.SitePvKey, rk.PagePvKey)
10077

10178
return Counts{
10279
SitePv: sitePv,
@@ -106,21 +83,6 @@ func Get(ctx context.Context, host string, path string) Counts {
10683
}
10784
}
10885

109-
// setExpire
110-
// @description set the expiration time of the key
111-
// TODO 使用 MQ 进行防抖消重, 避免频繁调用
112-
func setExpire(ctx context.Context, key ...string) {
113-
if viper.GetInt("bsz.expire") == 0 {
114-
return
115-
}
116-
117-
_redis := redisutil.RDB
118-
119-
for _, k := range key {
120-
_redis.Expire(ctx, k, viper.GetDuration("bsz.expire")*time.Second)
121-
}
122-
}
123-
12486
func getKeys(host string, path string) RKeys {
12587
var siteUnique = host
12688
var pathUnique = path

‎core/expire.go

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package core
2+
3+
import (
4+
"context"
5+
camp "github.com/orcaman/concurrent-map/v2"
6+
"github.com/soxft/busuanzi/process/redisutil"
7+
"github.com/spf13/viper"
8+
"log"
9+
"time"
10+
)
11+
12+
var expQ expireQueue
13+
var expTime = time.Second * 20
14+
15+
// InitExpire init expireQueue
16+
func InitExpire() {
17+
if viper.GetInt("bsz.expire") == 0 {
18+
return
19+
}
20+
21+
expQ.Queue = make(chan string, 100)
22+
expQ.Cache = camp.New[time.Time]()
23+
24+
defer func() {
25+
if err := recover(); err != nil {
26+
close(expQ.Queue)
27+
expQ.Cache.Clear()
28+
29+
log.Printf("[ERROR] core.Init %s \n", err)
30+
31+
InitExpire()
32+
}
33+
}()
34+
35+
// 监听 Expire chan 事件 消费
36+
go func() {
37+
for t := range expQ.Queue {
38+
// log.Printf("Get Expire Queue %s \n", t)
39+
40+
redisutil.RDB.Expire(context.Background(), t, viper.GetDuration("bsz.expire")*time.Second)
41+
}
42+
}()
43+
44+
go func() {
45+
// 定期清理过期数据
46+
timer := time.NewTicker(time.Second * 1)
47+
48+
for {
49+
select {
50+
case <-timer.C:
51+
for k, v := range expQ.Cache.Items() {
52+
if time.Now().Sub(v) > expTime {
53+
expQ.Cache.Remove(k)
54+
55+
// log.Printf("[%s] expire left %.2f delete \n", k, time.Now().Sub(v).Seconds())
56+
}
57+
}
58+
}
59+
}
60+
}()
61+
}
62+
63+
func isInList(str string) bool {
64+
if t, ok := expQ.Cache.Get(str); ok {
65+
66+
var tNow = time.Now()
67+
var tSub = tNow.Sub(t)
68+
69+
if tSub < expTime {
70+
// log.Printf("[%s] expire left %.2f (%d - %d) ignore \n", str, tSub.Seconds(), tNow.Unix(), t.Unix())
71+
72+
return true
73+
}
74+
//else {
75+
// expQ.Cache.Remove(str)
76+
// log.Printf("[%s] expire left %.2f (%d - %d) delete \n", str, tSub.Seconds(), tNow.Unix(), t.Unix())
77+
//}
78+
}
79+
80+
return false
81+
}
82+
83+
// setExpire
84+
// @description set the expiration time of the key
85+
func setExpire(key ...string) {
86+
if viper.GetInt("bsz.expire") == 0 {
87+
return
88+
}
89+
90+
for _, k := range key {
91+
if isInList(k) {
92+
continue
93+
}
94+
95+
expQ.Queue <- k
96+
97+
expQ.Cache.Set(k, time.Now())
98+
}
99+
}

‎core/types.go

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package core
2+
3+
import (
4+
cmap "github.com/orcaman/concurrent-map/v2"
5+
"time"
6+
)
7+
8+
// index 数据类型 key
9+
// sitePv Str bsz:site_pv:md5(host)
10+
// siteUv HyperLogLog bsz:site_uv:md5(host)
11+
// pagePv zset bsz:page_pv:md5(host) / md5(path)
12+
// pageUv HyperLogLog bsz:site_uv:md5(host):md5(path)
13+
14+
type Counts struct {
15+
SitePv int64
16+
SiteUv int64
17+
PagePv int64
18+
PageUv int64
19+
}
20+
21+
type RKeys struct {
22+
SitePvKey string
23+
SiteUvKey string
24+
PagePvKey string
25+
PageUvKey string
26+
SiteUnique string
27+
PathUnique string
28+
}
29+
30+
type expireQueue struct {
31+
Queue chan string
32+
Cache cmap.ConcurrentMap[string, time.Time]
33+
}

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ require (
3131
github.com/mitchellh/mapstructure v1.5.0 // indirect
3232
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
3333
github.com/modern-go/reflect2 v1.0.2 // indirect
34+
github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
3435
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
3536
github.com/sagikazarmark/locafero v0.6.0 // indirect
3637
github.com/sagikazarmark/slog-shim v0.1.0 // indirect

‎go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
6767
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
6868
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
6969
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
70+
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
71+
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
7072
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
7173
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
7274
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

‎main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"github.com/soxft/busuanzi/config"
5+
"github.com/soxft/busuanzi/core"
56
"github.com/soxft/busuanzi/process/redisutil"
67
"github.com/soxft/busuanzi/process/webutil"
78
)
@@ -10,5 +11,7 @@ func main() {
1011
config.Init()
1112
redisutil.Init()
1213

14+
core.InitExpire()
15+
1316
webutil.Init()
1417
}

0 commit comments

Comments
 (0)