访问记录存储优化

cmd/migraterequesthistory/migraterequesthistory.go

@@ -1,62 +1,112 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/go-redis/redis"
 	"github.com/jwma/jump-jump/internal/app/db"
+	"github.com/jwma/jump-jump/internal/app/models"
 	"github.com/jwma/jump-jump/internal/app/utils"
 	"sync"
 )
 
 var wg sync.WaitGroup
 
-func migrate(id string) {
+func getRequestHistoryKeyV120(id string) string {
+	return fmt.Sprintf("history:%s", id)
+}
+
+func handleV120Migrate(id string) {
 	rdb := db.GetRedisClient()
-	rhKey := utils.GetRequestHistoryKey(id)
+	rhKey := getRequestHistoryKeyV120(id)
 
 	if exists, _ := rdb.Exists(rhKey).Result(); exists == 1 {
 		fmt.Printf("%s 无需迁移\n", rhKey)
 		wg.Done()
 		return
 	}
 
-	q := fmt.Sprintf("%s:*", rhKey)
+	keys, _ := rdb.Keys(fmt.Sprintf("%s:*", rhKey)).Result()
 
-	keys, _ := rdb.Keys(q).Result()
 	if len(keys) == 0 {
 		wg.Done()
 		return
 	}
 
 	for _, key := range keys {
-		r, _ := rdb.LRange(key, 0, 99999).Result()
+		r, _ := rdb.LRange(key, 0, -1).Result()
 		rdb.RPush(rhKey, r)
 	}
 
 	rdb.Del(keys...)
-
 	wg.Done()
 }
 
-func main() {
+func getShortLinkIds() []string {
 	rdb := db.GetRedisClient()
 	c, err := rdb.ZCard(utils.GetShortLinksKey()).Result()
 
 	if err != nil {
 		panic(err)
 	}
 
-	fmt.Printf("总共有%d个短链接的访问记录可能需要迁移\n", c)
 	ids, err := rdb.ZRange(utils.GetShortLinksKey(), 0, c).Result()
 
 	if err != nil {
 		panic(err)
 	}
 
+	return ids
+}
+
+func startV120Migration() {
+	ids := getShortLinkIds()
+	fmt.Printf("[V1.2.0] 总共有 %d 个短链接的访问记录可能需要迁移\n", len(ids))
+
 	for _, id := range ids {
 		wg.Add(1)
-		go migrate(id)
+		go handleV120Migrate(id)
 	}
 
 	wg.Wait()
-	fmt.Println("迁移完毕")
+	fmt.Println("[V1.2.0] 迁移完毕")
+}
+
+func handleV130Migrate(id string) {
+	rdb := db.GetRedisClient()
+	key := utils.GetRequestHistoryKey(id)
+	v120key := getRequestHistoryKeyV120(id)
+	r, _ := rdb.LRange(v120key, 0, -1).Result()
+
+	for _, rhStr := range r {
+		rh := &models.RequestHistory{}
+		_ = json.Unmarshal([]byte(rhStr), rh)
+		rh.Id = utils.RandStringRunes(6)
+		rdb.ZAdd(key, redis.Z{
+			Score:  float64(rh.Time.Unix()),
+			Member: rh,
+		})
+	}
+
+	rdb.Del(v120key)
+	wg.Done()
+}
+
+func startV130Migration() {
+	ids := getShortLinkIds()
+	fmt.Printf("[V1.3.0] 总共有 %d 个短链接的访问记录可能需要迁移\n", len(ids))
+
+	for _, id := range ids {
+		wg.Add(1)
+		go handleV130Migrate(id)
+	}
+
+	wg.Wait()
+	fmt.Println("[V1.3.0] 迁移完毕")
+}
+
+func main() {
+	startV120Migration()
+	fmt.Println()
+	startV130Migration()
 }

internal/app/handlers/shortlink.go

@@ -10,7 +10,6 @@ import (
 	"github.com/jwma/jump-jump/internal/app/utils"
 	"log"
 	"net/http"
-	"strconv"
 )
 
 func GetShortLinkAPI() gin.HandlerFunc {
@@ -237,12 +236,7 @@ func ShortLinkActionAPI() gin.HandlerFunc {
 			}
 
 			repo := repository.GetRequestHistoryRepo(db.GetRedisClient())
-			sizeStr := c.Query("size")
-			size, err := strconv.Atoi(sizeStr)
-
-			if err != nil {
-				size = 20
-			}
+			size := utils.GetIntQueryValue(c, "size", 20)
 
 			r, err := repo.FindLatest(s.Id, int64(size))
 

internal/app/models/models.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"encoding/json"
 	"time"
 )
 
@@ -52,13 +53,18 @@ type UpdateShortLinkParameter struct {
 }
 
 type RequestHistory struct {
+	Id   string     `json:"id"`
 	Link *ShortLink `json:"-"`
 	Url  string     `json:"url"` // 由于短链接的目标连接可能会被修改，可以在访问历史记录中记录一下当前的目标连接
 	IP   string     `json:"ip"`
 	UA   string     `json:"ua"`
 	Time time.Time  `json:"time"`
 }
 
+func (r *RequestHistory) MarshalBinary() (data []byte, err error) {
+	return json.Marshal(r)
+}
+
 func NewRequestHistory(link *ShortLink, IP string, UA string) *RequestHistory {
 	return &RequestHistory{Link: link, IP: IP, UA: UA, Url: link.Url}
 }

internal/app/repository/repository.go

@@ -7,7 +7,6 @@ import (
 	"github.com/go-redis/redis"
 	"github.com/jwma/jump-jump/internal/app/models"
 	"github.com/jwma/jump-jump/internal/app/utils"
-	"github.com/thoas/go-funk"
 	"log"
 	"time"
 )
@@ -43,29 +42,33 @@ func GetRequestHistoryRepo(rdb *redis.Client) *requestHistoryRepository {
 }
 
 func (r *requestHistoryRepository) Save(rh *models.RequestHistory) {
+	rh.Id = utils.RandStringRunes(6)
 	rh.Time = time.Now()
 	key := utils.GetRequestHistoryKey(rh.Link.Id)
-	j, err := json.Marshal(rh)
+
+	_, err := r.db.ZAdd(key, redis.Z{
+		Score:  float64(rh.Time.Unix()),
+		Member: rh,
+	}).Result()
+
 	if err != nil {
-		log.Printf("fail to save request history with key: %s, error: %v\n", key, err)
-		return
+		log.Printf("fail to save request history with key: %s, error: %v, data: %v\n", key, err, rh)
 	}
-
-	r.db.RPush(key, j)
 }
 
 func (r *requestHistoryRepository) FindLatest(linkId string, size int64) (*requestHistoryListResult, error) {
 	key := utils.GetRequestHistoryKey(linkId)
-	rawRs, err := r.db.LRange(key, -size, -1).Result()
+	rs, err := r.db.ZRangeWithScores(key, -size, -1).Result()
 
 	if err != nil {
 		log.Printf("failed to find request history latest records with key: %s, err: %v\n", key, err)
 	}
 
+	utils.ReverseAny(rs)
 	result := newEmptyRequestHistoryResult()
-	for _, one := range funk.ReverseStrings(rawRs) {
+	for _, one := range rs {
 		rh := &models.RequestHistory{}
-		_ = json.Unmarshal([]byte(one), rh)
+		_ = json.Unmarshal([]byte(one.Member.(string)), rh)
 		result.addHistory(rh)
 	}
 
@@ -250,10 +253,7 @@ func (r *shortLinkRepository) Delete(s *models.ShortLink) {
 	_, _ = pipeline.Exec()
 
 	// 删除访问历史
-	keys, _ := r.db.Keys(fmt.Sprintf("history:%s:*", s.Id)).Result()
-	if len(keys) > 0 {
-		r.db.Del(keys...)
-	}
+	r.db.Del(utils.GetRequestHistoryKey(s.Id))
 }
 
 func (r *shortLinkRepository) Get(id string) (*models.ShortLink, error) {

internal/app/utils/keys.go

@@ -21,5 +21,5 @@ func GetUserShortLinksKey(username string) string {
 }
 
 func GetRequestHistoryKey(linkId string) string {
-	return fmt.Sprintf("history:%s", linkId)
+	return fmt.Sprintf("rh:%s", linkId)
 }

internal/app/utils/slice.go

@@ -0,0 +1,11 @@
+package utils
+
+import "reflect"
+
+func ReverseAny(s interface{}) {
+	n := reflect.ValueOf(s).Len()
+	swap := reflect.Swapper(s)
+	for i, j := 0, n-1; i < j; i, j = i+1, j-1 {
+		swap(i, j)
+	}
+}