add:first commit

.gitignore

@@ -0,0 +1,3 @@
+.idea
+*.exe
+*.log

README.md

@@ -0,0 +1,25 @@
+# MomentsCleaner 
+群晖Moments重复文件清理工具<br>
+扫描当前目录及所有子目录下的重复文件(根据文件md5值)，将同文件目录下的重复文件删除<br>
+
+下载链接：
+
+## 工具来由
+ 群晖moments经常在同个文件夹下出现好多重复文件,比如xxx_1.jpg,xxx_2.jpg,占用空间<br>
+ 群晖自带的存储空间分析器能够将重复文件列出并删除，但都需要手动判别删除，重复文件过多就难受了<br>
+ 需要一个安全的自动化的工具，so MomentsCleaner come<br> 
+
+## 使用
+1，需要在windows系统中使用SynologyDrive设置同步Moments,将所有照片同步到本地<br>
+2，在本地Moments文件目录下或子文件目录，将下载好的MomentsCleaner.exe放入，执行<br>
+![momentscleaner](doc/momentscleaner.png)
+执行完成后，会生成"被删除的文件"目录，所有被删除的文件全在这里，检查后再删除<br>
+3，清理后(将momentscleaner.exe,cleaner_info.log,被删除的文件全删)，开启Drive的双向同步（或单向上传）将更新同步到群晖上
+
+## 说明
+  为什么只删除同文件目录下的重复文件？<br>
+  这是因为群晖moments在同个文件夹下容易出现重复文件，只处理同文件目录下的重复文件，安全并在大部分场景下效果达到<br>
+
+  两个文件重复，删除哪一个？<br>
+  根据名字长度，名称较短的保留，其它删除<br>
+

cleaner/cleaner.go

@@ -0,0 +1,115 @@
+package cleaner
+
+import (
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+	"os"
+	"strings"
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const BACKUP_DIR_NAME = "被删除的文件"
+
+var allDelCount int32
+
+func DoClean() {
+	dirWalk("./")
+	logrus.Infof("总共有%d个重复文件被移除", allDelCount)
+}
+
+func dirWalk(path string) {
+
+	if strings.Contains(path, BACKUP_DIR_NAME) {
+		return
+	}
+	log := logrus.WithField("目录", path)
+	hidden, err := isFileHidden(path)
+	if err != nil {
+		log.WithError(err).Info("isFileHidden")
+	}
+	if hidden {
+		return
+	}
+
+	fs, err := ioutil.ReadDir(path)
+	if err != nil {
+		logrus.Panic(err)
+	}
+	hash2files := make(map[string][]os.FileInfo, 0)
+	for _, file := range fs {
+		if file.IsDir() {
+			dirWalk(path + file.Name() + "/")
+		} else {
+			name := path + file.Name()
+			md5, err := MD5File(name)
+			if err != nil {
+				logrus.Panic(err)
+			}
+			hash2files[md5] = append(hash2files[md5], file)
+		}
+	}
+
+	log.Info("扫描开始")
+	var delCount int32
+	for _, files := range hash2files {
+		if len(files) < 2 {
+			continue
+		}
+		//保留名称最短的文件，其它重复文件删除
+		min := len(files[0].Name())
+		for i := 1; i < len(files); i++ {
+			lname := len(files[i].Name())
+			if lname < min {
+				min = lname
+			}
+		}
+		for _, file := range files {
+			if len(file.Name()) == min {
+				log.WithField("filename", file.Name()).Info("保留")
+				continue
+			}
+			backupDir := "./" + BACKUP_DIR_NAME + "/" + path[2:]
+			createDirIfNoExist(backupDir)
+			err = os.Rename(path+file.Name(), backupDir+file.Name())
+			if err != nil {
+				logrus.Panic(err)
+			}
+			delCount++
+			allDelCount++
+			log.WithField("filename", file.Name()).Info("删除")
+		}
+	}
+
+	if delCount > 0 {
+		log.Infof("%d个文件被移除", delCount)
+	}
+}
+
+func createDirIfNoExist(path string) {
+	_, err := os.Stat(path) //os.Stat获取文件信息
+	if err != nil {
+		if os.IsNotExist(err) {
+			os.MkdirAll(path, os.ModePerm) //  Everyone can read write and execute
+			return
+		}
+		return
+	}
+}
+
+func isFileHidden(path string) (bool, error) {
+
+	name := utf16.Encode([]rune(path + "\x00"))
+
+	attributes, err := syscall.GetFileAttributes((*uint16)(unsafe.Pointer(&name[0])))
+
+	if err != nil {
+
+		return false, err
+
+	}
+
+	return attributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil
+
+}

cleaner/md5.go

@@ -0,0 +1,84 @@
+package cleaner
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"sort"
+	"sync"
+)
+
+func MD5Bytes(s []byte) string {
+	ret := md5.Sum(s)
+	return hex.EncodeToString(ret[:])
+}
+
+//计算字符串MD5值
+func MD5(s string) string {
+	return MD5Bytes([]byte(s))
+}
+
+//计算文件MD5值
+func MD5File(file string) (string, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return "", err
+	}
+	return MD5Bytes(data), nil
+}
+
+type md5result struct {
+	file string
+	md5  string
+	err  error
+}
+
+//多个文件计算时，result = md5(md5(file1)+md5(file2))
+func MD5Files(files ...string) (string, error) {
+	length := len(files)
+
+	if length == 0 {
+		return "", errors.New("input param error")
+	}
+
+	if length == 1 {
+		return MD5File(files[0])
+	}
+
+	var wg sync.WaitGroup
+	var m sync.Map
+
+	wg.Add(length)
+	for _, v := range files {
+		file := v
+		go func() {
+			md5, err := MD5File(file)
+			md5result := &md5result{
+				file: file,
+				md5:  md5,
+				err:  err,
+			}
+			m.Store(file, md5result)
+			wg.Done()
+		}()
+	}
+
+	var sortfiles []string
+	sortfiles = append(sortfiles, files...)
+	sort.Strings(sortfiles)
+	wg.Wait()
+
+	var sum string
+	for _, file := range sortfiles {
+		if v, ok := m.Load(file); ok {
+			result := v.(*md5result)
+			if result.err != nil {
+				return "", result.err
+			}
+			sum += result.md5
+		}
+	}
+
+	return MD5(sum), nil
+}

go.mod

@@ -0,0 +1,11 @@
+module github.com/0990/momentscleaner
+
+go 1.13
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/natefinch/lumberjack v2.0.0+incompatible
+	github.com/sirupsen/logrus v1.4.2
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.2.8 // indirect
+)

go.sum

@@ -0,0 +1,23 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

logconfig/log.go

@@ -0,0 +1,62 @@
+package logconfig
+
+import (
+	"fmt"
+	"github.com/natefinch/lumberjack"
+	"github.com/sirupsen/logrus"
+	"io"
+)
+
+func InitLogrus(name string, maxMB int) {
+	formatter := &logrus.TextFormatter{
+		DisableColors:    true,
+		DisableTimestamp: false,
+		TimestampFormat:  "2006-01-02 15:04:05",
+	}
+	logrus.SetFormatter(formatter)
+	logrus.SetLevel(logrus.DebugLevel)
+	logrus.AddHook(NewDefaultHook(name, maxMB))
+}
+
+type DefaultHook struct {
+	writers   map[logrus.Level]io.Writer
+	errWriter io.Writer
+	fmt       logrus.Formatter
+}
+
+func NewDefaultHook(name string, maxSize int) *DefaultHook {
+	formatter := &logrus.TextFormatter{
+		DisableColors:    true,
+		DisableTimestamp: false,
+	}
+
+	writers := make(map[logrus.Level]io.Writer)
+	for _, level := range logrus.AllLevels {
+		writers[level] = &lumberjack.Logger{
+			Filename:   fmt.Sprintf("%s_%s.log", name, level.String()),
+			MaxSize:    maxSize,
+			MaxAge:     100,
+			MaxBackups: 100,
+			LocalTime:  true,
+			Compress:   false,
+		}
+	}
+
+	return &DefaultHook{
+		writers: writers,
+		fmt:     formatter,
+	}
+}
+
+func (p *DefaultHook) Fire(entry *logrus.Entry) error {
+	data, err := p.fmt.Format(entry)
+	if err != nil {
+		return err
+	}
+	_, err = p.writers[entry.Level].Write(data)
+	return err
+}
+
+func (p *DefaultHook) Levels() []logrus.Level {
+	return logrus.AllLevels
+}

main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"github.com/0990/momentscleaner/cleaner"
+	"github.com/0990/momentscleaner/logconfig"
+)
+
+func main() {
+	logconfig.InitLogrus("cleaner", 10)
+	cleaner.DoClean()
+}