CachedMd5Comparer

X1011 · Feb 6, 2016 · 69fb273 · 69fb273
1 parent b7e45b0
commit 69fb273
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 12 deletions.
diff --git a/compare.go b/compare.go
@@ -1,27 +1,74 @@
 package main
 
 import (
-	"fmt"
     "os"
-    "io"
-    "crypto/md5"
+    "encoding/json"
 	"./drive"
 )
 
+const MinCacheFileSize = 5 * 1024 * 1024
+
 type Md5Comparer struct {}
 
 func (self Md5Comparer) Changed(local *drive.LocalFile, remote *drive.RemoteFile) bool {
     return remote.Md5() != md5sum(local.AbsPath())
 }
 
-func md5sum(path string) string {
-    h := md5.New()
+type CachedFileInfo struct {
+    Size int64 `json:"size"`
+    Modified int64 `json:"modified"`
+    Md5 string `json:"md5"`
+}
+
+func NewCachedMd5Comparer(path string) CachedMd5Comparer {
+    cache := map[string]*CachedFileInfo{}
+
     f, err := os.Open(path)
-    if err != nil {
-        return ""
+    if err == nil {
+        json.NewDecoder(f).Decode(&cache)
+    }
+    f.Close()
+    return CachedMd5Comparer{path, cache}
+}
+
+type CachedMd5Comparer struct {
+    path string
+    cache map[string]*CachedFileInfo
+}
+
+func (self CachedMd5Comparer) Changed(local *drive.LocalFile, remote *drive.RemoteFile) bool {
+    return remote.Md5() != self.md5(local)
+}
+
+func (self CachedMd5Comparer) md5(local *drive.LocalFile) string {
+    // See if file exist in cache
+    cached, found := self.cache[local.AbsPath()]
+
+    // If found and modification time and size has not changed, return cached md5
+    if found && local.Modified().UnixNano() == cached.Modified && local.Size() == cached.Size {
+        return cached.Md5
+    }
+
+    // Calculate new md5 sum
+    md5 := md5sum(local.AbsPath())
+
+    // Cache file info if file meets size criteria
+    if local.Size() > MinCacheFileSize {
+        self.cacheAdd(local, md5)
+        self.persist()
     }
-    defer f.Close()
 
-    io.Copy(h, f)
-    return fmt.Sprintf("%x", h.Sum(nil))
+    return md5
+}
+
+func (self CachedMd5Comparer) cacheAdd(lf *drive.LocalFile, md5 string) {
+    self.cache[lf.AbsPath()] = &CachedFileInfo{
+        Size: lf.Size(),
+        Modified: lf.Modified().UnixNano(),
+        Md5: md5,
+    }
+}
+
+func (self CachedMd5Comparer) persist() {
+    writeJson(self.path, self.cache)
 }
diff --git a/handlers_drive.go b/handlers_drive.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"io"
 	"io/ioutil"
+	"path/filepath"
     "./cli"
 	"./auth"
 	"./drive"
@@ -13,6 +14,7 @@ import (
 const ClientId     = "367116221053-7n0vf5akeru7on6o2fjinrecpdoe99eg.apps.googleusercontent.com"
 const ClientSecret = "1qsNodXNaWq1mQuBjUjmvhoO"
 const TokenFilename = "token_v2.json"
+const DefaultCacheFileName = "file_cache.json"
 
 
 func listHandler(ctx cli.Context) {
@@ -58,14 +60,15 @@ func downloadHandler(ctx cli.Context) {
 
 func downloadSyncHandler(ctx cli.Context) {
     args := ctx.Args()
+    cachePath := filepath.Join(args.String("configDir"), DefaultCacheFileName)
     err := newDrive(args).DownloadSync(drive.DownloadSyncArgs{
         Out: os.Stdout,
         Progress: progressWriter(args.Bool("noProgress")),
         Path: args.String("path"),
         RootId: args.String("id"),
         DryRun: args.Bool("dryRun"),
         DeleteExtraneous: args.Bool("deleteExtraneous"),
-        Comparer: Md5Comparer{},
+        Comparer: NewCachedMd5Comparer(cachePath),
     })
     checkErr(err)
 }
@@ -115,6 +118,7 @@ func uploadStdinHandler(ctx cli.Context) {
 
 func uploadSyncHandler(ctx cli.Context) {
     args := ctx.Args()
+    cachePath := filepath.Join(args.String("configDir"), DefaultCacheFileName)
     err := newDrive(args).UploadSync(drive.UploadSyncArgs{
         Out: os.Stdout,
         Progress: progressWriter(args.Bool("noProgress")),
@@ -123,7 +127,7 @@ func uploadSyncHandler(ctx cli.Context) {
         DryRun: args.Bool("dryRun"),
         DeleteExtraneous: args.Bool("deleteExtraneous"),
         ChunkSize: args.Int64("chunksize"),
-        Comparer: Md5Comparer{},
+        Comparer: NewCachedMd5Comparer(cachePath),
     })
     checkErr(err)
 }

diff --git a/util.go b/util.go
@@ -4,7 +4,10 @@ import (
     "runtime"
     "path/filepath"
     "fmt"
+    "encoding/json"
     "os"
+    "io"
+    "crypto/md5"
 )
 
 func GetDefaultConfigDir() string {
@@ -56,3 +59,32 @@ func checkErr(err error) {
         os.Exit(1)
     }
 }
+
+func writeJson(path string, data interface{}) error {
+    tmpFile := path + ".tmp"
+    f, err := os.Create(tmpFile)
+    if err != nil {
+        return err
+    }
+
+    err = json.NewEncoder(f).Encode(data)
+    f.Close()
+    if err != nil {
+        os.Remove(tmpFile)
+        return err
+    }
+
+    return os.Rename(tmpFile, path)
+}
+
+func md5sum(path string) string {
+    h := md5.New()
+    f, err := os.Open(path)
+    if err != nil {
+        return ""
+    }
+    defer f.Close()
+
+    io.Copy(h, f)
+    return fmt.Sprintf("%x", h.Sum(nil))
+}