fix: explicit milliseconds precision of timestamp columns

muety · Mar 18, 2022 · 91b4cb2 · 91b4cb2
1 parent a3acdc7
commit 91b4cb2
Show file tree

Hide file tree

Showing 6 changed files with 64 additions and 8 deletions.
diff --git a/migrations/20220318_mysql_timestamp_precision.go b/migrations/20220318_mysql_timestamp_precision.go
@@ -0,0 +1,41 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220318-mysql_timestamp_precision"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			if cfg.Db.IsMySQL() {
+				logbuch.Info("altering heartbeats table, this may take a while (up to hours)")
+
+				db.Exec("SET foreign_key_checks=0;")
+				db.Exec("SET unique_checks=0;")
+				if err := db.Exec("ALTER TABLE heartbeats MODIFY COLUMN `time` TIMESTAMP(3) NOT NULL").Error; err != nil {
+					return err
+				}
+				if err := db.Exec("ALTER TABLE heartbeats MODIFY COLUMN `created_at` TIMESTAMP(3) NOT NULL").Error; err != nil {
+					return err
+				}
+				db.Exec("SET foreign_key_checks=1;")
+				db.Exec("SET unique_checks=1;")
+
+				logbuch.Info("migrated timestamp columns to millisecond precision")
+			}
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}
diff --git a/models/heartbeat.go b/models/heartbeat.go
@@ -23,11 +23,11 @@ type Heartbeat struct {
 	OperatingSystem string     `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
 	Machine         string     `json:"machine" gorm:"index:idx_machine" hash:"ignore"`                   // ignored because wakatime api doesn't return machines currently
 	UserAgent       string     `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"`
-	Time            CustomTime `json:"time" gorm:"type:timestamp; index:idx_time,idx_time_user" swaggertype:"primitive,number"`
+	Time            CustomTime `json:"time" gorm:"type:timestamp(3); index:idx_time,idx_time_user" swaggertype:"primitive,number"`
 	Hash            string     `json:"-" gorm:"type:varchar(17); uniqueIndex"`
 	Origin          string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
 	OriginId        string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
-	CreatedAt       CustomTime `json:"created_at" gorm:"type:timestamp" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt
+	CreatedAt       CustomTime `json:"created_at" gorm:"type:timestamp(3)" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt
 }
 
 func (h *Heartbeat) Valid() bool {

diff --git a/scripts/clean_duplicates.sql b/scripts/clean_duplicates.sql
@@ -0,0 +1,12 @@
+DELETE t1
+FROM heartbeats t1
+         INNER JOIN heartbeats t2
+WHERE t1.id < t2.id
+  AND t1.time = t2.time
+  AND t1.entity = t2.entity
+  AND t1.is_write = t2.is_write
+  AND t1.branch = t2.branch
+  AND t1.editor = t2.editor
+  AND t1.machine = t2.machine
+  AND t1.operating_system = t2.operating_system
+  AND t1.user_id = t2.user_id;
diff --git a/scripts/count_duplicates_by_user.sql b/scripts/count_duplicates_by_user.sql
@@ -1,11 +1,8 @@
 SELECT s2.user_id, sum(c) as count, total, (sum(c) / total) as ratio
 FROM (
-         SELECT time,
-                user_id,
-                entity,
-                COUNT(time) as c
+         SELECT time, user_id, entity, is_write, branch, editor, machine, operating_system, COUNT(time) as c
          FROM heartbeats
-         GROUP BY time, user_id, entity
+         GROUP BY time, user_id, entity, is_write, branch, editor, machine, operating_system
          HAVING COUNT(time) > 1
      ) s2
          LEFT JOIN (SELECT user_id, count(id) AS total FROM heartbeats GROUP BY user_id) s3 ON s2.user_id = s3.user_id

diff --git a/services/duration.go b/services/duration.go
@@ -36,6 +36,8 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *
 	}
 
 	// Aggregation
+	// the below logic is approximately equivalent to the SQL query at scripts/aggregate_durations.sql,
+	// but unfortunately we cannot use it, as it features mysql-specific functions (lag(), timediff(), ...)
 	var count int
 	var latest *models.Duration
 
@@ -91,5 +93,9 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *
 		}
 	}
 
+	if len(heartbeats) == 1 && len(durations) == 1 {
+		durations[0].Duration = HeartbeatDiffThreshold
+	}
+
 	return durations.Sorted(), nil
 }
diff --git a/services/duration_test.go b/services/duration_test.go
@@ -171,7 +171,7 @@ func (suite *DurationServiceTestSuite) TestDurationService_Get() {
 	assert.Equal(suite.T(), TestEditorGoland, durations[0].Editor)
 	assert.Equal(suite.T(), TestEditorGoland, durations[1].Editor)
 	assert.Equal(suite.T(), TestEditorVscode, durations[2].Editor)
-	assert.Equal(suite.T(), 2, durations[0].NumHeartbeats)
+	assert.Equal(suite.T(), 3, durations[0].NumHeartbeats)
 	assert.Equal(suite.T(), 1, durations[1].NumHeartbeats)
 	assert.Equal(suite.T(), 3, durations[2].NumHeartbeats)
 }