Restore lost csvupload snowplow schema (metabase#40277)

ttanay · Mar 21, 2024 · b4949f7 · b4949f7
1 parent efffd58
commit b4949f7
Show file tree

Hide file tree

Showing 5 changed files with 123 additions and 26 deletions.
diff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/csvupload/jsonschema/1-0-1 b/snowplow/iglu-client-embedded/schemas/com.metabase/csvupload/jsonschema/1-0-1
@@ -1,29 +1,37 @@
 {
   "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
-  "description": "CSV upload events",
   "self": {
     "vendor": "com.metabase",
     "name": "csvupload",
     "format": "jsonschema",
     "version": "1-0-1"
   },
-  "type": "object",
+  "description": "CSV upload events",
   "properties": {
     "event": {
       "description": "Event name",
       "type": "string",
-      "enum": ["csv_upload_successful", "csv_upload_failed", "csv_append_successful", "csv_append_failed"],
+      "enum": [
+        "csv_upload_successful",
+        "csv_upload_failed"
+      ],
       "maxLength": 128
     },
     "model_id": {
       "description": "Unique identifier for the newly created model",
-      "type": ["integer", "null"],
+      "type": [
+        "integer",
+        "null"
+      ],
       "minimum": 0,
       "maximum": 2147483647
     },
     "upload_seconds": {
       "description": "Number of seconds the csv took to upload",
-      "type": ["number", "null"],
+      "type": [
+        "number",
+        "null"
+      ],
       "minimum": 0,
       "maximum": 2147483647
     },
@@ -34,7 +42,7 @@
       "maximum": 2147483647
     },
     "num_columns": {
-      "description": "Number of columns in the CSV",
+      "description": "Number of columns in the CSV file (does not include generated columns)",
       "type": "integer",
       "minimum": 0,
       "maximum": 2147483647
@@ -44,8 +52,21 @@
       "type": "integer",
       "minimum": 0,
       "maximum": 2147483647
+    },
+    "generated_columns": {
+      "description": "Number of new columns we added to the CSV (e.g., a PK)",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 2147483647
     }
   },
-  "required": ["event", "size_mb", "num_columns", "num_rows"],
-  "additionalProperties": true
+  "additionalProperties": true,
+  "type": "object",
+  "required": [
+    "event",
+    "size_mb",
+    "num_columns",
+    "num_rows",
+    "generated_columns"
+  ]
 }
diff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/csvupload/jsonschema/1-0-2 b/snowplow/iglu-client-embedded/schemas/com.metabase/csvupload/jsonschema/1-0-2
@@ -0,0 +1,73 @@
+{
+  "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+  "self": {
+    "vendor": "com.metabase",
+    "name": "csvupload",
+    "format": "jsonschema",
+    "version": "1-0-2"
+  },
+  "description": "CSV upload events",
+  "properties": {
+    "event": {
+      "description": "Event name",
+      "type": "string",
+      "enum": [
+        "csv_upload_successful",
+        "csv_upload_failed",
+        "csv_append_successful",
+        "csv_append_failed"],
+      "maxLength": 128
+    },
+    "model_id": {
+      "description": "Unique identifier for the newly created model",
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 2147483647
+    },
+    "upload_seconds": {
+      "description": "Number of seconds the csv took to upload",
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 0,
+      "maximum": 2147483647
+    },
+    "size_mb": {
+      "description": "File size of the CSV in megabytes",
+      "type": "number",
+      "minimum": 0,
+      "maximum": 2147483647
+    },
+    "num_columns": {
+      "description": "Number of columns in the CSV file (does not include generated columns)",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 2147483647
+    },
+    "num_rows": {
+      "description": "Number of rows in the CSV",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 2147483647
+    },
+    "generated_columns": {
+      "description": "Number of new columns we added to the CSV (e.g., a PK)",
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 2147483647
+    }
+  },
+  "additionalProperties": true,
+  "type": "object",
+  "required": [
+    "event",
+    "size_mb",
+    "num_columns",
+    "num_rows",
+    "generated_columns"
+  ]
+}
diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.clj
@@ -32,7 +32,7 @@
   {::account              "1-0-1"
    ::browse_data          "1-0-0"
    ::invite               "1-0-1"
-   ::csvupload            "1-0-1"
+   ::csvupload            "1-0-2"
    ::dashboard            "1-1-4"
    ::database             "1-0-1"
    ::instance             "1-1-2"

diff --git a/src/metabase/upload.clj b/src/metabase/upload.clj
@@ -536,9 +536,10 @@
   [file]
   (with-open [reader (bom/bom-reader file)]
     (let [rows (csv/read-csv reader)]
-      {:size-mb     (file-size-mb file)
-       :num-columns (count (first rows))
-       :num-rows    (count (rest rows))})))
+      {:size-mb           (file-size-mb file)
+       :num-columns       (count (first rows))
+       :num-rows          (count (rest rows))
+       :generated-columns 0})))
 
 (mu/defn create-csv-upload!
   "Main entry point for CSV uploading.

diff --git a/test/metabase/upload_test.clj b/test/metabase/upload_test.clj
@@ -1132,13 +1132,13 @@
     (snowplow-test/with-fake-snowplow-collector
       (with-upload-table! [_table (card->table (upload-example-csv!))]
         (testing "Successfully creating a CSV Upload publishes statistics to Snowplow"
-          (is (=? {:data    {"model_id"          pos?
+          (is (=? {:data    {"event"             "csv_upload_successful"
+                             "model_id"          pos?
                              "size_mb"           3.910064697265625E-5
                              "num_columns"       2
                              "num_rows"          2
                              "generated_columns" 1
-                             "upload_seconds"    pos?
-                             "event"             "csv_upload_successful"}
+                             "upload_seconds"    pos?}
                    :user-id (str (mt/user->id :rasta))}
                   (last (snowplow-test/pop-event-data-and-user-id!)))))
 
@@ -1147,10 +1147,11 @@
             (try (upload-example-csv!)
                  (catch Throwable _
                    nil))
-            (is (= {:data    {"size_mb"     3.910064697265625E-5
-                              "num_columns" 2
-                              "num_rows"    2
-                              "event"       "csv_upload_failed"}
+            (is (= {:data    {"event"             "csv_upload_failed"
+                              "size_mb"           3.910064697265625E-5
+                              "num_columns"       2
+                              "num_rows"          2
+                              "generated_columns" 0}
                     :user-id (str (mt/user->id :rasta))}
                    (last (snowplow-test/pop-event-data-and-user-id!))))))))))
 
@@ -1557,12 +1558,12 @@
                file     (csv-file-with csv-rows (mt/random-name))]
            (append-csv! {:file file, :table-id (:id table)})
 
-           (is (=? {:data    {"size_mb"           1.811981201171875E-5
+           (is (=? {:data    {"event"             "csv_append_successful"
+                              "size_mb"           1.811981201171875E-5
                               "num_columns"       1
                               "num_rows"          1
                               "generated_columns" 0
-                              "upload_seconds"    pos?
-                              "event"             "csv_append_successful"}
+                              "upload_seconds"    pos?}
                     :user-id (str (mt/user->id :crowberto))}
                    (last (snowplow-test/pop-event-data-and-user-id!))))
 
@@ -1578,10 +1579,11 @@
                (finally
                  (io/delete-file file))))
 
-           (is (= {:data    {"size_mb"     5.245208740234375E-5
-                             "num_columns" 2
-                             "num_rows"    1
-                             "event"       "csv_append_failed"}
+           (is (= {:data    {"event"             "csv_append_failed"
+                             "size_mb"           5.245208740234375E-5
+                             "num_columns"       2
+                             "num_rows"          1
+                             "generated_columns" 0}
                    :user-id (str (mt/user->id :crowberto))}
                   (last (snowplow-test/pop-event-data-and-user-id!))))))))))