Skip to content

Commit

Permalink
Adds missing schemas + data for metrics to snowplow stats ping (metab…
Browse files Browse the repository at this point in the history
…ase#48476)

* adds data + schema for metrics stats ping

* remove comment

* annotate todos

* fill in the rest of the metrics values

- and add defaults

* fix some definitions + use a single timestamp

* shuffle stuff around to appease the namespace linter

- we aren't reaching into the api API, so the linter is more of a
  formality here.

* update docstring

* fix jsonschema, maybe

* review responses

- revert changes to 1-0-0
- add metrics section to new file: 1-0-1
- bump ::instance_stats to "1-0-1"
- add tags into 1-0-1
- make the code return tags (empty for now until we know what to tag things.)

- also fix test that broke from shuffling settings around

* remove a footgun

* add tags to metrics,

add grouped_metrics to jsonschema 1-0-1

format 1-0-1

* indent

* cljfmt

* version should match filename

* update instance stats 1-0-1 schema

* require `tags` in grouped_metric + snowcat comment

* fix formatting noise

* update schema to make it valid

* remove grouped_metrics from instance_stats schema

* cljfmt appeasement

* unbin cache_num_queries_cached value

- alphabetize metrics generation

* we can now guarantee metric values will be ints

* jsonschema for instance_uuid, settings, and grouped_metrics

* add analytics_uuid and make it required

* lint + alphabetize instance stats json schema

* update setting key type

- add maxLength to some strings

* lint jsonschema

- Add description to setting.items.tags
- add maxLength, and {min,max}imum to setting.items.value

* Bump instance stats to 2-0-0

- remove analytics_uuid string length == 36 check
- adds assertion to ensure required fields are set (and it passes)
- adds info for embedding settings

* adds a grouped-metric to stats ping

- adds length info to the schema to pass jsonschema linting

* cljfmt
  • Loading branch information
escherize authored Oct 15, 2024
1 parent 8e950e8 commit e5c181b
Show file tree
Hide file tree
Showing 7 changed files with 406 additions and 57 deletions.
1 change: 1 addition & 0 deletions .clj-kondo/config.edn
Original file line number Diff line number Diff line change
Expand Up @@ -591,6 +591,7 @@
metabase.driver.sql-jdbc.test-util sql-jdbc.tu
metabase.driver.sql.parameters.substitute sql.params.substitute
metabase.driver.sql.parameters.substitution sql.params.substitution
metabase.eid-translation eid-translation
metabase.email-test et
metabase.email.messages messages
metabase.formatter formatter
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
{
"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
"description": "Schema for daily stats ping, tracking instance metrics and settings",
"self": {
"vendor": "com.metabase",
"name": "instance_stats",
"format": "jsonschema",
"version": "2-0-0"
},
"type": "object",
"properties": {
"analytics_uuid": {
"description": "The UUID for the instance",
"type": "string",
"maxLength": 255
},
"features": {
"description": "Features",
"type": "array",
"items": {
"type": "object",
"description": "A single instance feature",
"properties": {
"name": {
"description": "The unique name of the feature",
"type": "string",
"maxLength": 255
},
"available": {
"description": "Whether the feature is available, i.e. can it be enabled/disabled or is it always on",
"type": "boolean"
},
"enabled": {
"description": "Whether the feature is enabled, i.e. can it be used by the users/instance",
"type": "boolean"
}
},
"required": ["name", "available", "enabled"],
"additionalProperties": true
}
},
"grouped_metrics": {
"description": "Key-value pairs of grouped metrics, with tags.",
"type": "array",
"items": {
"description": "a Grouped Metric, which has a key a value and tags",
"type": "object",
"properties": {
"name": {
"description": "The unique name of the grouped metric",
"type": "string",
"maxLength": 255
},
"values": {
"description": "Values for the grouped metric",
"type": "array",
"items": {
"type": "object",
"description": "Items in a groped metric value",
"properties": {
"group": {
"type": "string",
"description": "The group name",
"maxLength": 255
},
"value": {
"type": "number",
"description": "The value for the group",
"minimum": 0,
"maximum": 9007199254740991
}
},
"required": ["group", "value"],
"additionalProperties": false
}
},
"tags": {
"description": "Tags that can be used flagging teams / features the grouped_metric belongs to",
"type": "array",
"items": {
"description": "a single tag",
"type": "string",
"maxLength": 255
}
}
},
"required": ["name", "values", "tags"],
"additionalProperties": false
}
},
"instance_attributes": {
"description": "Key-value pairs of instance attributes",
"type": "array",
"items": {
"type": "object",
"description": "A single instance attribute",
"properties": {
"key": {
"description": "The key for this attribute",
"type": "string",
"maxLength": 255
},
"value": {
"description": "The value of this attribute",
"type": ["string", "boolean", "integer", "null"],
"maxLength": 255,
"minimum": 0,
"maximum": 2147483647
}
},
"required": ["key", "value"]
}
},
"metadata": {
"description": "Metadata about the anonymous stats collection",
"type": "array",
"items": {
"type": "object",
"description": "A single metadata key/value",
"properties": {
"key": {
"description": "The key for this metadata",
"type": "string",
"maxLength": 255
},
"value": {
"description": "The value of this metadata",
"type": ["string", "boolean", "integer", "null"],
"maxLength": 255,
"minimum": 0,
"maximum": 2147483647
}
},
"required": ["key", "value"]
}
},
"metrics": {
"description": "Key-value pairs of metrics, with tags.",
"type": "array",
"items": {
"type": "object",
"description": "A single metric attribute",
"properties": {
"name": {
"description": "The unique name of the metric",
"type": "string",
"maxLength": 255
},
"value": {
"type": "integer",
"description": "The value of the metric",
"minimum": 0,
"maximum": 2147483647
},
"tags": {
"type": "array",
"description": "Tags that can be used for flagging teams / features the metric belongs to",
"items": {
"description": "a tag",
"type": "string",
"maxLength": 255
}
}
},
"required": ["name", "value", "tags"]
}
},
"settings": {
"type": "array",
"description": "Key-value pairs of settings, with tags.",
"items": {
"type": "object",
"description": "A single setting attribute",
"properties": {
"key": {
"type": "string",
"maxLength": 255,
"description": "The unique name of the setting"
},
"value": {
"type": ["string", "boolean", "integer", "null"],
"description": "The value of this setting",
"maxLength": 255,
"minimum": 0,
"maximum": 2147483647
},
"tags": {
"type": "array",
"description": "Tags that can be used for flagging teams / features the setting belongs to",
"items": {
"description": "a tag for the setting to help categorize it",
"type": "string",
"maxLength": 255
}
}
},
"required": ["key", "value", "tags"]
}
}
},
"additionalProperties": false,
"required": [
"analytics_uuid",
"features",
"grouped_metrics",
"instance_attributes",
"metadata",
"metrics",
"settings"
]
}
10 changes: 8 additions & 2 deletions src/metabase/analytics/snowplow.clj
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,21 @@

(set! *warn-on-reflection* true)

;; Adding or updating a Snowplow schema? Make sure that the map below is updated accordingly.
;; Adding or updating a Snowplow schema? Here are some things to keep in mind:
;; - Snowplow schemata are versioned and immutable, so if you need to make changes to a schema, you should create a new
;; version of it. The version number should be updated in the `schema->version` map below.
;; - Schemas live inside the `/snowplow/iglu-client-embedded/schemas` directory.
;; - The new schema should be added to the Metabase repo via the normal pull request workflow before it is uploaded to
;; SnowcatCloud in the last step. Make sure to sanity check your schema with SnowcatCloud in the
;; #external-snowcat-cloud channel since there might be some back and forth on the format.

(def ^:private schema->version
"The most recent version for each event schema. This should be updated whenever a new version of a schema is added
to SnowcatCloud, at the same time that the data sent to the collector is updated."
{::account "1-0-1"
::browse_data "1-0-0"
::invite "1-0-1"
::instance_stats "1-0-0"
::instance_stats "2-0-0"
::csvupload "1-0-3"
::dashboard "1-1-4"
::database "1-0-1"
Expand Down
Loading

0 comments on commit e5c181b

Please sign in to comment.