Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PMM-2019 innodb compression statistics #2

Open
wants to merge 24 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
894b4cc
PMM-2019 Innodb Compression statistics
adivinho Mar 13, 2018
1683768
Update info_schema_innodb_cmp.go
adivinho Mar 22, 2018
ff49996
PMM-2019 Moved buffer from values to parameters
adivinho Apr 2, 2018
2f5d732
Update info_schema_innodb_cmp.go
adivinho Apr 2, 2018
0ab57a3
Update info_schema_innodb_cmpmem.go
adivinho Apr 2, 2018
513551c
Update .travis.yml
rnovikovP May 11, 2018
94169d5
Update .travis.yml
rnovikovP May 11, 2018
bc31896
PMM-2569: Support for MySQL 8.
arvenil May 21, 2018
99b91d0
drop master
rnovikovP May 24, 2018
9c6a638
drop matrix and PS 5.5
rnovikovP May 24, 2018
04c3694
Added MariaDB versions we support
rnovikovP May 24, 2018
4bff621
additional Mysql version
rnovikovP May 24, 2018
29ee91f
PMM-2569: Update MySQL driver to 1.4.0.
arvenil Jun 12, 2018
b660030
Merge pull request #30 from percona/PMM-2569_MySQL_8
arvenil Jun 12, 2018
cad76cb
Merge pull request #29 from percona/wip-add-additional-db
arvenil Jun 13, 2018
59ac9b1
PMM-2574: `dep`
arvenil Jun 15, 2018
aae1dc9
PMM-2574: Downgrade prometheus/common to version before introducing k…
arvenil Jun 16, 2018
f1caeee
Merge pull request #31 from percona/PMM-2574_dep
rnovikovP Jun 18, 2018
b8c928d
PMM-2019 Innodb Compression statistics
adivinho Mar 13, 2018
f7e64ff
Update info_schema_innodb_cmp.go
adivinho Mar 22, 2018
f175ec9
PMM-2019 Moved buffer from values to parameters
adivinho Apr 2, 2018
a4a2f96
Update info_schema_innodb_cmp.go
adivinho Apr 2, 2018
37b3c9a
Update info_schema_innodb_cmpmem.go
adivinho Apr 2, 2018
947400d
Merge branch 'PMM-2019-Innodb-Compression-statistics' of github.com:p…
adivinho Jun 19, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
10 changes: 9 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,22 @@ language: go
go:
- 1.9.x
- 1.10.x
- master

env:
- MYSQL_IMAGE=mysql/mysql-server:5.5
- MYSQL_IMAGE=mysql/mysql-server:5.6
- MYSQL_IMAGE=mysql/mysql-server:5.7
- MYSQL_IMAGE=mysql/mysql-server:8.0
- MYSQL_IMAGE=mariadb:5.5
- MYSQL_IMAGE=mariadb:10.0
- MYSQL_IMAGE=mariadb:10.1
- MYSQL_IMAGE=mariadb:10.2
- MYSQL_IMAGE=mariadb:10.3
- MYSQL_IMAGE=percona/percona-server:5.6
- MYSQL_IMAGE=percona/percona-server:5.7
- MYSQL_IMAGE=percona:5.5
- MYSQL_IMAGE=percona:5.6
- MYSQL_IMAGE=percona:5.7

services:
- docker
Expand Down
149 changes: 149 additions & 0 deletions Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions Gopkg.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[prune]
go-tests = true
non-go = true
unused-packages = true

[[override]]
name = "github.com/prometheus/common"
# Version before introducing kingpin.
# https://github.com/prometheus/common/pull/96
revision = "185c63bfc5a8c7a703687edb52940c895c818cb5"
9 changes: 8 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ DOCKER_IMAGE_NAME ?= mysqld-exporter
DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))


all: format build test-short
all: verify-vendor format build test-short

style:
@echo ">> checking code style"
Expand All @@ -36,6 +36,13 @@ test:
@echo ">> running tests"
@$(GO) test -race $(pkgs)

verify-vendor:
@echo ">> verify that vendor/ is in sync with code and Gopkg.*"
curl https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 -L -o ~/dep && chmod +x ~/dep
rm -fr vendor/
~/dep ensure -v -vendor-only
git diff --exit-code

format:
@echo ">> formatting code"
@$(GO) fmt $(pkgs)
Expand Down
117 changes: 117 additions & 0 deletions collector/info_schema_innodb_cmp.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Scrape `information_schema.client_statistics`.

package collector

import (
"database/sql"
"fmt"
"strings"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/log"
)

const innodbCmpQuery = `
SELECT
page_size, compress_ops, compress_ops_ok, compress_time, uncompress_ops, uncompress_time
FROM information_schema.INNODB_CMP
`

var (
// Map known innodb_cmp values to types. Unknown types will be mapped as
// untyped.
informationSchemaInnodbCmpTypes = map[string]struct {
vtype prometheus.ValueType
desc *prometheus.Desc
}{
"compress_ops": {prometheus.CounterValue,
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_compress_ops_total"),
"Number of times a B-tree page of the size PAGE_SIZE has been compressed.",
[]string{"page_size"}, nil)},
"compress_ops_ok": {prometheus.CounterValue,
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_compress_ops_ok_total"),
"Number of times a B-tree page of the size PAGE_SIZE has been successfully compressed.",
[]string{"page_size"}, nil)},
"compress_time": {prometheus.CounterValue,
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_compress_time_seconds_total"),
"Total time in seconds spent in attempts to compress B-tree pages.",
[]string{"page_size"}, nil)},
"uncompress_ops": {prometheus.CounterValue,
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_uncompress_ops_total"),
"Number of times a B-tree page has been uncompressed.",
[]string{"page_size"}, nil)},
"uncompress_time": {prometheus.CounterValue,
prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, "innodb_cmp_uncompress_time_seconds_total"),
"Total time in seconds spent in uncompressing B-tree pages.",
[]string{"page_size"}, nil)},
}
)

// ScrapeInnodbCmp collects from `information_schema.innodb_cmp`.
type ScrapeInnodbCmp struct{}

// Name of the Scraper.
func (ScrapeInnodbCmp) Name() string {
return "info_schema.innodb_cmp"
}

// Help returns additional information about Scraper.
func (ScrapeInnodbCmp) Help() string {
return "Please set next variables SET GLOBAL innodb_file_per_table=1;SET GLOBAL innodb_file_format=Barracuda;"
}

// Version of MySQL from which scraper is available.
func (ScrapeInnodbCmp) Version() float64 {
return 5.5
}

// Scrape collects data.
func (ScrapeInnodbCmp) Scrape(db *sql.DB, ch chan<- prometheus.Metric) error {
informationSchemaInnodbCmpRows, err := db.Query(innodbCmpQuery)
if err != nil {
log.Debugln("INNODB_CMP stats are not available.")
return err
}
defer informationSchemaInnodbCmpRows.Close()

// The client column is assumed to be column[0], while all other data is assumed to be coerceable to float64.
// Because of the client column, clientStatData[0] maps to columnNames[1] when reading off the metrics
// (because clientStatScanArgs is mapped as [ &client, &clientData[0], &clientData[1] ... &clientdata[n] ]
// To map metrics to names therefore we always range over columnNames[1:]
columnNames, err := informationSchemaInnodbCmpRows.Columns()
if err != nil {
log.Debugln("INNODB_CMP stats are not available.")
return err
}

var (
client string // Holds the client name, which should be in column 0.
clientStatData = make([]float64, len(columnNames)-1) // 1 less because of the client column.
clientStatScanArgs = make([]interface{}, len(columnNames))
)

clientStatScanArgs[0] = &client
for i := range clientStatData {
clientStatScanArgs[i+1] = &clientStatData[i]
}

for informationSchemaInnodbCmpRows.Next() {
if err := informationSchemaInnodbCmpRows.Scan(clientStatScanArgs...); err != nil {
return err
}

// Loop over column names, and match to scan data. Unknown columns
// will be filled with an untyped metric number. We assume other then
// cient, that we'll only get numbers.
for idx, columnName := range columnNames[1:] {
if metricType, ok := informationSchemaInnodbCmpTypes[columnName]; ok {
ch <- prometheus.MustNewConstMetric(metricType.desc, metricType.vtype, float64(clientStatData[idx]), client)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unnecessary conversion

} else {
// Unknown metric. Report as untyped.
desc := prometheus.NewDesc(prometheus.BuildFQName(namespace, informationSchema, fmt.Sprintf("innodb_cmp_%s", strings.ToLower(columnName))), fmt.Sprintf("Unsupported metric from column %s", columnName), []string{"page_size"}, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(clientStatData[idx]), client)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unnecessary conversion

}
}
}
return nil
}
50 changes: 50 additions & 0 deletions collector/info_schema_innodb_cmp_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package collector

import (
"testing"

"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"github.com/smartystreets/goconvey/convey"
"gopkg.in/DATA-DOG/go-sqlmock.v1"
)

func TestScrapeInnodbCmp(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("error opening a stub database connection: %s", err)
}
defer db.Close()

columns := []string{"page_size", "compress_ops", "compress_ops_ok", "compress_time", "uncompress_ops", "uncompress_time"}
rows := sqlmock.NewRows(columns).
AddRow("1024", 10, 20, 30, 40, 50)
mock.ExpectQuery(sanitizeQuery(innodbCmpQuery)).WillReturnRows(rows)

ch := make(chan prometheus.Metric)
go func() {
if err = (ScrapeInnodbCmp{}).Scrape(db, ch); err != nil {
t.Errorf("error calling function on test: %s", err)
}
close(ch)
}()

expected := []MetricResult{
{labels: labelMap{"page_size": "1024"}, value: 10, metricType: dto.MetricType_COUNTER},
{labels: labelMap{"page_size": "1024"}, value: 20, metricType: dto.MetricType_COUNTER},
{labels: labelMap{"page_size": "1024"}, value: 30, metricType: dto.MetricType_COUNTER},
{labels: labelMap{"page_size": "1024"}, value: 40, metricType: dto.MetricType_COUNTER},
{labels: labelMap{"page_size": "1024"}, value: 50, metricType: dto.MetricType_COUNTER},
}
convey.Convey("Metrics comparison", t, func() {
for _, expect := range expected {
got := readMetric(<-ch)
convey.So(expect, convey.ShouldResemble, got)
}
})

// Ensure all SQL queries were executed
if err := mock.ExpectationsWereMet(); err != nil {
t.Errorf("there were unfulfilled expections: %s", err)
}
}
Loading