Skip to content

Commit

Permalink
........S. [ZBX-24643] fixed auditlog compression by changing compres…
Browse files Browse the repository at this point in the history 
…sion policy

* commit 'c7f8cd00ceffd9e7615d55bb2e6d5d9e8027406b':
  ........S. [ZBX-24643] fixed code style issue
  ........S. [ZBX-24643] fixed code style issues
  ........S. [ZBX-24643] changed compression policy name macros
  ........S. [ZBX-24643] fixed uninitialized variable warning
  ........S. [ZBX-24643] added changelog
  ........S. [ZBX-24643] improved compression code
  ........S. [ZBX-24643] added chunk created before policy to compression
  • Loading branch information
@arseniuss-zabbix
arseniuss-zabbix committed Jul 5, 2024
2 parents 1377657 + c7f8cd0 commit 46a6214
Show file tree
Hide file tree
Showing 2 changed files with 69 additions and 34 deletions.
1 change: 1 addition & 0 deletions ChangeLog.d/bugfix/ZBX-24643
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
........S. [ZBX-24643] fixed auditlog compression by changing compression policy (askolmeisters)
102 changes: 68 additions & 34 deletions src/zabbix_server/housekeeper/history_compress.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,30 @@
#define COMPRESSION_POLICY_REMOVE "remove_compression_policy"
#define COMPRESSION_POLICY_ADD "add_compression_policy"

/* Compression policy: chunks containing data older than provided data are compressed. */
#define POLICY_COMPRESS_AFTER 0
/* Compression policy: chunks with creation time older than this cut-off point are compressed. */
#define POLICY_COMPRESS_CREATED_BEFORE 1

typedef struct
{
const char *name;
const char *segmentby;
const char *orderby;
int compression_policy;
} zbx_history_table_compression_options_t;

static zbx_history_table_compression_options_t compression_tables[] = {
{"history", "itemid", "clock,ns"},
{"history_uint", "itemid", "clock,ns"},
{"history_str", "itemid", "clock,ns"},
{"history_text", "itemid", "clock,ns"},
{"history_log", "itemid", "clock,ns"},
{"trends", "itemid", "clock"},
{"trends_uint", "itemid", "clock"},
{"auditlog", "auditid", "clock"}
{"history", "itemid", "clock,ns", POLICY_COMPRESS_AFTER},
{"history_uint", "itemid", "clock,ns", POLICY_COMPRESS_AFTER},
{"history_str", "itemid", "clock,ns", POLICY_COMPRESS_AFTER},
{"history_text", "itemid", "clock,ns", POLICY_COMPRESS_AFTER},
{"history_log", "itemid", "clock,ns", POLICY_COMPRESS_AFTER},
{"trends", "itemid", "clock", POLICY_COMPRESS_AFTER},
{"trends_uint", "itemid", "clock", POLICY_COMPRESS_AFTER},
/* Since auditlog table uses CUID from auditid field to partition table into chunks we need to use different */
/* compression policy due to internal TimescaleDB bug. */
{"auditlog", "auditid", "clock", POLICY_COMPRESS_CREATED_BEFORE}
};

static unsigned char compression_status_cache = 0;
Expand Down Expand Up @@ -93,22 +101,25 @@ static void hk_check_table_segmentation(const char *table_name, const char *segm
* *
* Purpose: returns data compression age configured for hypertable *
* *
* Parameters: table_name - [IN] hypertable name *
* Parameters: table_name - [IN] hypertable name *
* compression_policy - [IN] *
* *
* Return value: data compression age in seconds *
* *
******************************************************************************/
static int hk_get_table_compression_age(const char *table_name)
static int hk_get_compression_age(const char *table_name, int compression_policy)
{
int age = 0;
zbx_db_result_t result;
zbx_db_row_t row;
const char *field = compression_policy == POLICY_COMPRESS_AFTER ? "compress_after" :
"compress_created_before";

zabbix_log(LOG_LEVEL_DEBUG, "In %s(): table: %s", __func__, table_name);

result = zbx_db_select("select extract(epoch from (config::json->>'compress_after')::interval) from"
result = zbx_db_select("select extract(epoch from (config::json->>'%s')::interval) from"
" timescaledb_information.jobs where application_name like 'Compression%%' and"
" hypertable_schema='%s' and hypertable_name='%s'", zbx_db_get_schema_esc(), table_name);
" hypertable_schema='%s' and hypertable_name='%s'", field, zbx_db_get_schema_esc(), table_name);

if (NULL != (row = zbx_db_fetch(result)))
age = atoi(row[0]);
Expand All @@ -123,26 +134,40 @@ static int hk_get_table_compression_age(const char *table_name)
* *
* Purpose: ensures that table compression is configured to specified age *
* *
* Parameters: table_name - [IN] hypertable name *
* age - [IN] compression age *
* Parameters: table_name - [IN] hypertable name *
* age - [IN] compression age *
* compression_policy - [IN] *
* *
******************************************************************************/
static void hk_check_table_compression_age(const char *table_name, int age)
static void hk_set_table_compression_age(const char *table_name, int age, int compression_policy)
{
int compress_after;

zabbix_log(LOG_LEVEL_DEBUG, "In %s(): table: %s age %d", __func__, table_name, age);

if (age != (compress_after = hk_get_table_compression_age(table_name)))
if (age != (compress_after = hk_get_compression_age(table_name, compression_policy)))
{
zbx_db_result_t res;
zbx_db_result_t res = NULL;

if (0 != compress_after)
zbx_db_free_result(zbx_db_select("select %s('%s')", COMPRESSION_POLICY_REMOVE, table_name));

zabbix_log(LOG_LEVEL_DEBUG, "adding compression policy to table: %s age %d", table_name, age);

res = zbx_db_select("select %s('%s', integer '%d')", COMPRESSION_POLICY_ADD, table_name, age);
switch (compression_policy)
{
case POLICY_COMPRESS_AFTER:
res = zbx_db_select("select %s('%s', integer '%d')", COMPRESSION_POLICY_ADD, table_name,
age);
break;
case POLICY_COMPRESS_CREATED_BEFORE:
res = zbx_db_select("select %s('%s', compress_created_before => interval '%d')",
COMPRESSION_POLICY_ADD, table_name, age);
break;
default:
THIS_SHOULD_NEVER_HAPPEN;
break;
}

if (NULL == res)
zabbix_log(LOG_LEVEL_ERR, "failed to add compression policy to table '%s'", table_name);
Expand All @@ -155,7 +180,8 @@ static void hk_check_table_compression_age(const char *table_name, int age)

/******************************************************************************
* *
* Purpose: turns on table compression for items older than specified age *
* Purpose: turns on table compression for items or chunks older than *
* specified age *
* *
* Parameters: age - [IN] compression age *
* *
Expand All @@ -166,22 +192,29 @@ static void hk_history_enable_compression(int age)

for (size_t i = 0; i < ARRSIZE(compression_tables); i++)
{
zbx_db_result_t res;
const zbx_history_table_compression_options_t *table = &compression_tables[i];

res = zbx_db_select("select set_integer_now_func('%s', '"ZBX_TS_UNIX_NOW"', true)",
compression_tables[i].name);
if(NULL == res)
if (POLICY_COMPRESS_AFTER == table->compression_policy)
{
zabbix_log(LOG_LEVEL_WARNING, "Table \"%s\" is not a hypertable. Execute TimescaleDB"
" configuration step as described in Zabbix documentation to upgrade schema.",
compression_tables[i].name);
continue;
zbx_db_result_t res;

res = zbx_db_select("select set_integer_now_func('%s', '"ZBX_TS_UNIX_NOW"', true)",
table->name);

if (NULL == res)
{
zabbix_log(LOG_LEVEL_WARNING, "Table \"%s\" is not a hypertable. Execute TimescaleDB"
" configuration step as described in Zabbix documentation to upgrade"
" schema.", table->name);
continue;
}

zbx_db_free_result(res);
}

zbx_db_free_result(res);
hk_check_table_segmentation(compression_tables[i].name, compression_tables[i].segmentby,
compression_tables[i].orderby);
hk_check_table_compression_age(compression_tables[i].name, age);
hk_check_table_segmentation(table->name, table->segmentby, table->orderby);

hk_set_table_compression_age(table->name, age, table->compression_policy);
}

zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __func__);
Expand All @@ -198,11 +231,12 @@ static void hk_history_disable_compression(void)

for (size_t i = 0; i < ARRSIZE(compression_tables); i++)
{
if (0 == hk_get_table_compression_age(compression_tables[i].name))
const zbx_history_table_compression_options_t *table = &compression_tables[i];

if (0 == hk_get_compression_age(table->name, table->compression_policy))
continue;

zbx_db_free_result(zbx_db_select("select %s('%s')", COMPRESSION_POLICY_REMOVE,
compression_tables[i].name));
zbx_db_free_result(zbx_db_select("select %s('%s')", COMPRESSION_POLICY_REMOVE, table->name));
}

zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __func__);
Expand Down

0 comments on commit 46a6214

Please sign in to comment.