From 0eaa0eedc547bfa6b2cb88dfaee1fcd833f2d485 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 00:56:47 +0900 Subject: [PATCH 01/44] fix ruletypes.rst typo --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ff3763712..bc583a6cc 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -938,7 +938,7 @@ Optional: ``field_value``: When set, uses the value of the field in the document and not the number of matching documents. This is useful to monitor for example a temperature sensor and raise an alarm if the temperature grows too fast. Note that the means of the field on the reference and current windows are used to determine if the ``spike_height`` value is reached. -Note also that the threshold parameters are ignored in this smode. +Note also that the threshold parameters are ignored in this mode. ``threshold_ref``: The minimum number of events that must exist in the reference window for an alert to trigger. For example, if From 9c563b26425a73d3c53ca4bb805c6a91a2318b4d Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 00:59:07 +0900 Subject: [PATCH 02/44] Fix Pagertree --- elastalert/loaders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 771194768..13e5eea83 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -77,6 +77,7 @@ class RulesLoader(object): 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, 'post': alerts.HTTPPostAlerter, + 'pagertree': alerts.PagerTreeAlerter, 'hivealerter': alerts.HiveAlerter } From c6671c59eb7a34b4265927b0eb57338f8efd1139 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 00:59:39 +0900 Subject: [PATCH 03/44] Fix LineNotify --- elastalert/loaders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 13e5eea83..a7d14c1e8 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -78,6 +78,7 @@ class RulesLoader(object): 'alerta': alerts.AlertaAlerter, 'post': alerts.HTTPPostAlerter, 'pagertree': alerts.PagerTreeAlerter, + 'linenotify': alerts.LineNotifyAlerter, 'hivealerter': alerts.HiveAlerter } From eacc861a8d2199482c1e7383b42d0eea6207e0b4 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:04:46 +0900 Subject: [PATCH 04/44] Fix docker test etc. coverage 4.5.4 to 5.5 pluggy>=0.12.0 pylint <1.4 to <2.8 pytest <3.3.0 to <3.7.0 tox <2.0 to ==3.23.0 --- Dockerfile-test | 2 ++ requirements-dev.txt | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index 3c153e644..fb8a78409 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,6 +1,8 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y +RUN apt-get install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git WORKDIR /home/elastalert diff --git a/requirements-dev.txt b/requirements-dev.txt index 558761d9e..f06659826 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,10 @@ -r requirements.txt -coverage==4.5.4 +coverage==5.5 flake8 pre-commit -pylint<1.4 -pytest<3.3.0 +pluggy>=0.12.0 +pylint<2.8 +pytest<3.7.0 setuptools sphinx_rtd_theme -tox<2.0 +tox==3.23.0 From 679a9e36b53a1f23ff10a1d4e9f5222ede500f6c Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:05:34 +0900 Subject: [PATCH 05/44] Remove Duplicate Key in Schema YAML --- elastalert/schema.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 1241315dc..9035f54e4 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -362,7 +362,6 @@ properties: alerta_origin: {type: string} # Python format string alerta_group: {type: string} # Python format string alerta_service: {type: array, items: {type: string}} # Python format string - alerta_service: {type: array, items: {type: string}} # Python format string alerta_correlate: {type: array, items: {type: string}} # Python format string alerta_tags: {type: array, items: {type: string}} # Python format string alerta_event: {type: string} # Python format string From f0a0e29c42252a8d8357543ac8cd16052a6fe30f Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:07:52 +0900 Subject: [PATCH 06/44] Fix travis-ci job error --- requirements.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 9c32052d0..17e805d46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ boto3>=1.4.4 cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 +cryptography<3.4 elasticsearch>=7.0.0 envparse>=0.2.0 exotel>=0.1.3 diff --git a/setup.py b/setup.py index 2845836a7..293e0fb41 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ 'boto3>=1.4.4', 'configparser>=3.5.0', 'croniter>=0.3.16', + 'cryptography<3.4', 'elasticsearch==7.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', From be1cc1f82dff1def3ae2c8d2e02fdc0a8328880c Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:08:42 +0900 Subject: [PATCH 07/44] Pin elasticsearch to 7.0.0 in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 17e805d46..9507f5bf6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 cryptography<3.4 -elasticsearch>=7.0.0 +elasticsearch==7.0.0 envparse>=0.2.0 exotel>=0.1.3 jira>=1.0.10,<1.0.15 From 2022b5a120185b5b04b1f3cfc5f726c7d9d34550 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:10:01 +0900 Subject: [PATCH 08/44] requirements.txt to match the jira-version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9507f5bf6..02a12fc2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ cryptography<3.4 elasticsearch==7.0.0 envparse>=0.2.0 exotel>=0.1.3 -jira>=1.0.10,<1.0.15 +jira>=2.0.0 jsonschema>=3.0.2 mock>=2.0.0 prison>=0.1.2 From b3e1f8fc37e974a85e1f6438da53cb0b4af8b921 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:10:43 +0900 Subject: [PATCH 09/44] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99acc02e7..df15315e1 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert in ```bash git clone https://github.com/bitsensor/elastalert.git; cd elastalert -docker run -d -p 3030:3030 \ +docker run -d -p 3030:3030 -p 3333:3333 \ -v `pwd`/config/elastalert.yaml:/opt/elastalert/config.yaml \ -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \ -v `pwd`/rules:/opt/elastalert/rules \ From c62f084b7b6f50582239eefda04b980f9a3484ca Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:11:37 +0900 Subject: [PATCH 10/44] add opsgenie_addr to docs --- docs/source/ruletypes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index bc583a6cc..38b01ff92 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1630,6 +1630,7 @@ Optional: ``opsgenie_account``: The OpsGenie account to integrate with. +``opsgenie_addr``: The OpsGenie URL to to connect against, default is ``https://api.opsgenie.com/v2/alerts`` ``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert. ``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients. ``opsgenie_default_recipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. From 26eee107613c4cf6da5dc7e7f3b7dbe38fd527c2 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:14:23 +0900 Subject: [PATCH 11/44] Fix Zabbix(Docs & schema.yaml) --- docs/source/ruletypes.rst | 2 +- elastalert/schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 38b01ff92..7c882e521 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2243,4 +2243,4 @@ Required: ``zbx_sender_host``: The address where zabbix server is running. ``zbx_sender_port``: The port where zabbix server is listenning. ``zbx_host``: This field setup the host in zabbix that receives the value sent by Elastalert. -``zbx_item``: This field setup the item in the host that receives the value sent by Elastalert. +``zbx_key``: This field setup the key in the host that receives the value sent by Elastalert. diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 9035f54e4..6dd03cb59 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -385,4 +385,4 @@ properties: zbx_sender_host: {type: string} zbx_sender_port: {type: integer} zbx_host: {type: string} - zbx_item: {type: string} + zbx_key: {type: string} From 95c65a5938451a4e225546b6531a8b9e3c03ae63 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:28:54 +0900 Subject: [PATCH 12/44] Fix SNS(Program & Docs) --- docs/source/elastalert.rst | 2 +- docs/source/ruletypes.rst | 47 ++++++++++++++++++++++++++++++-------- elastalert/alerts.py | 23 +++++++++++-------- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index b1008c3c4..3bd400d0c 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -35,7 +35,7 @@ Currently, we have support built in for these alert types: - Email - JIRA - OpsGenie -- SNS +- AWS SNS - HipChat - Slack - Telegram diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 7c882e521..f9c0adc48 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1658,26 +1658,55 @@ Example usage:: Environment: '$VAR' # environment variable Message: { field: message } # field in the first match -SNS -~~~ +AWS SNS +~~~~~~~ -The SNS alerter will send an SNS notification. The body of the notification is formatted the same as with other alerters. -The SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or +The AWS SNS alerter will send an AWS SNS notification. The body of the notification is formatted the same as with other alerters. +The AWS SNS alerter uses boto3 and can use credentials in the rule yaml, in a standard AWS credential and config files, or via environment variables. See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html for details. -SNS requires one option: +AWS SNS requires one option: ``sns_topic_arn``: The SNS topic's ARN. For example, ``arn:aws:sns:us-east-1:123456789:somesnstopic`` Optional: -``aws_access_key``: An access key to connect to SNS with. +``sns_aws_access_key_id``: An access key to connect to SNS with. + +``sns_aws_secret_access_key``: The secret key associated with the access key. + +``sns_aws_region``: The AWS region in which the SNS resource is located. Default is us-east-1 + +``sns_aws_profile``: The AWS profile to use. If none specified, the default will be used. + +Example When not using aws_profile usage:: -``aws_secret_key``: The secret key associated with the access key. + alert: + - sns + sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' + sns_aws_access_key_id: 'XXXXXXXXXXXXXXXXXX'' + sns_aws_secret_access_key: 'YYYYYYYYYYYYYYYYYYYY' + sns_aws_region: 'us-east-1' # You must nest aws_region within your alert configuration so it is not used to sign AWS requests. + +Example When to use aws_profile usage:: + + # Create ~/.aws/credentials + + [default] + aws_access_key_id = xxxxxxxxxxxxxxxxxxxx + aws_secret_access_key = yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy -``aws_region``: The AWS region in which the SNS resource is located. Default is us-east-1 + # Create ~/.aws/config -``profile``: The AWS profile to use. If none specified, the default will be used. + [default] + region = us-east-1 + + # alert rule setting + + alert: + - sns + sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' + sns_aws_profile: 'default' HipChat ~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index f2f31853f..96eb5c936 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -937,11 +937,11 @@ class SnsAlerter(Alerter): def __init__(self, *args): super(SnsAlerter, self).__init__(*args) self.sns_topic_arn = self.rule.get('sns_topic_arn', '') - self.aws_access_key_id = self.rule.get('aws_access_key_id') - self.aws_secret_access_key = self.rule.get('aws_secret_access_key') - self.aws_region = self.rule.get('aws_region', 'us-east-1') + self.sns_aws_access_key_id = self.rule.get('sns_aws_access_key_id') + self.sns_aws_secret_access_key = self.rule.get('sns_aws_secret_access_key') + self.sns_aws_region = self.rule.get('sns_aws_region', 'us-east-1') self.profile = self.rule.get('boto_profile', None) # Deprecated - self.profile = self.rule.get('aws_profile', None) + self.profile = self.rule.get('sns_aws_profile', None) def create_default_title(self, matches): subject = 'ElastAlert: %s' % (self.rule['name']) @@ -950,12 +950,15 @@ def create_default_title(self, matches): def alert(self, matches): body = self.create_alert_body(matches) - session = boto3.Session( - aws_access_key_id=self.aws_access_key_id, - aws_secret_access_key=self.aws_secret_access_key, - region_name=self.aws_region, - profile_name=self.profile - ) + if self.profile is None: + session = boto3.Session( + aws_access_key_id=self.sns_aws_access_key_id, + aws_secret_access_key=self.sns_aws_access_key_id, + region_name=self.sns_aws_region + ) + else: + session = boto3.Session(profile_name=self.profile) + sns_client = session.client('sns') sns_client.publish( TopicArn=self.sns_topic_arn, From e03cb5c32a3bcc754a6d3189cf704d09ec378786 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:31:10 +0900 Subject: [PATCH 13/44] Fix opsgenie_default_recipients --- elastalert/opsgenie.py | 2 +- tests/alerts_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index bcdaf2d05..b65e4718b 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -19,7 +19,7 @@ def __init__(self, *args): super(OpsGenieAlerter, self).__init__(*args) self.account = self.rule.get('opsgenie_account') self.api_key = self.rule.get('opsgenie_key', 'key') - self.default_reciepients = self.rule.get('opsgenie_default_receipients', None) + self.default_reciepients = self.rule.get('opsgenie_default_recipients', None) self.recipients = self.rule.get('opsgenie_recipients') self.recipients_args = self.rule.get('opsgenie_recipients_args') self.default_teams = self.rule.get('opsgenie_default_teams', None) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 5cd61ae75..7250234a5 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -445,7 +445,7 @@ def test_opsgenie_default_alert_routing(): 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}], 'alert': 'opsgenie', 'opsgenie_teams': ['{TEAM_PREFIX}-Team'], - 'opsgenie_default_receipients': ["devops@test.com"], 'opsgenie_default_teams': ["Test"] + 'opsgenie_default_recipients': ["devops@test.com"], 'opsgenie_default_teams': ["Test"] } with mock.patch('requests.post'): From 04e2c528b53b64d942a8f56510f83e380f512f86 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:32:17 +0900 Subject: [PATCH 14/44] added docs opsgenie_proxy --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index f9c0adc48..4bad77a63 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1651,6 +1651,8 @@ Optional: ``opsgenie_details``: Map of custom key/value pairs to include in the alert's details. The value can sourced from either fields in the first match, environment variables, or a constant value. +``opsgenie_proxy``: By default ElastAlert will not use a network proxy to send notifications to OpsGenie. Set this option using ``hostname:port`` if you need to use a proxy. + Example usage:: opsgenie_details: From 595e02a437b77fa485eb565942c6a9afbcf9c2ba Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:33:55 +0900 Subject: [PATCH 15/44] added docs pagertree_proxy --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 4bad77a63..61080bbb1 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1956,6 +1956,8 @@ The alerter requires the following options: ``pagertree_integration_url``: URL generated by PagerTree for the integration. +``pagertree_proxy``: By default ElastAlert will not use a network proxy to send notifications to PagerTree. Set this option using ``hostname:port`` if you need to use a proxy. + Exotel ~~~~~~ From 8cb061104152d0dcbb9462935a43e8357e0ac9d5 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:35:05 +0900 Subject: [PATCH 16/44] added docs telegram telegram_proxy_login telegram_proxy_pass --- docs/source/ruletypes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 61080bbb1..44406c219 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1874,6 +1874,10 @@ Optional: ``telegram_proxy``: By default ElastAlert will not use a network proxy to send notifications to Telegram. Set this option using ``hostname:port`` if you need to use a proxy. +``telegram_proxy_login``: The Telegram proxy auth username. + +``telegram_proxy_pass``: The Telegram proxy auth password. + GoogleChat ~~~~~~~~~~ GoogleChat alerter will send a notification to a predefined GoogleChat channel. The body of the notification is formatted the same as with other alerters. From 297e9666fc9ca531d2670346f14abba4519672aa Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:35:40 +0900 Subject: [PATCH 17/44] added docs slack_ca_certs --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 44406c219..a42d9e310 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1828,6 +1828,8 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. +``slack_ca_certs``: path to a CA cert bundle to use to verify SSL connections. + Mattermost ~~~~~~~~~~ From 088d6374781bfef1463e1c57a06f2724521906db Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:36:20 +0900 Subject: [PATCH 18/44] added docs slack_ignore_ssl_errors --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a42d9e310..5d8a1807d 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1816,6 +1816,8 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. +``slack_ignore_ssl_errors``: By default ElastAlert will verify SSL certificate. Set this option to False if you want to ignore SSL errors. + ``slack_title``: Sets a title for the message, this shows up as a blue text at the start of the message ``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Requires slack_title to be set. From 33efb23933760d0f3030fa78fa82896cbdbe0d60 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:38:36 +0900 Subject: [PATCH 19/44] Change Library blist to sortedcontainers --- elastalert/ruletypes.py | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 2f1d2f82c..7a889e80a 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -3,7 +3,7 @@ import datetime import sys -from blist import sortedlist +from sortedcontainers import SortedKeyList as sortedlist from .util import add_raw_postfix from .util import dt_to_ts diff --git a/requirements.txt b/requirements.txt index 02a12fc2e..2ddb27cf6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ apscheduler>=3.3.0 aws-requests-auth>=0.3.0 -blist>=1.3.6 boto3>=1.4.4 cffi>=1.11.5 configparser>=3.5.0 @@ -18,6 +17,7 @@ PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 PyYAML>=5.1 requests>=2.0.0 +sortedcontainers>=2.2.2 stomp.py>=4.1.17 texttable>=0.8.8 twilio==6.0.0 diff --git a/setup.py b/setup.py index 293e0fb41..bd87fff3d 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ install_requires=[ 'apscheduler>=3.3.0', 'aws-requests-auth>=0.3.0', - 'blist>=1.3.6', 'boto3>=1.4.4', 'configparser>=3.5.0', 'croniter>=0.3.16', @@ -45,6 +44,7 @@ 'python-dateutil>=2.6.0,<2.7.0', 'PyYAML>=3.12', 'requests>=2.10.0', + 'sortedcontainers>=2.2.2', 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', From 9aaa9c95537719b9f4bf3c6635f84b7018e6aef2 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:40:15 +0900 Subject: [PATCH 20/44] Fix docs stomp --- docs/source/ruletypes.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 5d8a1807d..ce025d82e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2085,12 +2085,20 @@ Stomp This alert type will use the STOMP protocol in order to push a message to a broker like ActiveMQ or RabbitMQ. The message body is a JSON string containing the alert details. The default values will work with a pristine ActiveMQ installation. -Optional: +The alerter requires the following options: ``stomp_hostname``: The STOMP host to use, defaults to localhost. + ``stomp_hostport``: The STOMP port to use, defaults to 61613. + ``stomp_login``: The STOMP login to use, defaults to admin. + ``stomp_password``: The STOMP password to use, defaults to admin. + +Optional: + +``stomp_ssl``: Connect the STOMP host using TLS, defaults to False. + ``stomp_destination``: The STOMP destination to use, defaults to /queue/ALERT The stomp_destination field depends on the broker, the /queue/ALERT example is the nomenclature used by ActiveMQ. Each broker has its own logic. From 76ff2511740d5bf89572ff5426ae5876536b5341 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:40:48 +0900 Subject: [PATCH 21/44] Fix Stomp --- elastalert/alerts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 96eb5c936..3b21d1590 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -371,7 +371,6 @@ def alert(self, matches): conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)], use_ssl=self.stomp_ssl) - conn.start() conn.connect(self.stomp_login, self.stomp_password) # Ensures that the CONNECTED frame is received otherwise, the disconnect call will fail. time.sleep(1) From 63a968d39d4410ed82256e6768f7083b84892de1 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:42:08 +0900 Subject: [PATCH 22/44] added docs alerta_api_skip_ssl --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ce025d82e..c75c66eb8 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2123,6 +2123,8 @@ Optional: ``alerta_use_match_timestamp``: If true, it will use the timestamp of the first match as the ``createTime`` of the alert. otherwise, the current server time is used. +``alerta_api_skip_ssl``: Defaults to False. + ``alert_missing_value``: Text to replace any match field not found when formating strings. Defaults to ````. The following options dictate the values of the API JSON payload: From 0a08104a386fd720ba61d99457d1f952eb8708d7 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:42:42 +0900 Subject: [PATCH 23/44] added docs hive_verify --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index c75c66eb8..f85db6aae 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2253,6 +2253,8 @@ Optional: ``hive_proxies``: Proxy configuration. +``hive_verify``: Wether or not to enable SSL certificate validation. Defaults to False. + ``hive_observable_data_mapping``: If needed, matched data fields can be mapped to TheHive observable types using python string formatting. Example usage:: From a8eed460543913ac7d0b58b1dfa5cc82ec3e85ad Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:44:45 +0900 Subject: [PATCH 24/44] Fix elasticsearch-py test_rule.py error --- elastalert/test_rule.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 06100aa0f..af1eaa497 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -83,7 +83,7 @@ def test_file(self, conf, args): # Get one document for schema try: - res = es_client.search(index, size=1, body=query, ignore_unavailable=True) + res = es_client.search(index=index, size=1, body=query, ignore_unavailable=True) except Exception as e: print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) @@ -109,7 +109,7 @@ def test_file(self, conf, args): five=conf['five'] ) try: - res = es_client.count(index, doc_type=doc_type, body=count_query, ignore_unavailable=True) + res = es_client.count(index=index, doc_type=doc_type, body=count_query, ignore_unavailable=True) except Exception as e: print("Error querying Elasticsearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) @@ -153,7 +153,7 @@ def test_file(self, conf, args): # Download up to max_query_size (defaults to 10,000) documents to save if (args.save or args.formatted_output) and not args.count: try: - res = es_client.search(index, size=args.max_query_size, body=query, ignore_unavailable=True) + res = es_client.search(index=index, size=args.max_query_size, body=query, ignore_unavailable=True) except Exception as e: print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) From 0a60564b4d0180efbe9a707aa124c57e4bb9ed7a Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:45:27 +0900 Subject: [PATCH 25/44] Typo in examples_rules/ssh.yaml --- example_rules/ssh.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_rules/ssh.yaml b/example_rules/ssh.yaml index 7af890784..a7147217b 100644 --- a/example_rules/ssh.yaml +++ b/example_rules/ssh.yaml @@ -1,5 +1,5 @@ # Rule name, must be unique - name: SSH abuse (ElastAlert 3.0.1) - 2 +name: SSH abuse (ElastAlert 3.0.1) - 2 # Alert on x events in y seconds type: frequency From e4225ac322a8eae2b86f595c140543f864b6ff3f Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:48:15 +0900 Subject: [PATCH 26/44] Sync requirements.txt and setup.py --- requirements.txt | 4 ++-- setup.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2ddb27cf6..a94e386f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,8 +16,8 @@ py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 PyYAML>=5.1 -requests>=2.0.0 +requests>=2.10.0 sortedcontainers>=2.2.2 stomp.py>=4.1.17 texttable>=0.8.8 -twilio==6.0.0 +twilio>=6.0.0,<6.1 diff --git a/setup.py b/setup.py index bd87fff3d..e22efdee1 100644 --- a/setup.py +++ b/setup.py @@ -40,9 +40,10 @@ 'jsonschema>=3.0.2', 'mock>=2.0.0', 'prison>=0.1.2', + 'py-zabbix==1.1.3', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', - 'PyYAML>=3.12', + 'PyYAML>=5.1', 'requests>=2.10.0', 'sortedcontainers>=2.2.2', 'stomp.py>=4.1.17', From 7039bdd04fec85262db222b88455efd61df91b0c Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:50:03 +0900 Subject: [PATCH 27/44] fix docs slack_timeout --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index f85db6aae..59a23e7fc 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1822,7 +1822,7 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Requires slack_title to be set. -``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. +``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slack. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. ``slack_attach_kibana_discover_url``: Enables the attachment of the ``kibana_discover_url`` to the slack notification. The config ``generate_kibana_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. From 8fc961cb34d3c5dbbd897bb3f1d26de38801f958 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:50:35 +0900 Subject: [PATCH 28/44] added docs jira_assignee --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 59a23e7fc..9b0a9a1fc 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1528,6 +1528,8 @@ For an example JIRA account file, see ``example_rules/jira_acct.yaml``. The acco Optional: +``jira_assignee``: Assigns an issue to a user. + ``jira_component``: The name of the component or components to set the ticket to. This can be a single string or a list of strings. This is provided for backwards compatibility and will eventually be deprecated. It is preferable to use the plural ``jira_components`` instead. ``jira_components``: The name of the component or components to set the ticket to. This can be a single string or a list of strings. From b3db65e7018db0cd82dd75736d66024c21586495 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:51:41 +0900 Subject: [PATCH 29/44] added docs slack slack_parse_override slack_text_string --- docs/source/ruletypes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 9b0a9a1fc..b6be1c978 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1814,6 +1814,10 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_msg_color``: By default the alert will be posted with the 'danger' color. You can also use 'good' or 'warning' colors. +``slack_parse_override``: By default the notification message is escaped 'none'. You can also use 'full'. + +``slack_text_string``: Notification message you want to add. + ``slack_proxy``: By default ElastAlert will not use a network proxy to send notifications to Slack. Set this option using ``hostname:port`` if you need to use a proxy. ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. From 92cda321a4590adbac0bea01bb1fe4003e567232 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 01:53:01 +0900 Subject: [PATCH 30/44] remove alerta_new_style_string_format --- elastalert/schema.yaml | 1 - tests/alerts_test.py | 1 - 2 files changed, 2 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 6dd03cb59..120faa964 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -371,7 +371,6 @@ properties: alerta_value: {type: string} # Python format string alerta_attributes_keys: {type: array, items: {type: string}} alerta_attributes_values: {type: array, items: {type: string}} # Python format string - alerta_new_style_string_format: {type: boolean} ### Simple diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 7250234a5..f886bb44d 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -2534,7 +2534,6 @@ def test_alerta_new_style(ea): 'alerta_severity': "debug", 'alerta_text': "Probe {hostname} is UP at {logdate} GMT", 'alerta_value': "UP", - 'alerta_new_style_string_format': True, 'type': 'any', 'alerta_use_match_timestamp': True, 'alert': 'alerta' From 0a59e08098156bfe5935d32d812aaaf2045834e1 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 02:00:45 +0900 Subject: [PATCH 31/44] Remove new_style_string_format --- elastalert/alerts.py | 4 ---- tests/alerts_test.py | 26 -------------------------- 2 files changed, 30 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 3b21d1590..d8fd93618 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -897,10 +897,6 @@ def __init__(self, *args): logging.warning('Warning! You could be vulnerable to shell injection!') self.rule['command'] = [self.rule['command']] - self.new_style_string_format = False - if 'new_style_string_format' in self.rule and self.rule['new_style_string_format']: - self.new_style_string_format = True - def alert(self, matches): # Format the command and arguments try: diff --git a/tests/alerts_test.py b/tests/alerts_test.py index f886bb44d..cbe6e2b36 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1072,32 +1072,6 @@ def test_command(): alert.alert([match]) assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True) - # Test command as string with formatted arg (new-style string format) - rule = {'command': '/bin/test/ --arg {match[somefield]}', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test --arg foobarbaz', stdin=subprocess.PIPE, shell=False) - - rule = {'command': '/bin/test/ --arg {match[nested][field]}', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test --arg 1', stdin=subprocess.PIPE, shell=False) - - # Test command as string without formatted arg (new-style string format) - rule = {'command': '/bin/test/foo.sh', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test/foo.sh', stdin=subprocess.PIPE, shell=True) - - rule = {'command': '/bin/test/foo.sh {{bar}}', 'new_style_string_format': True} - alert = CommandAlerter(rule) - with mock.patch("elastalert.alerts.subprocess.Popen") as mock_popen: - alert.alert([match]) - assert mock_popen.called_with('/bin/test/foo.sh {bar}', stdin=subprocess.PIPE, shell=True) - # Test command with pipe_match_json rule = {'command': ['/bin/test/', '--arg', '%(somefield)s'], 'pipe_match_json': True} From 3d8e95e172aaad13efe5474c48fcc92650213723 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 02:04:33 +0900 Subject: [PATCH 32/44] Fix docs external link URL --- docs/source/ruletypes.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index b6be1c978..1d99ee2e2 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1622,7 +1622,7 @@ OpsGenie alerter will create an alert which can be used to notify Operations peo integration must be created in order to acquire the necessary ``opsgenie_key`` rule variable. Currently the OpsGenieAlerter only creates an alert, however it could be extended to update or close existing alerts. -It is necessary for the user to create an OpsGenie Rest HTTPS API `integration page `_ in order to create alerts. +It is necessary for the user to create an OpsGenie Rest HTTPS API `integration page `_ in order to create alerts. The OpsGenie alert requires one option: @@ -1874,7 +1874,7 @@ Telegram alerter will send a notification to a predefined Telegram username or c The alerter requires the following two options: -``telegram_bot_token``: The token is a string along the lines of ``110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw`` that will be required to authorize the bot and send requests to the Bot API. You can learn about obtaining tokens and generating new ones in this document https://core.telegram.org/bots#botfather +``telegram_bot_token``: The token is a string along the lines of ``110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw`` that will be required to authorize the bot and send requests to the Bot API. You can learn about obtaining tokens and generating new ones in this document https://core.telegram.org/bots#6-botfather ``telegram_room_id``: Unique identifier for the target chat or username of the target channel using telegram chat_id (in the format "-xxxxxxxx") @@ -1939,7 +1939,7 @@ V2 API Options (Optional): These options are specific to the PagerDuty V2 API -See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 +See https://developer.pagerduty.com/docs/events-api-v2/trigger-events/ ``pagerduty_api_version``: Defaults to `v1`. Set to `v2` to enable the PagerDuty V2 Event API. @@ -1983,7 +1983,7 @@ The alerter requires the following option: ``exotel_auth_token``: Auth token assosiated with your Exotel account. -If you don't know how to find your accound sid and auth token, refer - http://support.exotel.in/support/solutions/articles/3000023019-how-to-find-my-exotel-token-and-exotel-sid- +If you don't know how to find your accound sid and auth token, refer - https://support.exotel.com/support/solutions/articles/3000023019-how-to-find-my-exotel-token-and-exotel-sid ``exotel_to_number``: The phone number where you would like send the notification. @@ -2054,7 +2054,7 @@ The ServiceNow alerter will create a ne Incident in ServiceNow. The body of the The alerter requires the following options: -``servicenow_rest_url``: The ServiceNow RestApi url, this will look like https://instancename.service-now.com/api/now/v1/table/incident +``servicenow_rest_url``: The ServiceNow RestApi url, this will look like https://developer.servicenow.com/dev.do#!/reference/api/orlando/rest/c_TableAPI#r_TableAPI-POST ``username``: The ServiceNow Username to access the api. @@ -2113,7 +2113,7 @@ Alerta ~~~~~~ Alerta alerter will post an alert in the Alerta server instance through the alert API endpoint. -See http://alerta.readthedocs.io/en/latest/api/alert.html for more details on the Alerta JSON format. +See https://docs.alerta.io/en/latest/api/alert.html for more details on the Alerta JSON format. For Alerta 5.0 From 0f0ed8174954b0cdec7a7c1fc1281310f95d6b95 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 02:10:39 +0900 Subject: [PATCH 33/44] Add tzlocal<3.0 --- requirements.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index a94e386f8..b05c38c50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ sortedcontainers>=2.2.2 stomp.py>=4.1.17 texttable>=0.8.8 twilio>=6.0.0,<6.1 +tzlocal<3.0 diff --git a/setup.py b/setup.py index e22efdee1..6a60fc630 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', + 'tzlocal<3.0', 'cffi>=1.11.5' ] ) From 7df8e96b84a610a3f84c5970172a804d9935ac5e Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 03:05:05 +0900 Subject: [PATCH 34/44] Fix Logging to elastalert_logger --- elastalert/alerts.py | 15 +++++++-------- elastalert/elastalert.py | 32 ++++++++++++++++---------------- elastalert/kibana_discover.py | 9 +++++---- elastalert/loaders.py | 14 +++++++------- elastalert/opsgenie.py | 15 +++++++-------- elastalert/util.py | 6 +++--- 6 files changed, 45 insertions(+), 46 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d8fd93618..64a339197 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2,7 +2,6 @@ import copy import datetime import json -import logging import os import re import subprocess @@ -585,7 +584,7 @@ def __init__(self, rule): msg = '%s Both have common statuses of (%s). As such, no tickets will ever be found.' % ( msg, ','.join(intersection)) msg += ' This should be simplified to use only one or the other.' - logging.warning(msg) + elastalert_logger.warning(msg) self.reset_jira_args() @@ -605,7 +604,7 @@ def set_priority(self): if self.priority is not None and self.client is not None: self.jira_args['priority'] = {'id': self.priority_ids[self.priority]} except KeyError: - logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) + elastalert_logger.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) def reset_jira_args(self): self.jira_args = {'project': {'key': self.project}, @@ -748,7 +747,7 @@ def find_existing_ticket(self, matches): try: issues = self.client.search_issues(jql) except JIRAError as e: - logging.exception("Error while searching for JIRA ticket using jql '%s': %s" % (jql, e)) + elastalert_logger.exception("Error while searching for JIRA ticket using jql '%s': %s" % (jql, e)) return None if len(issues): @@ -791,19 +790,19 @@ def alert(self, matches): try: self.comment_on_ticket(ticket, match) except JIRAError as e: - logging.exception("Error while commenting on ticket %s: %s" % (ticket, e)) + elastalert_logger.exception("Error while commenting on ticket %s: %s" % (ticket, e)) if self.labels: for label in self.labels: try: ticket.fields.labels.append(label) except JIRAError as e: - logging.exception("Error while appending labels to ticket %s: %s" % (ticket, e)) + elastalert_logger.exception("Error while appending labels to ticket %s: %s" % (ticket, e)) if self.transition: elastalert_logger.info('Transitioning existing ticket %s' % (ticket.key)) try: self.transition_ticket(ticket) except JIRAError as e: - logging.exception("Error while transitioning ticket %s: %s" % (ticket, e)) + elastalert_logger.exception("Error while transitioning ticket %s: %s" % (ticket, e)) if self.pipeline is not None: self.pipeline['jira_ticket'] = ticket @@ -894,7 +893,7 @@ def __init__(self, *args): if isinstance(self.rule['command'], str): self.shell = True if '%' in self.rule['command']: - logging.warning('Warning! You could be vulnerable to shell injection!') + elastalert_logger.warning('Warning! You could be vulnerable to shell injection!') self.rule['command'] = [self.rule['command']] def alert(self, matches): diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index b078c86db..b288dac5b 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -401,7 +401,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): # Different versions of ES have this formatted in different ways. Fallback to str-ing the whole thing raise ElasticsearchException(str(res['_shards']['failures'])) - logging.debug(str(res)) + elastalert_logger.debug(str(res)) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -844,7 +844,7 @@ def enhance_filter(self, rule): filters.append(query_str_filter) else: filters.append({'query': query_str_filter}) - logging.debug("Enhanced filter with {} terms: {}".format(listname, str(query_str_filter))) + elastalert_logger.debug("Enhanced filter with {} terms: {}".format(listname, str(query_str_filter))) def run_rule(self, rule, endtime, starttime=None): """ Run a rule for a given time period, including querying and alerting on results. @@ -873,7 +873,7 @@ def run_rule(self, rule, endtime, starttime=None): # Don't run if starttime was set to the future if ts_now() <= rule['starttime']: - logging.warning("Attempted to use query start time in the future (%s), sleeping instead" % (starttime)) + elastalert_logger.warning("Attempted to use query start time in the future (%s), sleeping instead" % (starttime)) return 0 # Run the rule. If querying over a large time period, split it up into segments @@ -1082,7 +1082,7 @@ def load_rule_changes(self): try: new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if not new_rule: - logging.error('Invalid rule file skipped: %s' % rule_file) + elastalert_logger.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) @@ -1122,7 +1122,7 @@ def load_rule_changes(self): try: new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if not new_rule: - logging.error('Invalid rule file skipped: %s' % rule_file) + elastalert_logger.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue @@ -1205,12 +1205,12 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): time.sleep(1.0) if self.writeback_es.ping(): - logging.error( + elastalert_logger.error( 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', self.writeback_alias, ) else: - logging.error( + elastalert_logger.error( 'Could not reach ElasticSearch at "%s:%d".', self.conf['es_host'], self.conf['es_port'], @@ -1285,7 +1285,7 @@ def handle_rule_execution(self, rule): # We were processing for longer than our refresh interval # This can happen if --start was specified with a large time period # or if we are running too slow to process events in real time. - logging.warning( + elastalert_logger.warning( "Querying from %s to %s took longer than %s!" % ( old_starttime, pretty_ts(endtime, rule.get('use_local_time')), @@ -1618,7 +1618,7 @@ def writeback(self, doc_type, body, rule=None, match_body=None): res = self.writeback_es.index(index=index, doc_type=doc_type, body=body) return res except ElasticsearchException as e: - logging.exception("Error writing alert info to Elasticsearch: %s" % (e)) + elastalert_logger.exception("Error writing alert info to Elasticsearch: %s" % (e)) def find_recent_pending_alerts(self, time_limit): """ Queries writeback_es to find alerts that did not send @@ -1646,7 +1646,7 @@ def find_recent_pending_alerts(self, time_limit): if res['hits']['hits']: return res['hits']['hits'] except ElasticsearchException as e: - logging.exception("Error finding recent pending alerts: %s %s" % (e, query)) + elastalert_logger.exception("Error finding recent pending alerts: %s %s" % (e, query)) return [] def send_pending_alerts(self): @@ -1846,11 +1846,11 @@ def add_aggregated_alert(self, match, rule): def silence(self, silence_cache_key=None): """ Silence an alert for a period of time. --silence and --rule must be passed as args. """ if self.debug: - logging.error('--silence not compatible with --debug') + elastalert_logger.error('--silence not compatible with --debug') exit(1) if not self.args.rule: - logging.error('--silence must be used with --rule') + elastalert_logger.error('--silence must be used with --rule') exit(1) # With --rule, self.rules will only contain that specific rule @@ -1860,11 +1860,11 @@ def silence(self, silence_cache_key=None): try: silence_ts = parse_deadline(self.args.silence) except (ValueError, TypeError): - logging.error('%s is not a valid time period' % (self.args.silence)) + elastalert_logger.error('%s is not a valid time period' % (self.args.silence)) exit(1) if not self.set_realert(silence_cache_key, silence_ts, 0): - logging.error('Failed to save silence command to Elasticsearch') + elastalert_logger.error('Failed to save silence command to Elasticsearch') exit(1) elastalert_logger.info('Success. %s will be silenced until %s' % (silence_cache_key, silence_ts)) @@ -1925,7 +1925,7 @@ def is_silenced(self, rule_name): def handle_error(self, message, data=None): ''' Logs message at error level and writes message, data and traceback to Elasticsearch. ''' - logging.error(message) + elastalert_logger.error(message) body = {'message': message} tb = traceback.format_exc() body['traceback'] = tb.strip().split('\n') @@ -1935,7 +1935,7 @@ def handle_error(self, message, data=None): def handle_uncaught_exception(self, exception, rule): """ Disables a rule and sends a notification. """ - logging.error(traceback.format_exc()) + elastalert_logger.error(traceback.format_exc()) self.handle_error('Uncaught exception running rule %s: %s' % (rule['name'], exception), {'rule': rule['name']}) if self.disable_rules_on_error: self.rules = [running_rule for running_rule in self.rules if running_rule['name'] != rule['name']] diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 7e4dbb5d1..b36baaac3 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -8,6 +8,7 @@ import urllib.parse from .util import EAException +from .util import elastalert_logger from .util import lookup_es_key from .util import ts_add @@ -21,7 +22,7 @@ def generate_kibana_discover_url(rule, match): discover_app_url = rule.get('kibana_discover_app_url') if not discover_app_url: - logging.warning( + elastalert_logger.warning( 'Missing kibana_discover_app_url for rule %s' % ( rule.get('name', '') ) @@ -30,7 +31,7 @@ def generate_kibana_discover_url(rule, match): kibana_version = rule.get('kibana_discover_version') if not kibana_version: - logging.warning( + elastalert_logger.warning( 'Missing kibana_discover_version for rule %s' % ( rule.get('name', '') ) @@ -39,7 +40,7 @@ def generate_kibana_discover_url(rule, match): index = rule.get('kibana_discover_index_pattern_id') if not index: - logging.warning( + elastalert_logger.warning( 'Missing kibana_discover_index_pattern_id for rule %s' % ( rule.get('name', '') ) @@ -70,7 +71,7 @@ def generate_kibana_discover_url(rule, match): appState = kibana_discover_app_state(index, columns, filters, query_keys, match) else: - logging.warning( + elastalert_logger.warning( 'Unknown kibana discover application version %s for rule %s' % ( kibana_version, rule.get('name', '') diff --git a/elastalert/loaders.py b/elastalert/loaders.py index a7d14c1e8..fcfb5a92f 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -2,7 +2,6 @@ import copy import datetime import hashlib -import logging import os import sys @@ -20,6 +19,7 @@ from .util import dt_to_unix from .util import dt_to_unixms from .util import EAException +from .util import elastalert_logger from .util import get_module from .util import ts_to_dt from .util import ts_to_dt_with_format @@ -117,7 +117,7 @@ def load(self, conf, args=None): rule = self.load_configuration(rule_file, conf, args) # A rule failed to load, don't try to process it if not rule: - logging.error('Invalid rule file skipped: %s' % rule_file) + elastalert_logger.error('Invalid rule file skipped: %s' % rule_file) continue # By setting "is_enabled: False" in rule file, a rule is easily disabled if 'is_enabled' in rule and not rule['is_enabled']: @@ -395,10 +395,10 @@ def _dt_to_ts_with_format(dt): if rule.get('use_strftime_index'): for token in ['%y', '%M', '%D']: if token in rule.get('index'): - logging.warning('Did you mean to use %s in the index? ' - 'The index will be formatted like %s' % (token, - datetime.datetime.now().strftime( - rule.get('index')))) + elastalert_logger.warning('Did you mean to use %s in the index? ' + 'The index will be formatted like %s' % (token, + datetime.datetime.now().strftime( + rule.get('index')))) if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') @@ -487,7 +487,7 @@ def adjust_deprecated_values(rule): rule['http_post_proxy'] = rule['simple_proxy'] if 'simple_webhook_url' in rule: rule['http_post_url'] = rule['simple_webhook_url'] - logging.warning( + elastalert_logger.warning( '"simple" alerter has been renamed "post" and comptability may be removed in a future release.') diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index b65e4718b..e9ff574f8 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import json -import logging import os.path import requests @@ -46,11 +45,11 @@ def _parse_responders(self, responders, responder_args, matches, default_respond try: formated_responders.append(responder.format(**responders_values)) except KeyError as error: - logging.warn("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. " % (error)) + elastalert_logger.warning("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. " % (error)) if not formated_responders: - logging.warn("OpsGenieAlerter: no responders can be formed. Trying the default responder ") + elastalert_logger.warning("OpsGenieAlerter: no responders can be formed. Trying the default responder ") if not default_responders: - logging.warn("OpsGenieAlerter: default responder not set. Falling back") + elastalert_logger.warning("OpsGenieAlerter: default responder not set. Falling back") formated_responders = responders else: formated_responders = default_responders @@ -90,7 +89,7 @@ def alert(self, matches): post['tags'] = self.tags if self.priority and self.priority not in ('P1', 'P2', 'P3', 'P4', 'P5'): - logging.warn("Priority level does not appear to be specified correctly. \ + elastalert_logger.warning("Priority level does not appear to be specified correctly. \ Please make sure to set it to a value between P1 and P5") else: post['priority'] = self.priority @@ -102,7 +101,7 @@ def alert(self, matches): if details: post['details'] = details - logging.debug(json.dumps(post)) + elastalert_logger.debug(json.dumps(post)) headers = { 'Content-Type': 'application/json', @@ -114,12 +113,12 @@ def alert(self, matches): try: r = requests.post(self.to_addr, json=post, headers=headers, proxies=proxies) - logging.debug('request response: {0}'.format(r)) + elastalert_logger.debug('request response: {0}'.format(r)) if r.status_code != 202: elastalert_logger.info("Error response from {0} \n " "API Response: {1}".format(self.to_addr, r)) r.raise_for_status() - logging.info("Alert sent to OpsGenie") + elastalert_logger.info("Alert sent to OpsGenie") except Exception as err: raise EAException("Error sending alert: {0}".format(err)) diff --git a/elastalert/util.py b/elastalert/util.py index bbb0600ff..3e9c9f664 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -152,7 +152,7 @@ def ts_to_dt(timestamp): def dt_to_ts(dt): if not isinstance(dt, datetime.datetime): - logging.warning('Expected datetime, got %s' % (type(dt))) + elastalert_logger.warning('Expected datetime, got %s' % (type(dt))) return dt ts = dt.isoformat() # Round microseconds to milliseconds @@ -176,7 +176,7 @@ def ts_to_dt_with_format(timestamp, ts_format): def dt_to_ts_with_format(dt, ts_format): if not isinstance(dt, datetime.datetime): - logging.warning('Expected datetime, got %s' % (type(dt))) + elastalert_logger.warning('Expected datetime, got %s' % (type(dt))) return dt ts = dt.strftime(ts_format) return ts @@ -361,7 +361,7 @@ def build_es_conn_config(conf): # Deprecated if 'boto_profile' in conf: - logging.warning('Found deprecated "boto_profile", use "profile" instead!') + elastalert_logger.warning('Found deprecated "boto_profile", use "profile" instead!') parsed_conf['profile'] = conf['boto_profile'] if 'profile' in conf: From 04f2cd00920e4d6b7a75c68ce2d3e0045d8cc9b8 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 03:07:48 +0900 Subject: [PATCH 35/44] Fixed the logging property in config.yaml.example --- config.yaml.example | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index 9d9176382..cc659a75f 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -78,38 +78,38 @@ alert_time_limit: # logline: # format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s' # -# handlers: -# console: -# class: logging.StreamHandler -# formatter: logline -# level: DEBUG -# stream: ext://sys.stderr +# handlers: +# console: +# class: logging.StreamHandler +# formatter: logline +# level: DEBUG +# stream: ext://sys.stderr # -# file: -# class : logging.FileHandler -# formatter: logline -# level: DEBUG -# filename: elastalert.log +# file: +# class : logging.FileHandler +# formatter: logline +# level: DEBUG +# filename: elastalert.log # -# loggers: -# elastalert: -# level: WARN -# handlers: [] -# propagate: true +# loggers: +# elastalert: +# level: WARN +# handlers: [] +# propagate: true # -# elasticsearch: -# level: WARN -# handlers: [] -# propagate: true +# elasticsearch: +# level: WARN +# handlers: [] +# propagate: true # -# elasticsearch.trace: -# level: WARN -# handlers: [] -# propagate: true +# elasticsearch.trace: +# level: WARN +# handlers: [] +# propagate: true # -# '': # root logger -# level: WARN -# handlers: -# - console -# - file -# propagate: false +# '': # root logger +# level: WARN +# handlers: +# - console +# - file +# propagate: false From f16c0db4c7b171f8ca7bd14576ee259c444cbff2 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 03:12:05 +0900 Subject: [PATCH 36/44] Remove duplicate property in example config file --- config.yaml.example | 1 - 1 file changed, 1 deletion(-) diff --git a/config.yaml.example b/config.yaml.example index cc659a75f..89db954be 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -48,7 +48,6 @@ es_port: 9200 # Use SSL authentication with client certificates client_cert must be # a pem file containing both cert and key for client -#verify_certs: True #ca_certs: /path/to/cacert.pem #client_cert: /path/to/client_cert.pem #client_key: /path/to/client_key.key From f39ed873693c0b0e1f5f6562500e92c35cb8f975 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 29 Mar 2021 03:24:58 +0900 Subject: [PATCH 37/44] Kibana Discover app link 7.4 -7.11 support --- docs/source/ruletypes.rst | 4 ++-- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 1d99ee2e2..d51398960 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -553,9 +553,9 @@ The currently supported versions of Kibana Discover are: - `5.6` - `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` -- `7.0`, `7.1`, `7.2`, `7.3` +- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10`, `7.11` -``kibana_discover_version: '7.3'`` +``kibana_discover_version: '7.11'`` kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index b36baaac3..0cbbc2116 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -15,7 +15,7 @@ kibana_default_timedelta = datetime.timedelta(minutes=10) kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11']) def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 120faa964..d1e06f49a 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -217,7 +217,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string, format: uri} - kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index f06fe4e0c..aedef1085 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -171,7 +171,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -204,7 +204,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -238,7 +238,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -271,7 +271,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -305,7 +305,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.3', + 'kibana_discover_version': '7.11', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' From 94323bb28cae4da6cc89fb6e8683eb297d4396ed Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 30 Mar 2021 00:20:08 +0900 Subject: [PATCH 38/44] Remove Stride integration --- docs/source/ruletypes.rst | 19 --- elastalert/alerts.py | 94 ------------- elastalert/loaders.py | 1 - elastalert/schema.yaml | 6 - tests/alerts_test.py | 278 -------------------------------------- 5 files changed, 398 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index d51398960..060e5fa76 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1750,25 +1750,6 @@ Valid values: list of strings. Defaults to ``[]``. -Stride -~~~~~~~ - -Stride alerter will send a notification to a predefined Stride room. The body of the notification is formatted the same as with other alerters. -Simple HTML such as and tags will be parsed into a format that Stride can consume. - -The alerter requires the following two options: - -``stride_access_token``: The randomly generated notification token created by Stride. - -``stride_cloud_id``: The site_id associated with the Stride site you want to send the alert to. - -``stride_conversation_id``: The conversation_id associated with the Stride conversation you want to send the alert to. - -``stride_ignore_ssl_errors``: Ignore TLS errors (self-signed certificates, etc.). Default is false. - -``stride_proxy``: By default ElastAlert will not use a network proxy to send notifications to Stride. Set this option using ``hostname:port`` if you need to use a proxy. - - MS Teams ~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 64a339197..10253856b 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -11,7 +11,6 @@ import warnings from email.mime.text import MIMEText from email.utils import formatdate -from html.parser import HTMLParser from smtplib import SMTP from smtplib import SMTP_SSL from smtplib import SMTPAuthenticationError @@ -1984,99 +1983,6 @@ def get_info(self): 'http_post_webhook_url': self.post_url} -class StrideHTMLParser(HTMLParser): - """Parse html into stride's fabric structure""" - - def __init__(self): - """ - Define a couple markup place holders. - """ - self.content = [] - self.mark = None - HTMLParser.__init__(self) - - def handle_starttag(self, tag, attrs): - """Identify and verify starting tag is fabric compatible.""" - if tag == 'b' or tag == 'strong': - self.mark = dict(type='strong') - if tag == 'u': - self.mark = dict(type='underline') - if tag == 'a': - self.mark = dict(type='link', attrs=dict(attrs)) - - def handle_endtag(self, tag): - """Clear mark on endtag.""" - self.mark = None - - def handle_data(self, data): - """Construct data node for our data.""" - node = dict(type='text', text=data) - if self.mark: - node['marks'] = [self.mark] - self.content.append(node) - - -class StrideAlerter(Alerter): - """ Creates a Stride conversation message for each alert """ - - required_options = frozenset( - ['stride_access_token', 'stride_cloud_id', 'stride_conversation_id']) - - def __init__(self, rule): - super(StrideAlerter, self).__init__(rule) - - self.stride_access_token = self.rule['stride_access_token'] - self.stride_cloud_id = self.rule['stride_cloud_id'] - self.stride_conversation_id = self.rule['stride_conversation_id'] - self.stride_ignore_ssl_errors = self.rule.get('stride_ignore_ssl_errors', False) - self.stride_proxy = self.rule.get('stride_proxy', None) - self.url = 'https://api.atlassian.com/site/%s/conversation/%s/message' % ( - self.stride_cloud_id, self.stride_conversation_id) - - def alert(self, matches): - body = self.create_alert_body(matches).strip() - - # parse body with StrideHTMLParser - parser = StrideHTMLParser() - parser.feed(body) - - # Post to Stride - headers = { - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(self.stride_access_token) - } - - # set https proxy, if it was provided - proxies = {'https': self.stride_proxy} if self.stride_proxy else None - - # build stride json payload - # https://developer.atlassian.com/cloud/stride/apis/document/structure/ - payload = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': parser.content} - ]} - ]}} - - try: - if self.stride_ignore_ssl_errors: - requests.packages.urllib3.disable_warnings() - response = requests.post( - self.url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, verify=not self.stride_ignore_ssl_errors, - proxies=proxies) - warnings.resetwarnings() - response.raise_for_status() - except RequestException as e: - raise EAException("Error posting to Stride: %s" % e) - elastalert_logger.info( - "Alert sent to Stride conversation %s" % self.stride_conversation_id) - - def get_info(self): - return {'type': 'stride', - 'stride_cloud_id': self.stride_cloud_id, - 'stride_converstation_id': self.stride_converstation_id} - - class LineNotifyAlerter(Alerter): """ Created a Line Notify for each alert """ required_option = frozenset(["linenotify_access_token"]) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index fcfb5a92f..b463383f9 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -63,7 +63,6 @@ class RulesLoader(object): 'command': alerts.CommandAlerter, 'sns': alerts.SnsAlerter, 'hipchat': alerts.HipChatAlerter, - 'stride': alerts.StrideAlerter, 'ms_teams': alerts.MsTeamsAlerter, 'slack': alerts.SlackAlerter, 'mattermost': alerts.MattermostAlerter, diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index d1e06f49a..734b0344b 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -270,12 +270,6 @@ properties: hipchat_from: {type: string} hipchat_mentions: {type: array, items: {type: string}} - ### Stride - stride_access_token: {type: string} - stride_cloud_id: {type: string} - stride_conversation_id: {type: string} - stride_ignore_ssl_errors: {type: boolean} - ### Slack slack_webhook_url: *arrayOfString slack_username_override: {type: string} diff --git a/tests/alerts_test.py b/tests/alerts_test.py index cbe6e2b36..523c80bd8 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -20,7 +20,6 @@ from elastalert.alerts import MsTeamsAlerter from elastalert.alerts import PagerDutyAlerter from elastalert.alerts import SlackAlerter -from elastalert.alerts import StrideAlerter from elastalert.loaders import FileRulesLoader from elastalert.opsgenie import OpsGenieAlerter from elastalert.util import ts_add @@ -2060,283 +2059,6 @@ def test_resolving_rule_references(ea): assert 'the_owner' == alert.rule['nested_dict']['nested_owner'] -def test_stride_plain_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "{0}\n\n@timestamp: {1}\nsomefield: {2}".format( - rule['name'], match['@timestamp'], match['somefield'] - ) - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_underline_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Underline Text', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Underline Text" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'underline'} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_bold_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Bold Text', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Bold Text" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'strong'} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_strong_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Bold Text', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Bold Text" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'strong'} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_hyperlink(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Link', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - body = "Link" - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': body, 'marks': [ - {'type': 'link', 'attrs': {'href': 'http://stride.com'}} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - -def test_stride_html(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'stride_access_token': 'token', - 'stride_cloud_id': 'cloud_id', - 'stride_conversation_id': 'conversation_id', - 'alert_subject': 'Cool subject', - 'alert_text': 'Alert: we found something. Link', - 'alert_text_type': 'alert_text_only', - 'alert': [] - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = StrideAlerter(rule) - match = { - '@timestamp': '2016-01-01T00:00:00', - 'somefield': 'foobarbaz' - } - with mock.patch('requests.post') as mock_post_request: - alert.alert([match]) - - expected_data = {'body': {'version': 1, 'type': "doc", 'content': [ - {'type': "panel", 'attrs': {'panelType': "warning"}, 'content': [ - {'type': 'paragraph', 'content': [ - {'type': 'text', 'text': 'Alert', 'marks': [ - {'type': 'strong'} - ]}, - {'type': 'text', 'text': ': we found something. '}, - {'type': 'text', 'text': 'Link', 'marks': [ - {'type': 'link', 'attrs': {'href': 'http://stride.com'}} - ]} - ]} - ]} - ]}} - - mock_post_request.assert_called_once_with( - alert.url, - data=mock.ANY, - headers={ - 'content-type': 'application/json', - 'Authorization': 'Bearer {}'.format(rule['stride_access_token'])}, - verify=True, - proxies=None - ) - assert expected_data == json.loads( - mock_post_request.call_args_list[0][1]['data']) - - def test_hipchat_body_size_limit_text(): rule = { 'name': 'Test Rule', From b07bd7b5b63a6a21c73dc8ba249db90e462c734d Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 30 Mar 2021 00:28:45 +0900 Subject: [PATCH 39/44] Remove hipchat integration --- README.md | 1 - docs/source/elastalert.rst | 1 - docs/source/ruletypes.rst | 37 ---------------- elastalert/alerts.py | 86 -------------------------------------- elastalert/loaders.py | 8 ---- elastalert/schema.yaml | 9 ---- tests/alerts_test.py | 58 ------------------------- 7 files changed, 200 deletions(-) diff --git a/README.md b/README.md index df15315e1..1114b0f4d 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Currently, we have built-in support for the following alert types: - JIRA - OpsGenie - Commands -- HipChat - MS Teams - Slack - Telegram diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 3bd400d0c..2d962f1d0 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -36,7 +36,6 @@ Currently, we have support built in for these alert types: - JIRA - OpsGenie - AWS SNS -- HipChat - Slack - Telegram - GoogleChat diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 060e5fa76..bf845829b 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1712,43 +1712,6 @@ Example When to use aws_profile usage:: sns_topic_arn: 'arn:aws:sns:us-east-1:123456789:somesnstopic' sns_aws_profile: 'default' -HipChat -~~~~~~~ - -HipChat alerter will send a notification to a predefined HipChat room. The body of the notification is formatted the same as with other alerters. - -The alerter requires the following two options: - -``hipchat_auth_token``: The randomly generated notification token created by HipChat. Go to https://XXXXX.hipchat.com/account/api and use -'Create new token' section, choosing 'Send notification' in Scopes list. - -``hipchat_room_id``: The id associated with the HipChat room you want to send the alert to. Go to https://XXXXX.hipchat.com/rooms and choose -the room you want to post to. The room ID will be the numeric part of the URL. - -``hipchat_msg_color``: The color of the message background that is sent to HipChat. May be set to green, yellow or red. Default is red. - -``hipchat_domain``: The custom domain in case you have HipChat own server deployment. Default is api.hipchat.com. - -``hipchat_ignore_ssl_errors``: Ignore TLS errors (self-signed certificates, etc.). Default is false. - -``hipchat_proxy``: By default ElastAlert will not use a network proxy to send notifications to HipChat. Set this option using ``hostname:port`` if you need to use a proxy. - -``hipchat_notify``: When set to true, triggers a hipchat bell as if it were a user. Default is true. - -``hipchat_from``: When humans report to hipchat, a timestamp appears next to their name. For bots, the name is the name of the token. The from, instead of a timestamp, defaults to empty unless set, which you can do here. This is optional. - -``hipchat_message_format``: Determines how the message is treated by HipChat and rendered inside HipChat applications -html - Message is rendered as HTML and receives no special treatment. Must be valid HTML and entities must be escaped (e.g.: '&' instead of '&'). May contain basic tags: a, b, i, strong, em, br, img, pre, code, lists, tables. -text - Message is treated just like a message sent by a user. Can include @mentions, emoticons, pastes, and auto-detected URLs (Twitter, YouTube, images, etc). -Valid values: html, text. -Defaults to 'html'. - -``hipchat_mentions``: When using a ``html`` message format, it's not possible to mentions specific users using the ``@user`` syntax. -In that case, you can set ``hipchat_mentions`` to a list of users which will be first mentioned using a single text message, then the normal ElastAlert message will be sent to Hipchat. -If set, it will mention the users, no matter if the original message format is set to HTML or text. -Valid values: list of strings. -Defaults to ``[]``. - MS Teams ~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 10253856b..368972960 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -961,92 +961,6 @@ def alert(self, matches): elastalert_logger.info("Sent sns notification to %s" % (self.sns_topic_arn)) -class HipChatAlerter(Alerter): - """ Creates a HipChat room notification for each alert """ - required_options = frozenset(['hipchat_auth_token', 'hipchat_room_id']) - - def __init__(self, rule): - super(HipChatAlerter, self).__init__(rule) - self.hipchat_msg_color = self.rule.get('hipchat_msg_color', 'red') - self.hipchat_message_format = self.rule.get('hipchat_message_format', 'html') - self.hipchat_auth_token = self.rule['hipchat_auth_token'] - self.hipchat_room_id = self.rule['hipchat_room_id'] - self.hipchat_domain = self.rule.get('hipchat_domain', 'api.hipchat.com') - self.hipchat_ignore_ssl_errors = self.rule.get('hipchat_ignore_ssl_errors', False) - self.hipchat_notify = self.rule.get('hipchat_notify', True) - self.hipchat_from = self.rule.get('hipchat_from', '') - self.url = 'https://%s/v2/room/%s/notification?auth_token=%s' % ( - self.hipchat_domain, self.hipchat_room_id, self.hipchat_auth_token) - self.hipchat_proxy = self.rule.get('hipchat_proxy', None) - - def create_alert_body(self, matches): - body = super(HipChatAlerter, self).create_alert_body(matches) - - # HipChat sends 400 bad request on messages longer than 10000 characters - if self.hipchat_message_format == 'html': - # Use appropriate line ending for text/html - br = '
' - body = body.replace('\n', br) - - truncated_message = '
...(truncated)' - truncate_to = 10000 - len(truncated_message) - else: - truncated_message = '..(truncated)' - truncate_to = 10000 - len(truncated_message) - - if (len(body) > 9999): - body = body[:truncate_to] + truncated_message - - return body - - def alert(self, matches): - body = self.create_alert_body(matches) - - # Post to HipChat - headers = {'content-type': 'application/json'} - # set https proxy, if it was provided - proxies = {'https': self.hipchat_proxy} if self.hipchat_proxy else None - payload = { - 'color': self.hipchat_msg_color, - 'message': body, - 'message_format': self.hipchat_message_format, - 'notify': self.hipchat_notify, - 'from': self.hipchat_from - } - - try: - if self.hipchat_ignore_ssl_errors: - requests.packages.urllib3.disable_warnings() - - if self.rule.get('hipchat_mentions', []): - ping_users = self.rule.get('hipchat_mentions', []) - ping_msg = payload.copy() - ping_msg['message'] = "ping {}".format( - ", ".join("@{}".format(user) for user in ping_users) - ) - ping_msg['message_format'] = "text" - - response = requests.post( - self.url, - data=json.dumps(ping_msg, cls=DateTimeEncoder), - headers=headers, - verify=not self.hipchat_ignore_ssl_errors, - proxies=proxies) - - response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, - verify=not self.hipchat_ignore_ssl_errors, - proxies=proxies) - warnings.resetwarnings() - response.raise_for_status() - except RequestException as e: - raise EAException("Error posting to HipChat: %s" % e) - elastalert_logger.info("Alert sent to HipChat room %s" % self.hipchat_room_id) - - def get_info(self): - return {'type': 'hipchat', - 'hipchat_room_id': self.hipchat_room_id} - - class MsTeamsAlerter(Alerter): """ Creates a Microsoft Teams Conversation Message for each alert """ required_options = frozenset(['ms_teams_webhook_url', 'ms_teams_alert_summary']) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index b463383f9..3f2515d97 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -62,7 +62,6 @@ class RulesLoader(object): 'debug': alerts.DebugAlerter, 'command': alerts.CommandAlerter, 'sns': alerts.SnsAlerter, - 'hipchat': alerts.HipChatAlerter, 'ms_teams': alerts.MsTeamsAlerter, 'slack': alerts.SlackAlerter, 'mattermost': alerts.MattermostAlerter, @@ -316,13 +315,6 @@ def _dt_to_ts_with_format(dt): rule.setdefault('client_cert', conf.get('client_cert')) rule.setdefault('client_key', conf.get('client_key')) - # Set HipChat options from global config - rule.setdefault('hipchat_msg_color', 'red') - rule.setdefault('hipchat_domain', 'api.hipchat.com') - rule.setdefault('hipchat_notify', True) - rule.setdefault('hipchat_from', '') - rule.setdefault('hipchat_ignore_ssl_errors', False) - # Make sure we have required options if self.required_locals - frozenset(list(rule.keys())): raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(list(rule.keys()))))) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 734b0344b..2b0dbe481 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -261,15 +261,6 @@ properties: jira_max_age: {type: number} jira_watchers: *arrayOfString - ### HipChat - hipchat_auth_token: {type: string} - hipchat_room_id: {type: [string, integer]} - hipchat_domain: {type: string} - hipchat_ignore_ssl_errors: {type: boolean} - hipchat_notify: {type: boolean} - hipchat_from: {type: string} - hipchat_mentions: {type: array, items: {type: string}} - ### Slack slack_webhook_url: *arrayOfString slack_username_override: {type: string} diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 523c80bd8..81996ada8 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -13,7 +13,6 @@ from elastalert.alerts import BasicMatchString from elastalert.alerts import CommandAlerter from elastalert.alerts import EmailAlerter -from elastalert.alerts import HipChatAlerter from elastalert.alerts import HTTPPostAlerter from elastalert.alerts import JiraAlerter from elastalert.alerts import JiraFormattedMatchString @@ -2059,63 +2058,6 @@ def test_resolving_rule_references(ea): assert 'the_owner' == alert.rule['nested_dict']['nested_owner'] -def test_hipchat_body_size_limit_text(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'hipchat_auth_token': 'token', - 'hipchat_room_id': 'room_id', - 'hipchat_message_format': 'text', - 'alert_subject': 'Cool subject', - 'alert_text': 'Alert: we found something.\n\n{message}', - 'alert_text_type': 'alert_text_only', - 'alert': [], - 'alert_text_kw': { - '@timestamp': 'time', - 'message': 'message', - }, - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = HipChatAlerter(rule) - match = { - '@timestamp': '2018-01-01T00:00:00', - 'message': 'foo bar\n' * 5000, - } - body = alert.create_alert_body([match]) - - assert len(body) <= 10000 - - -def test_hipchat_body_size_limit_html(): - rule = { - 'name': 'Test Rule', - 'type': 'any', - 'hipchat_auth_token': 'token', - 'hipchat_room_id': 'room_id', - 'hipchat_message_format': 'html', - 'alert_subject': 'Cool subject', - 'alert_text': 'Alert: we found something.\n\n{message}', - 'alert_text_type': 'alert_text_only', - 'alert': [], - 'alert_text_kw': { - '@timestamp': 'time', - 'message': 'message', - }, - } - rules_loader = FileRulesLoader({}) - rules_loader.load_modules(rule) - alert = HipChatAlerter(rule) - match = { - '@timestamp': '2018-01-01T00:00:00', - 'message': 'foo bar\n' * 5000, - } - - body = alert.create_alert_body([match]) - - assert len(body) <= 10000 - - def test_alerta_no_auth(ea): rule = { 'name': 'Test Alerta rule!', From 86be6edacd9020e3a77edb53d4b7fd7dfd4ad92d Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Tue, 30 Mar 2021 01:20:24 +0900 Subject: [PATCH 40/44] Revert pytest version --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f06659826..11cc0902a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ flake8 pre-commit pluggy>=0.12.0 pylint<2.8 -pytest<3.7.0 +pytest<3.3.0 setuptools sphinx_rtd_theme tox==3.23.0 From f4123517be091119dc2b03bd64202e3a15681315 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 5 Apr 2021 00:18:46 +0900 Subject: [PATCH 41/44] Kibana Discover support kibana 7.12 --- docs/source/ruletypes.rst | 4 ++-- elastalert/kibana_discover.py | 2 +- elastalert/schema.yaml | 2 +- tests/kibana_discover_test.py | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index bf845829b..beb1253de 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -553,9 +553,9 @@ The currently supported versions of Kibana Discover are: - `5.6` - `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` -- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10`, `7.11` +- `7.0`, `7.1`, `7.2`, `7.3`, `7.4`, `7.5`, `7.6`, `7.7`, `7.8`, `7.9`, `7.10`, `7.11`, `7.12` -``kibana_discover_version: '7.11'`` +``kibana_discover_version: '7.12'`` kibana_discover_index_pattern_id ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 0cbbc2116..58e3476f4 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -15,7 +15,7 @@ kibana_default_timedelta = datetime.timedelta(minutes=10) kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12']) def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 2b0dbe481..d8aef3968 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -217,7 +217,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string, format: uri} - kibana_discover_version: {type: string, enum: ['7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.12', '7.11', '7.10', '7.9', '7.8', '7.7', '7.6', '7.5', '7.4', '7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string, minLength: 1} kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} kibana_discover_from_timedelta: *timedelta diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index aedef1085..0e796e480 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -38,7 +38,7 @@ def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): assert url == expectedUrl -@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3', '7.4', '7.5', '7.6', '7.7', '7.8', '7.9', '7.10', '7.11', '7.12']) def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): url = generate_kibana_discover_url( rule={ @@ -171,7 +171,7 @@ def test_generate_kibana_discover_url_with_from_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -204,7 +204,7 @@ def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -238,7 +238,7 @@ def test_generate_kibana_discover_url_with_to_timedelta(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timestamp_field': 'timestamp' @@ -271,7 +271,7 @@ def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), 'timeframe': timedelta(minutes=20), @@ -305,7 +305,7 @@ def test_generate_kibana_discover_url_with_timeframe(): url = generate_kibana_discover_url( rule={ 'kibana_discover_app_url': 'http://kibana:5601/#/discover', - 'kibana_discover_version': '7.11', + 'kibana_discover_version': '7.12', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), 'timestamp_field': 'timestamp' From bd102a247f2b1faef125a666addde096c961aabc Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Mon, 12 Apr 2021 23:27:36 +0900 Subject: [PATCH 42/44] apscheduler>=3.3.0,<4.0 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b05c38c50..5cdd2f0f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apscheduler>=3.3.0 +apscheduler>=3.3.0,<4.0 aws-requests-auth>=0.3.0 boto3>=1.4.4 cffi>=1.11.5 diff --git a/setup.py b/setup.py index 6a60fc630..9a8bfd88a 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ packages=find_packages(), package_data={'elastalert': ['schema.yaml', 'es_mappings/**/*.json']}, install_requires=[ - 'apscheduler>=3.3.0', + 'apscheduler>=3.3.0,<4.0', 'aws-requests-auth>=0.3.0', 'boto3>=1.4.4', 'configparser>=3.5.0', From 36af158e3745ee194b83cf9f59a5c26ce3f8b393 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 15 Apr 2021 00:01:16 +0900 Subject: [PATCH 43/44] sphinx 1.6.6 to 3.5.4 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 71099e17c..edcbc2ba4 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,6 @@ norecursedirs = .* virtualenv_run docs build venv env [testenv:docs] deps = {[testenv]deps} - sphinx==1.6.6 + sphinx==3.5.4 changedir = docs commands = sphinx-build -b html -d build/doctrees -W source build/html From ec9e60963643995aaa2693a0b827bd391618afd8 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Thu, 15 Apr 2021 00:14:49 +0900 Subject: [PATCH 44/44] python 3.6 to 3.8 --- .travis.yml | 16 ++++++++-------- Dockerfile-test | 2 +- README.md | 2 +- docs/source/running_elastalert.rst | 5 +++-- setup.py | 2 +- tox.ini | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 569bf12d6..1e3bb601b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python python: -- '3.6' +- '3.8' env: - TOXENV=docs -- TOXENV=py36 +- TOXENV=py38 install: - pip install tox - > @@ -23,12 +23,12 @@ script: jobs: include: - stage: 'Elasticsearch test' - env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64 - - env: TOXENV=py36 ES_VERSION=6.6.2 - - env: TOXENV=py36 ES_VERSION=6.3.2 - - env: TOXENV=py36 ES_VERSION=6.2.4 - - env: TOXENV=py36 ES_VERSION=6.0.1 - - env: TOXENV=py36 ES_VERSION=5.6.16 + env: TOXENV=py38 ES_VERSION=7.0.0-linux-x86_64 + - env: TOXENV=py38 ES_VERSION=6.6.2 + - env: TOXENV=py38 ES_VERSION=6.3.2 + - env: TOXENV=py38 ES_VERSION=6.2.4 + - env: TOXENV=py38 ES_VERSION=6.0.1 + - env: TOXENV=py38 ES_VERSION=5.6.16 deploy: provider: pypi diff --git a/Dockerfile-test b/Dockerfile-test index fb8a78409..c164b90c3 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -3,7 +3,7 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y RUN apt-get install software-properties-common -y RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git +RUN apt-get -y install build-essential python3.8 python3.8-dev python3-pip libssl-dev git WORKDIR /home/elastalert diff --git a/README.md b/README.md index 1114b0f4d..81a22d71e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. +Recent changes: As of Elastalert 0.2.0, you must use Python 3.8. Python 2 will not longer be supported. [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 7fdf1eeba..75086647c 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -8,9 +8,10 @@ Requirements - Elasticsearch - ISO8601 or Unix timestamped data -- Python 3.6 +- Python 3.8 - pip, see requirements.txt -- Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev +- Packages on Ubuntu 18.x: build-essential python3-pip python3.8 python3.8-dev libffi-dev libssl-dev +- Packages on Ubuntu 20.x: build-essential python3-pip python3.8 python3.8-dev libffi-dev libssl-dev Downloading and Configuring --------------------------- diff --git a/setup.py b/setup.py index 9a8bfd88a..744b26500 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup_requires='setuptools', license='Copyright 2014 Yelp', classifiers=[ - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', ], diff --git a/tox.ini b/tox.ini index edcbc2ba4..99912035b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] project = elastalert -envlist = py36,docs +envlist = py38,docs [testenv] deps = -rrequirements-dev.txt