ES8 support may need to be considered in addition to doc_type.

In ES7.X this include_type_name default has been changed to false, but elastalert explicitly specifies include_type_name = true to use the type, but it will be removed in ES8. Will be.

Also, the specification of dateOptionalTime is abolished and becomes date_optional_time.

elasticsearch>=7.0.0,<8.0.0

    ProcessController:  /home/node/.local/lib/python3.8/site-packages/elasticsearch/connection/base.py:193: ElasticsearchDeprecationWarning: Camel case format name dateOptionalTime is deprecated and will be removed in a future version. Use snake case name date_optional_time instead.
      warnings.warn(message, category=ElasticsearchDeprecationWarning)
    /home/node/.local/lib/python3.8/site-packages/elasticsearch/connection/base.py:193: ElasticsearchDeprecationWarning: [types removal] Using include_type_name in put mapping requests is deprecated. The parameter will be removed in the next major version.
      warnings.warn(message, category=ElasticsearchDeprecationWarning)

The following are expected changes that need to be addressed.

___init___.py

add

    def is_atleasteight(self):
        """
        Returns True when the Elasticsearch server version >= 8
        """
        return int(self.es_version.split(".")[0]) >= 8

elastalert/create_index.py

before

   if is_atleastseven(esversion):
        # TODO remove doc_type completely when elasicsearch client allows doc_type=None
        # doc_type is a deprecated feature and will be completely removed in Elasicsearch 8
        es_client.indices.put_mapping(index=ea_index, doc_type='_doc',
                                      body=es_index_mappings['elastalert'], include_type_name=True)
        es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc',
                                      body=es_index_mappings['elastalert_status'], include_type_name=True)
        es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc',
                                      body=es_index_mappings['silence'], include_type_name=True)
        es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc',
                                      body=es_index_mappings['elastalert_error'], include_type_name=True)
        es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc',
                                      body=es_index_mappings['past_elastalert'], include_type_name=True)

after

   if is_atleasteight(esversion):
        es_client.indices.put_mapping(index=ea_index,
                                      body=es_index_mappings['elastalert'])
        es_client.indices.put_mapping(index=ea_index + '_status'
                                      body=es_index_mappings['elastalert_status'])
        es_client.indices.put_mapping(index=ea_index + '_silence',
                                      body=es_index_mappings['silence'])
        es_client.indices.put_mapping(index=ea_index + '_error',
                                      body=es_index_mappings['elastalert_error'])
        es_client.indices.put_mapping(index=ea_index + '_past',
                                      body=es_index_mappings['past_elastalert'])
   elif is_atleastseven(esversion):
        es_client.indices.put_mapping(index=ea_index, doc_type='_doc',
                                      body=es_index_mappings['elastalert'])
        es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc',
                                      body=es_index_mappings['elastalert_status'])
        es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc',
                                      body=es_index_mappings['silence'])
        es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc',
                                      body=es_index_mappings['elastalert_error'])
        es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc',
                                      body=es_index_mappings['past_elastalert'])

elastalert/loaders.py

It is necessary to prevent the following checks from being performed on elasticsearch 8 or later versions

        if rule.get('use_count_query') or rule.get('use_terms_query'):
            if 'doc_type' not in rule:
                raise EAException('doc_type must be specified.')

elastalert/elastalert.py

before

        # Record doc_type for use in get_top_counts
        if 'doc_type' not in rule and len(hits):
          rule['doc_type'] = hits[0]['_type']

after

        # Record doc_type for use in get_top_counts
        if not self.thread_data.current_es.is_atleasteight():
            if 'doc_type' not in rule and len(hits):
                rule['doc_type'] = hits[0]['_type']

before

            if not rule['five']:
                res = self.thread_data.current_es.deprecated_search(
                    index=index,
                    doc_type=rule['doc_type'],
                    body=query,
                    search_type='count',
                    ignore_unavailable=True
                )
            else:
                res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule['doc_type'],
                                                                    body=query, size=0, ignore_unavailable=True)

after

            if not rule['five']:
                if self.thread_data.current_es.is_atleasteight():
                    res = self.thread_data.current_es.deprecated_search(
                        index=index,
                        body=query,
                        search_type='count',
                        ignore_unavailable=True
                    )
                else:
                    res = self.thread_data.current_es.deprecated_search(
                        index=index,
                        doc_type=rule['doc_type'],
                        body=query,
                        search_type='count',
                        ignore_unavailable=True
                    )
            else:
                res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule['doc_type'],
                                                                    body=query, size=0, ignore_unavailable=True)

before

            if not rule['five']:
                res = self.thread_data.current_es.deprecated_search(
                    index=index,
                    doc_type=rule.get('doc_type'),
                    body=query,
                    search_type='count',
                    ignore_unavailable=True
                )
            else:
                res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'),
                                                                    body=query, size=0, ignore_unavailable=True)

after

            if not rule['five']:
                if self.thread_data.current_es.is_atleasteight():
                    res = self.thread_data.current_es.deprecated_search(
                        index=index,
                        body=query,
                        search_type='count',
                        ignore_unavailable=True
                    )
                else:
                    res = self.thread_data.current_es.deprecated_search(
                        index=index,
                        doc_type=rule.get('doc_type'),
                        body=query,
                        search_type='count',
                        ignore_unavailable=True
                    )
            else:
                res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'),
                                                                    body=query, size=0, ignore_unavailable=True)

elastalert/test_rule.py

before

res = es_client.count(index=index, doc_type=doc_type, body=count_query, ignore_unavailable=True)

after

if es_client.is_atleasteight():
    res = es_client.count(index=index, body=count_query, ignore_unavailable=True)
else:
    res = es_client.count(index=index, doc_type=doc_type, body=count_query, ignore_unavailable=True)

elastalert/tests/conftest.py

mock_es_client

add

self.is_atleasteight = mock.Mock(return_value=False)

mock_es_sixsix_client

add

self.is_atleasteight = mock.Mock(return_value=False)

elastalert/es_mappings/8/elastalert.json

add file