Date: Tue, 20 Apr 2021 14:04:30 +0200
Subject: [PATCH] Index health check improvements (#5612)
* Health check / Add a check if the index is in readonly mode.
When in readonly mode (eg. when disk usage is too high), new record will not be created properly - at least added in db but not indexed. No error where reported at GN level, only in logs.
* Health / Add indicator about number of records in db and in index.
* Health check / Fix no index error query. Show status once loaded.
---
.../health/IndexReadOnlyHealthCheck.java | 79 +++++++++++++++++++
.../health/NoIndexErrorsHealthCheck.java | 2 +-
.../js/admin/DashboardStatusController.js | 10 +++
.../main/resources/catalog/locales/en-v4.json | 5 +-
.../templates/admin/dashboard/status.html | 20 +++++
.../config/config-service-monitoring.xml | 1 +
6 files changed, 115 insertions(+), 2 deletions(-)
create mode 100644 healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java
diff --git a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java
new file mode 100644
index 00000000000..cc0d104f485
--- /dev/null
+++ b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2021 Food and Agriculture Organization of the
+ * United Nations (FAO-UN), United Nations World Food Programme (WFP)
+ * and United Nations Environment Programme (UNEP)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ *
+ * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+ * Rome - Italy. email: geonetwork@osgeo.org
+ */
+
+package org.fao.geonet.monitor.health;
+
+import com.yammer.metrics.core.HealthCheck;
+import jeeves.monitor.HealthCheckFactory;
+import jeeves.server.context.ServiceContext;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.fao.geonet.ApplicationContextHolder;
+import org.fao.geonet.kernel.search.EsSearchManager;
+import org.springframework.context.ApplicationContext;
+
+/**
+ * Checks to ensure that the Elasticsearch index is not in readonly mode.
+ *
+ * This can happen when disk space available is lower than 10% (default).
+ * Index administrator can force state of the index using:
+ * + * curl -X PUT "localhost:9200/gn-records/_settings" \
+ * -H 'Content-Type: application/json' \
+ * -d'{ "index.blocks.read_only_allow_delete" : true } }'
+ *
+ */
+public class IndexReadOnlyHealthCheck implements HealthCheckFactory {
+ public HealthCheck create(final ServiceContext context) {
+ return new HealthCheck(this.getClass().getSimpleName()) {
+ @Override
+ protected Result check() {
+ try {
+ ApplicationContext applicationContext = ApplicationContextHolder.get();
+ EsSearchManager searchMan = applicationContext.getBean(EsSearchManager.class);
+ String indexBlockRead = "index.blocks.read_only_allow_delete";
+ GetSettingsRequest request = new GetSettingsRequest();
+ request.names(indexBlockRead);
+ GetSettingsResponse settings = searchMan.getClient().getClient()
+ .indices().getSettings(request, RequestOptions.DEFAULT);
+
+ Boolean isReadOnly = "true".equals(settings.getSetting(searchMan.getDefaultIndex(), indexBlockRead));
+
+ if (!isReadOnly) {
+ return Result.healthy(String.format(
+ "Index is writable.",
+ searchMan.getDefaultIndex()
+ ));
+ } else {
+ return Result.unhealthy(
+ "Index is in Readonly mode. Check disk usage and/or indexing server logs.");
+ }
+ } catch (Throwable e) {
+ return Result.unhealthy(e);
+ }
+ }
+ };
+ }
+}
diff --git a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/NoIndexErrorsHealthCheck.java b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/NoIndexErrorsHealthCheck.java
index 3121adf5d5c..c641634ef94 100644
--- a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/NoIndexErrorsHealthCheck.java
+++ b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/NoIndexErrorsHealthCheck.java
@@ -44,7 +44,7 @@ protected Result check() throws Exception {
GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
EsSearchManager searchMan = gc.getBean(EsSearchManager.class);
- long numDocs = searchMan.getNumDocs("+" + IndexFields.INDEXING_ERROR_FIELD + ":1");
+ long numDocs = searchMan.getNumDocs("+" + IndexFields.INDEXING_ERROR_FIELD + ":true");
if (numDocs > 0) {
return Result.unhealthy("Found " + numDocs + " metadata that had errors during indexing");
diff --git a/web-ui/src/main/resources/catalog/js/admin/DashboardStatusController.js b/web-ui/src/main/resources/catalog/js/admin/DashboardStatusController.js
index c897bc7d34a..cd99d9104ef 100644
--- a/web-ui/src/main/resources/catalog/js/admin/DashboardStatusController.js
+++ b/web-ui/src/main/resources/catalog/js/admin/DashboardStatusController.js
@@ -54,6 +54,16 @@
$scope.threadSortField = undefined;
$scope.threadSortReverse = false;
$scope.threadInfoLoading = false;
+
+ $scope.indexStatus = null;
+ function getIndexStatus() {
+ $http.get('../api/site/index/synchronized').
+ success(function(data) {
+ $scope.indexStatus = data;
+ });
+ }
+ getIndexStatus();
+
$scope.setThreadSortField = function(field) {
if (field === $scope.threadSortField) {
$scope.threadSortReverse = !$scope.threadSortReverse;
diff --git a/web-ui/src/main/resources/catalog/locales/en-v4.json b/web-ui/src/main/resources/catalog/locales/en-v4.json
index b403a3f728a..e3bcb68e642 100644
--- a/web-ui/src/main/resources/catalog/locales/en-v4.json
+++ b/web-ui/src/main/resources/catalog/locales/en-v4.json
@@ -154,5 +154,8 @@
"facets.temporalRange.seriesLegend": "Blue series: current search, Grey series: all records",
"recordFormats": "Distribution formats:",
"recordFormatDownload": "Download as ",
- "commonProtocols": "General protocols:"
+ "commonProtocols": "General protocols:",
+ "indexStatusSynchronized": "Records in index/db = {{indexStatus['index.count']}}/{{indexStatus['db.count']}}",
+ "indexStatusSynchronized-error": "When number of records is not equal in the database and in the index, check first that the catalogue point to the correct index. If yes, try to reindex the catalogue from admin > tools. If there is still a difference, check for indexing errors in the log file and fix the metadata record using the XML view.",
+ "IndexReadOnlyHealthCheck": "Index is in readonly mode"
}
diff --git a/web-ui/src/main/resources/catalog/templates/admin/dashboard/status.html b/web-ui/src/main/resources/catalog/templates/admin/dashboard/status.html
index 56780cd1e57..da4827a8a6f 100644
--- a/web-ui/src/main/resources/catalog/templates/admin/dashboard/status.html
+++ b/web-ui/src/main/resources/catalog/templates/admin/dashboard/status.html
@@ -31,6 +31,7 @@ {{info.name}}
{{info.status}}
+
@@ -64,6 +65,25 @@ {{info.name}}
{{info.status}}
+
+
+
+
+ indexStatusSynchronized
+
+
+
+
+ {{(indexStatus['db.count'] === indexStatus['index.count']) ? 'OK' : 'ERROR'}}
+
+
+
diff --git a/web/src/main/webapp/WEB-INF/config/config-service-monitoring.xml b/web/src/main/webapp/WEB-INF/config/config-service-monitoring.xml
index 6305452f391..acce46605fd 100644
--- a/web/src/main/webapp/WEB-INF/config/config-service-monitoring.xml
+++ b/web/src/main/webapp/WEB-INF/config/config-service-monitoring.xml
@@ -36,6 +36,7 @@
+