Index health check improvements (geonetwork#5612)

* Health check / Add a check if the index is in readonly mode. When in readonly mode (eg. when disk usage is too high), new record will not be created properly - at least added in db but not indexed. No error where reported at GN level, only in logs. * Health / Add indicator about number of records in db and in index. * Health check / Fix no index error query. Show status once loaded.
zoran995 · Apr 20, 2021 · 9504f94 · 9504f94
1 parent 012d06d
commit 9504f94
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 2 deletions.
diff --git a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/IndexReadOnlyHealthCheck.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2021 Food and Agriculture Organization of the
+ * United Nations (FAO-UN), United Nations World Food Programme (WFP)
+ * and United Nations Environment Programme (UNEP)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ *
+ * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+ * Rome - Italy. email: geonetwork@osgeo.org
+ */
+
+package org.fao.geonet.monitor.health;
+
+import com.yammer.metrics.core.HealthCheck;
+import jeeves.monitor.HealthCheckFactory;
+import jeeves.server.context.ServiceContext;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RequestOptions;
+import org.fao.geonet.ApplicationContextHolder;
+import org.fao.geonet.kernel.search.EsSearchManager;
+import org.springframework.context.ApplicationContext;
+
+/**
+ * Checks to ensure that the Elasticsearch index is not in readonly mode.
+ *
+ * This can happen when disk space available is lower than 10% (default).
+ * Index administrator can force state of the index using:
+ * <pre>
+ *     curl -X PUT "localhost:9200/gn-records/_settings" \
+ *     -H 'Content-Type: application/json' \
+ *     -d'{ "index.blocks.read_only_allow_delete" : true } }'
+ * </pre>
+ */
+public class IndexReadOnlyHealthCheck implements HealthCheckFactory {
+    public HealthCheck create(final ServiceContext context) {
+        return new HealthCheck(this.getClass().getSimpleName()) {
+            @Override
+            protected Result check() {
+                try {
+                    ApplicationContext applicationContext = ApplicationContextHolder.get();
+                    EsSearchManager searchMan = applicationContext.getBean(EsSearchManager.class);
+                    String indexBlockRead = "index.blocks.read_only_allow_delete";
+                    GetSettingsRequest request = new GetSettingsRequest();
+                    request.names(indexBlockRead);
+                    GetSettingsResponse settings = searchMan.getClient().getClient()
+                        .indices().getSettings(request, RequestOptions.DEFAULT);
+
+                    Boolean isReadOnly = "true".equals(settings.getSetting(searchMan.getDefaultIndex(),  indexBlockRead));
+
+                    if (!isReadOnly) {
+                        return Result.healthy(String.format(
+                            "Index is writable.",
+                            searchMan.getDefaultIndex()
+                        ));
+                    } else {
+                        return Result.unhealthy(
+                            "Index is in Readonly mode. Check disk usage and/or indexing server logs.");
+                    }
+                } catch (Throwable e) {
+                    return Result.unhealthy(e);
+                }
+            }
+        };
+    }
+}
diff --git a/healthmonitor/src/main/java/org/fao/geonet/monitor/health/NoIndexErrorsHealthCheck.java b/healthmonitor/src/main/java/org/fao/geonet/monitor/health/NoIndexErrorsHealthCheck.java
@@ -44,7 +44,7 @@ protected Result check() throws Exception {
                 GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
 
                 EsSearchManager searchMan = gc.getBean(EsSearchManager.class);
-                long numDocs = searchMan.getNumDocs("+" + IndexFields.INDEXING_ERROR_FIELD + ":1");
+                long numDocs = searchMan.getNumDocs("+" + IndexFields.INDEXING_ERROR_FIELD + ":true");
 
                 if (numDocs > 0) {
                     return Result.unhealthy("Found " + numDocs + " metadata that had errors during indexing");

diff --git a/web-ui/src/main/resources/catalog/js/admin/DashboardStatusController.js b/web-ui/src/main/resources/catalog/js/admin/DashboardStatusController.js
@@ -54,6 +54,16 @@
       $scope.threadSortField = undefined;
       $scope.threadSortReverse = false;
       $scope.threadInfoLoading = false;
+
+      $scope.indexStatus = null;
+      function getIndexStatus() {
+        $http.get('../api/site/index/synchronized').
+        success(function(data) {
+          $scope.indexStatus = data;
+        });
+      }
+      getIndexStatus();
+
       $scope.setThreadSortField = function(field) {
         if (field === $scope.threadSortField) {
           $scope.threadSortReverse = !$scope.threadSortReverse;

diff --git a/web-ui/src/main/resources/catalog/locales/en-v4.json b/web-ui/src/main/resources/catalog/locales/en-v4.json
@@ -154,5 +154,8 @@
   "facets.temporalRange.seriesLegend": "Blue series: current search, Grey series: all records",
   "recordFormats": "Distribution formats:",
   "recordFormatDownload": "Download as ",
-  "commonProtocols": "General protocols:"
+  "commonProtocols": "General protocols:",
+  "indexStatusSynchronized": "Records in index/db = <strong>{{indexStatus['index.count']}}/{{indexStatus['db.count']}}</strong>",
+  "indexStatusSynchronized-error": "When number of records is not equal in the database and in the index, check first that the catalogue point to the correct index. If yes, try to reindex the catalogue from admin > tools. If there is still a difference, check for indexing errors in the log file and fix the metadata record using the XML view.",
+  "IndexReadOnlyHealthCheck": "Index is in readonly mode"
 }
diff --git a/web-ui/src/main/resources/catalog/templates/admin/dashboard/status.html b/web-ui/src/main/resources/catalog/templates/admin/dashboard/status.html
@@ -31,6 +31,7 @@ <h4 class="list-group-item-heading" translate> {{info.name}} </h4>
               <td><span class="badge"
                         data-ng-class="info.status === 'OK' ? 'badge-success' : 'badge-danger'"> {{info.status}} </span>
               </td>
+            </tr>
           </table>
         </div>
       </div>
@@ -64,6 +65,25 @@ <h4 class="list-group-item-heading" translate> {{info.name}} </h4>
               <td><span class="badge"
                         data-ng-class="info.status === 'OK' ? 'badge-success' : 'badge-warning'"> {{info.status}} </span>
               </td>
+            </tr>
+            <tr class="health-check"
+                data-ng-show="indexStatus"
+                title="{{(indexStatus['db.count'] === indexStatus['index.count']) ? '' : ('indexStatusSynchronized-error' | translate)}}"
+                data-ng-class="(indexStatus['db.count'] === indexStatus['index.count']) ? '' : 'alert-danger'">
+              <td>
+                <h4 class="list-group-item-heading"
+                    data-translate=""
+                    data-translate-values="{indexStatus: indexStatus}">
+                  indexStatusSynchronized
+                </h4>
+              </td>
+              <td>
+                <span class="badge"
+                      data-ng-class="(indexStatus['db.count'] === indexStatus['index.count']) ? 'badge-success' : 'badge-danger'">
+                  {{(indexStatus['db.count'] === indexStatus['index.count']) ? 'OK' : 'ERROR'}}
+                </span>
+              </td>
+            </tr>
           </table>
         </div>
       </div>

diff --git a/web/src/main/webapp/WEB-INF/config/config-service-monitoring.xml b/web/src/main/webapp/WEB-INF/config/config-service-monitoring.xml
@@ -36,6 +36,7 @@
     <warningHealthCheck class=".HarvestersHealthCheck"/>
     <warningHealthCheck class=".DashboardAppHealthCheck"/>
     <warningHealthCheck class=".NoIndexErrorsHealthCheck"/>
+    <warningHealthCheck class=".IndexReadOnlyHealthCheck"/>
     <warningHealthCheck class=".FreeConnectionsHealthCheck"/>
     <warningHealthCheck class="jeeves.monitor.healthcheck.FreeFileHandlesHealthCheck"/>
     <expensiveHealthCheck class=".CswGetRecordsHealthCheck"/>