From c8cdfd8b1c80e794ddacf69dba413e16018b9c1c Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Tue, 16 May 2017 14:30:59 -0400 Subject: [PATCH 1/7] should eventually turn off split on whitespace in the query parser --- .../main/java/org/lumongo/server/search/LumongoQueryParser.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lumongo-cluster/src/main/java/org/lumongo/server/search/LumongoQueryParser.java b/lumongo-cluster/src/main/java/org/lumongo/server/search/LumongoQueryParser.java index 35615954..99a3e3fd 100644 --- a/lumongo-cluster/src/main/java/org/lumongo/server/search/LumongoQueryParser.java +++ b/lumongo-cluster/src/main/java/org/lumongo/server/search/LumongoQueryParser.java @@ -35,6 +35,8 @@ public LumongoQueryParser(Analyzer analyzer, IndexConfig indexConfig) { super(indexConfig.getIndexSettings().getDefaultSearchField(), analyzer); this.indexConfig = indexConfig; setAllowLeadingWildcard(true); + //TODO consider turning this on + //setSplitOnWhitespace(false); } private static Long getDateAsLong(String dateString) { From 9f3b99fade7c7f2cf81056b8bb8f1d3719f2d0e3 Mon Sep 17 00:00:00 2001 From: Matthew Davis Date: Sun, 21 May 2017 13:16:57 -0400 Subject: [PATCH 2/7] update to lucene 6.6-rc1 --- build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 121b53bc..2b70d4b0 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,7 @@ subprojects { version = '0.53-SNAPSHOT' ext.isReleaseVersion = !version.endsWith("SNAPSHOT") - ext.luceneVersion = '6.5.1' + ext.luceneVersion = '6.6.0' ext.mongoJavaVersion = '3.4.1' group = 'org.lumongo' @@ -34,6 +34,9 @@ subprojects { mavenCentral() flatDir name: 'directoryRepository', dirs: rootProject.file('notInMaven') + maven { + url "https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-6.6.0-RC1-rev4d055f00bba9a745737e4b6c3f9dff06dd35aa2e/lucene/maven/" + } } dependencies { From 1f2dbc243f3748d9a10022a93ca8c8724784006f Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 23 May 2017 12:12:25 -0400 Subject: [PATCH 3/7] allow cursor in query resource --- .../lumongo/server/rest/QueryResource.java | 19 ++++++++++++++++--- .../java/org/lumongo/LumongoConstants.java | 1 + 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java b/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java index 6afbef48..b3f326f3 100644 --- a/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java +++ b/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java @@ -7,6 +7,7 @@ import org.apache.log4j.Logger; import org.bson.Document; import org.lumongo.LumongoConstants; +import org.lumongo.client.command.CursorHelper; import org.lumongo.cluster.message.Lumongo; import org.lumongo.cluster.message.Lumongo.AnalyzerSettings.Similarity; import org.lumongo.cluster.message.Lumongo.CountRequest; @@ -59,10 +60,17 @@ public Response get(@QueryParam(LumongoConstants.INDEX) List indexName, @QueryParam(LumongoConstants.HIGHLIGHT_JSON) List highlightJsonList, @QueryParam(LumongoConstants.ANALYZE_JSON) List analyzeJsonList, @QueryParam(LumongoConstants.COS_SIM_JSON) List cosineSimJsonList, @QueryParam(LumongoConstants.FORMAT) @DefaultValue("json") String format, @QueryParam(LumongoConstants.BATCH) boolean batch, - @QueryParam(LumongoConstants.BATCH_SIZE) @DefaultValue("500") Integer batchSize) { + @QueryParam(LumongoConstants.BATCH_SIZE) @DefaultValue("500") Integer batchSize, @QueryParam(LumongoConstants.CURSOR) String cursor) { QueryRequest.Builder qrBuilder = QueryRequest.newBuilder().addAllIndex(indexName); + boolean outputCursor = true; + if (cursor != null) { + if (!cursor.equals("0")) { + qrBuilder.setLastResult(CursorHelper.getLastResultFromCursor(cursor)); + } + } + if (debug != null) { qrBuilder.setDebug(debug); } @@ -292,7 +300,7 @@ else if ("1".equals(sortDir) || "ASC".equalsIgnoreCase(sortDir)) { try { if (format.equals("json")) { QueryResponse qr = indexManager.query(qrBuilder.build()); - String response = getStandardResponse(qr, !pretty); + String response = getStandardResponse(qr, !pretty, outputCursor); if (pretty) { response = JsonWriter.formatJson(response); @@ -373,11 +381,16 @@ private void buildHeaderForCSV(@QueryParam(LumongoConstants.FIELDS) List } - private String getStandardResponse(QueryResponse qr, boolean strict) throws InvalidProtocolBufferException { + private String getStandardResponse(QueryResponse qr, boolean strict, boolean cursor) throws InvalidProtocolBufferException { StringBuilder responseBuilder = new StringBuilder(); responseBuilder.append("{"); responseBuilder.append("\"totalHits\": "); responseBuilder.append(qr.getTotalHits()); + if (cursor) { + responseBuilder.append(","); + responseBuilder.append("\"cursor\": "); + responseBuilder.append(CursorHelper.getUniqueSortedCursor(qr.getLastResult())); + } if (!qr.getAnalysisResultList().isEmpty()) { responseBuilder.append(","); diff --git a/lumongo-common/src/main/java/org/lumongo/LumongoConstants.java b/lumongo-common/src/main/java/org/lumongo/LumongoConstants.java index 594d4082..98e48a27 100644 --- a/lumongo-common/src/main/java/org/lumongo/LumongoConstants.java +++ b/lumongo-common/src/main/java/org/lumongo/LumongoConstants.java @@ -83,4 +83,5 @@ public class LumongoConstants { public static final String DONT_CACHE = "dontCache"; public static final String BATCH = "batch"; public static final String BATCH_SIZE = "batchSize"; + public static final String CURSOR = "cursor"; } From 0b9b7223bbb7d73d717494f2ba195c3902e33afb Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 23 May 2017 12:18:42 -0400 Subject: [PATCH 4/7] fix cursor logic --- .../main/java/org/lumongo/server/rest/QueryResource.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java b/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java index b3f326f3..816239bc 100644 --- a/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java +++ b/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java @@ -64,11 +64,12 @@ public Response get(@QueryParam(LumongoConstants.INDEX) List indexName, QueryRequest.Builder qrBuilder = QueryRequest.newBuilder().addAllIndex(indexName); - boolean outputCursor = true; + boolean outputCursor = false; if (cursor != null) { if (!cursor.equals("0")) { qrBuilder.setLastResult(CursorHelper.getLastResultFromCursor(cursor)); } + outputCursor = true; } if (debug != null) { @@ -388,8 +389,9 @@ private String getStandardResponse(QueryResponse qr, boolean strict, boolean cur responseBuilder.append(qr.getTotalHits()); if (cursor) { responseBuilder.append(","); - responseBuilder.append("\"cursor\": "); + responseBuilder.append("\"cursor\": \""); responseBuilder.append(CursorHelper.getUniqueSortedCursor(qr.getLastResult())); + responseBuilder.append("\""); } if (!qr.getAnalysisResultList().isEmpty()) { From 4db5ad9a1c418f3d869101510e3a0df5decee616 Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Tue, 23 May 2017 12:23:43 -0400 Subject: [PATCH 5/7] add better message --- .../src/main/java/org/lumongo/server/rest/QueryResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java b/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java index 816239bc..a07a829b 100644 --- a/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java +++ b/lumongo-cluster/src/main/java/org/lumongo/server/rest/QueryResource.java @@ -70,6 +70,10 @@ public Response get(@QueryParam(LumongoConstants.INDEX) List indexName, qrBuilder.setLastResult(CursorHelper.getLastResultFromCursor(cursor)); } outputCursor = true; + if (sort == null || sort.isEmpty()) { + return Response.status(LumongoConstants.INTERNAL_ERROR) + .entity("Sort on unique value or value combination is required to use a cursor (i.e. id or title,id)").build(); + } } if (debug != null) { From ada8c7fc6c34c0eac86e11063979373237f7e667 Mon Sep 17 00:00:00 2001 From: Payam Meyer Date: Mon, 5 Jun 2017 15:35:28 -0400 Subject: [PATCH 6/7] Fixed bug with deleting document from mongo even though it doesn't exist. --- .../lumongo/server/index/LumongoIndex.java | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/lumongo-cluster/src/main/java/org/lumongo/server/index/LumongoIndex.java b/lumongo-cluster/src/main/java/org/lumongo/server/index/LumongoIndex.java index 4dab4fd0..3dcad68d 100644 --- a/lumongo-cluster/src/main/java/org/lumongo/server/index/LumongoIndex.java +++ b/lumongo-cluster/src/main/java/org/lumongo/server/index/LumongoIndex.java @@ -34,30 +34,7 @@ import org.bson.Document; import org.lumongo.LumongoConstants; import org.lumongo.cluster.message.Lumongo; -import org.lumongo.cluster.message.Lumongo.AssociatedDocument; -import org.lumongo.cluster.message.Lumongo.DeleteRequest; -import org.lumongo.cluster.message.Lumongo.FetchType; -import org.lumongo.cluster.message.Lumongo.FieldConfig; -import org.lumongo.cluster.message.Lumongo.FieldSort; -import org.lumongo.cluster.message.Lumongo.GetFieldNamesResponse; -import org.lumongo.cluster.message.Lumongo.GetNumberOfDocsResponse; -import org.lumongo.cluster.message.Lumongo.GetTermsRequest; -import org.lumongo.cluster.message.Lumongo.GetTermsResponse; -import org.lumongo.cluster.message.Lumongo.GetTermsResponseInternal; -import org.lumongo.cluster.message.Lumongo.HighlightRequest; -import org.lumongo.cluster.message.Lumongo.IndexSegmentResponse; -import org.lumongo.cluster.message.Lumongo.IndexSettings; -import org.lumongo.cluster.message.Lumongo.LastIndexResult; -import org.lumongo.cluster.message.Lumongo.LastResult; -import org.lumongo.cluster.message.Lumongo.QueryRequest; -import org.lumongo.cluster.message.Lumongo.ResultDocument; -import org.lumongo.cluster.message.Lumongo.ScoredResult; -import org.lumongo.cluster.message.Lumongo.SegmentCountResponse; -import org.lumongo.cluster.message.Lumongo.SegmentResponse; -import org.lumongo.cluster.message.Lumongo.SortRequest; -import org.lumongo.cluster.message.Lumongo.SortValue; -import org.lumongo.cluster.message.Lumongo.SortValues; -import org.lumongo.cluster.message.Lumongo.StoreRequest; +import org.lumongo.cluster.message.Lumongo.*; import org.lumongo.server.config.ClusterConfig; import org.lumongo.server.config.IndexConfig; import org.lumongo.server.config.IndexConfigUtil; @@ -888,7 +865,9 @@ public void deleteDocument(DeleteRequest deleteRequest) throws Exception { if (deleteRequest.getDeleteDocument()) { LumongoSegment s = findSegmentFromUniqueId(deleteRequest.getUniqueId()); s.deleteDocument(uniqueId); - documentStorage.deleteSourceDocument(uniqueId); + if (indexConfig.getIndexSettings().getStoreDocumentInMongo()) { + documentStorage.deleteSourceDocument(uniqueId); + } } if (deleteRequest.getDeleteAllAssociated()) { From 79c02014d9a3d502a52970d0169e2e20888a25ae Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Wed, 7 Jun 2017 21:48:24 -0400 Subject: [PATCH 7/7] switch to 6.6 --- build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 2b70d4b0..59691a66 100644 --- a/build.gradle +++ b/build.gradle @@ -34,9 +34,9 @@ subprojects { mavenCentral() flatDir name: 'directoryRepository', dirs: rootProject.file('notInMaven') - maven { - url "https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-6.6.0-RC1-rev4d055f00bba9a745737e4b6c3f9dff06dd35aa2e/lucene/maven/" - } + //maven { + //url "https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-6.6.0-RC1-rev4d055f00bba9a745737e4b6c3f9dff06dd35aa2e/lucene/maven/" + //} } dependencies {