use diskindex instead of onheapIndex

BetFutureProject · May 4, 2018 · b493ef9 · b493ef9
1 parent 3c0452c
commit b493ef9
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 10 deletions.
diff --git a/.gitignore b/.gitignore
@@ -49,5 +49,6 @@ src/main/resources/META-INF/
 
 /output_manager/
 /output_witness/
+out-index
 
 nodeId.properties
diff --git a/src/main/java/org/tron/core/db/api/index/AbstractIndex.java b/src/main/java/org/tron/core/db/api/index/AbstractIndex.java
@@ -1,6 +1,7 @@
 package org.tron.core.db.api.index;
 
 import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
 import com.googlecode.cqengine.ConcurrentIndexedCollection;
 import com.googlecode.cqengine.persistence.Persistence;
 import com.googlecode.cqengine.query.Query;
@@ -12,14 +13,25 @@
 import org.tron.core.db.common.WrappedByteArray;
 import org.tron.core.db.common.WrappedResultSet;
 
+import java.io.File;
+import java.io.IOException;
 import java.util.Iterator;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 public abstract class AbstractIndex<E extends ProtoCapsule, T> implements Iface<T> {
 
   protected TronDatabase<E> database;
   protected ConcurrentIndexedCollection<WrappedByteArray> index;
+  private File parent = new File("out-index");
+  protected File indexPath;
+  private ExecutorService service = Executors.newSingleThreadExecutor();
 
   public AbstractIndex() {
+    if (!parent.exists()) {
+      parent.mkdirs();
+    }
+    indexPath = new File(parent, getName() + ".index");
     setAttribute();
   }
 
@@ -47,7 +59,10 @@ protected T getObject(final WrappedByteArray byteArray) {
   }
 
   protected void fill() {
-    database.forEach(e -> index.add(WrappedByteArray.of(e.getKey())));
+    int size = Iterables.size(database);
+    if (!indexPath.exists() || index.size() != size) {
+      service.execute(() -> database.forEach(e -> add(e.getKey())));
+    }
   }
 
   @Override
@@ -56,7 +71,7 @@ public boolean add(byte[] bytes) {
   }
 
   @Override
-  public boolean add(WrappedByteArray bytes) {
+  public synchronized boolean add(WrappedByteArray bytes) {
     return index.add(bytes);
   }
 

diff --git a/src/main/java/org/tron/core/db/api/index/AccountIndex.java b/src/main/java/org/tron/core/db/api/index/AccountIndex.java
@@ -1,5 +1,6 @@
 package org.tron.core.db.api.index;
 
+import com.google.common.io.Files;
 import com.googlecode.cqengine.attribute.Attribute;
 import com.googlecode.cqengine.attribute.SimpleAttribute;
 import com.googlecode.cqengine.index.disk.DiskIndex;
@@ -20,6 +21,8 @@
 
 import javax.annotation.PostConstruct;
 
+import java.io.File;
+
 import static com.googlecode.cqengine.query.QueryFactory.attribute;
 
 @Component
@@ -35,7 +38,7 @@ public AccountIndex(@Qualifier("accountStore") final TronDatabase<AccountCapsule
 
   @PostConstruct
   public void init() {
-    initIndex(DiskPersistence.onPrimaryKey(Account_ADDRESS));
+    initIndex(DiskPersistence.onPrimaryKeyInFile(Account_ADDRESS, indexPath));
 //    index.addIndex(DiskIndex.onAttribute(Account_ADDRESS));
     fill();
   }

diff --git a/src/main/java/org/tron/core/db/api/index/AssetIssueIndex.java b/src/main/java/org/tron/core/db/api/index/AssetIssueIndex.java
@@ -20,6 +20,8 @@
 
 import javax.annotation.PostConstruct;
 
+import java.io.File;
+
 import static com.googlecode.cqengine.query.QueryFactory.attribute;
 
 @Component
@@ -39,7 +41,7 @@ public AssetIssueIndex(
 
   @PostConstruct
   public void init() {
-    initIndex(DiskPersistence.onPrimaryKey(AssetIssue_NAME));
+    initIndex(DiskPersistence.onPrimaryKeyInFile(AssetIssue_NAME, indexPath));
     index.addIndex(DiskIndex.onAttribute(AssetIssue_OWNER_RADDRESS));
 //    index.addIndex(DiskIndex.onAttribute(AssetIssue_NAME));
     index.addIndex(DiskIndex.onAttribute(AssetIssue_START));

diff --git a/src/main/java/org/tron/core/db/api/index/BlockIndex.java b/src/main/java/org/tron/core/db/api/index/BlockIndex.java
@@ -21,6 +21,7 @@
 import org.tron.protos.Protocol.Block;
 
 import javax.annotation.PostConstruct;
+import java.io.File;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -47,7 +48,7 @@ public BlockIndex(
 
   @PostConstruct
   public void init() {
-    initIndex(DiskPersistence.onPrimaryKey(Block_ID));
+    initIndex(DiskPersistence.onPrimaryKeyInFile(Block_ID, indexPath));
 //    index.addIndex(DiskIndex.onAttribute(Block_ID));
     index.addIndex(DiskIndex.onAttribute(Block_NUMBER));
     index.addIndex(DiskIndex.onAttribute(TRANSACTIONS));

diff --git a/src/main/java/org/tron/core/db/api/index/TransactionIndex.java b/src/main/java/org/tron/core/db/api/index/TransactionIndex.java
@@ -21,6 +21,7 @@
 import org.tron.protos.Protocol.Transaction;
 
 import javax.annotation.PostConstruct;
+import java.io.File;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
@@ -43,7 +44,7 @@ public TransactionIndex(
 
   @PostConstruct
   public void init() {
-    initIndex(DiskPersistence.onPrimaryKey(Transaction_ID));
+    initIndex(DiskPersistence.onPrimaryKeyInFile(Transaction_ID, indexPath));
 //    index.addIndex(DiskIndex.onAttribute(Transaction_ID));
     index.addIndex(DiskIndex.onAttribute(OWNERS));
     index.addIndex(DiskIndex.onAttribute(TOS));

diff --git a/src/main/java/org/tron/core/db/api/index/WitnessIndex.java b/src/main/java/org/tron/core/db/api/index/WitnessIndex.java
@@ -19,6 +19,8 @@
 
 import javax.annotation.PostConstruct;
 
+import java.io.File;
+
 import static com.googlecode.cqengine.query.QueryFactory.attribute;
 
 @Component
@@ -37,7 +39,7 @@ public WitnessIndex(
 
   @PostConstruct
   public void init() {
-    initIndex(DiskPersistence.onPrimaryKey(Witness_ADDRESS));
+    initIndex(DiskPersistence.onPrimaryKeyInFile(Witness_ADDRESS, indexPath));
 //    index.addIndex(DiskIndex.onAttribute(Witness_ADDRESS));
     index.addIndex(DiskIndex.onAttribute(PUBLIC_KEY));
     index.addIndex(DiskIndex.onAttribute(Witness_URL));

diff --git a/src/main/java/org/tron/program/FullNode.java b/src/main/java/org/tron/program/FullNode.java
@@ -42,8 +42,8 @@ public static void main(String[] args) throws InterruptedException {
     rpcApiService.blockUntilShutdown();
   }
 
-  private static void shutdown(final Application app) {
-    logger.info("********register application shutdown ********");
+  public static void shutdown(final Application app) {
+    logger.info("********register application shutdown hook********");
     Runtime.getRuntime().addShutdownHook(new Thread(app::shutdown));
   }
 }
diff --git a/src/main/java/org/tron/program/SolidityNode.java b/src/main/java/org/tron/program/SolidityNode.java
@@ -51,7 +51,7 @@ private void syncLoop(Args args) {
         initGrpcClient(args.getTrustNodeAddr());
         syncSolidityBlock();
       } catch (Exception e) {
-        logger.error("Error in sync solidity block {}", e.getMessage());
+        logger.error("Error in sync solidity block " + e.getMessage(), e);
       }
       try {
         Thread.sleep(5000);
@@ -115,6 +115,7 @@ public static void main(String[] args) throws InterruptedException {
       return;
     }
     Application appT = ApplicationFactory.create(context);
+    FullNode.shutdown(appT);
     //appT.init(cfgArgs);
     RpcApiService rpcApiService = context.getBean(RpcApiService.class);
     appT.addService(rpcApiService);