finos#11 Added optimized row publish, only rows that have changed. th…

…ere are still further optimizations to come.
keikeicheung · Jan 30, 2021 · 5fc0ff6 · 5fc0ff6
1 parent 9009e63
commit 5fc0ff6
Showing 14 changed files with 636 additions and 67 deletions.
diff --git a/.idea/compiler.xml b/.idea/compiler.xml
diff --git a/.idea/encodings.xml b/.idea/encodings.xml
diff --git a/.idea/scala_compiler.xml b/.idea/scala_compiler.xml
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/benchmark/pom.xml b/benchmark/pom.xml
@@ -0,0 +1,220 @@
+<!--
+Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved.
+DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License version 2 only, as
+published by the Free Software Foundation.  Oracle designates this
+particular file as subject to the "Classpath" exception as provided
+by Oracle in the LICENSE file that accompanied this code.
+
+This code is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+version 2 for more details (a copy is included in the LICENSE file that
+accompanied this code).
+
+You should have received a copy of the GNU General Public License version
+2 along with this work; if not, write to the Free Software Foundation,
+Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+or visit www.oracle.com if you need additional information or have any
+questions.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>io.venuu</groupId>
+    <artifactId>benchmark</artifactId>
+    <version>0.1-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <name>Auto-generated JMH benchmark</name>
+
+    <parent>
+        <groupId>io.venuu</groupId>
+        <artifactId>vuu-parent</artifactId>
+        <version>0.1-SNAPSHOT</version>
+    </parent>
+
+    <prerequisites>
+        <maven>3.0</maven>
+    </prerequisites>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.12</artifactId>
+            <version>${scalatest.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+
+    </dependencies>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <jmh.version>1.0</jmh.version>
+        <javac.target>1.6</javac.target>
+        <uberjar.name>benchmarks</uberjar.name>
+    </properties>
+
+    <build>
+        <plugins>
+
+            <plugin>
+                <groupId>org.scala-tools</groupId>
+                <artifactId>maven-scala-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <sourceDir>src/main/scala</sourceDir>
+                    <testSourceDir>src/test/scala</testSourceDir>
+                    <jvmArgs>
+                        <jvmArg>-Xms256m</jvmArg>
+                        <jvmArg>-Xmx4g</jvmArg>
+                    </jvmArgs>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.1</version>
+                <configuration>
+                    <compilerVersion>${javac.target}</compilerVersion>
+                    <source>${javac.target}</source>
+                    <target>${javac.target}</target>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.7</version>
+                <configuration>
+                    <!--skipTests>true</skipTests-->
+                </configuration>
+            </plugin>
+            <!-- enable scalatest -->
+            <plugin>
+                <groupId>org.scalatest</groupId>
+                <artifactId>scalatest-maven-plugin</artifactId>
+                <version>1.0</version>
+                <configuration>
+                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+                    <junitxml>.</junitxml>
+                    <filereports>test-reports.txt</filereports>
+                    <!--argLine>-Xbootclasspath/p:/Users/chris/.m2/repository/org/mortbay/jetty/alpn/alpn-boot/8.1.3.v20150130/alpn-boot-8.1.3.v20150130.jar -XX:+UnlockCommercialFeatures -Xmx4G -Xms2G -XX:+FlightRecorder -XX:+UseG1GC -verbosegc</argLine-->
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>test</id>
+                        <goals>
+                            <goal>test</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>2.2</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>${uberjar.name}</finalName>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.openjdk.jmh.Main</mainClass>
+                                </transformer>
+                            </transformers>
+                            <filters>
+                                <filter>
+                                    <!--
+                                        Shading signed JARs will fail without this.
+                                        http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar
+                                    -->
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+        <!--pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-clean-plugin</artifactId>
+                    <version>2.5</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-deploy-plugin</artifactId>
+                    <version>2.8.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-install-plugin</artifactId>
+                    <version>2.5.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-jar-plugin</artifactId>
+                    <version>2.4</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-javadoc-plugin</artifactId>
+                    <version>2.9.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-resources-plugin</artifactId>
+                    <version>2.6</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-site-plugin</artifactId>
+                    <version>3.3</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-source-plugin</artifactId>
+                    <version>2.2.1</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>2.17</version>
+                </plugin>
+            </plugins>
+        </pluginManagement-->
+    </build>
+
+</project>
diff --git a/benchmark/src/test/scala/io/venuu/vuu/benchmark/QueueBenchmark.scala b/benchmark/src/test/scala/io/venuu/vuu/benchmark/QueueBenchmark.scala
@@ -0,0 +1,26 @@
+package io.venuu.vuu.benchmark
+
+import org.openjdk.jmh.annotations.Benchmark
+import org.scalatest.{FeatureSpec, Matchers}
+
+class QueueBenchmark extends FeatureSpec with Matchers {
+
+  feature("test"){
+    scenario("test2"){
+
+      val argv = Array[String]()
+
+      org.openjdk.jmh.Main.main(argv)
+
+    }
+
+  }
+
+  @Benchmark
+  def runSimpleBenchmark(): Unit ={
+
+    (1 to 10000).foreach( i => Math.asin(i))
+
+  }
+
+}
diff --git a/pom.xml b/pom.xml
@@ -32,8 +32,9 @@
     <modules>
         <module>toolbox</module>
         <module>vuu</module>
-        <module>murmur</module>
-        <module>zk</module>
+        <!--module>murmur</module-->
+        <!--module>zk</module-->
+        <!--module>benchmark</module-->
         <!--module>allocation-tools</module-->
     </modules>
 

diff --git a/vuu/pom.xml b/vuu/pom.xml
@@ -14,7 +14,6 @@
     <packaging>jar</packaging>
 
     <dependencies>
-
         <dependency>
             <groupId>io.venuu</groupId>
             <artifactId>toolbox</artifactId>

diff --git a/vuu/src/main/scala/io/venuu/vuu/provider/simulation/SimulatedBigInstrumentsProvider.scala b/vuu/src/main/scala/io/venuu/vuu/provider/simulation/SimulatedBigInstrumentsProvider.scala
@@ -15,6 +15,7 @@ import io.venuu.toolbox.time.Clock
 import io.venuu.vuu.core.table.{DataTable, RowWithData}
 import io.venuu.vuu.provider.Provider
 
+import scala.concurrent.duration.DurationInt
 import scala.util.Random
 
 class SimulatedBigInstrumentsProvider(table: DataTable)(implicit timeProvider: Clock, lifecycle:  LifecycleContainer) extends Provider with StrictLogging {
@@ -27,10 +28,10 @@ class SimulatedBigInstrumentsProvider(table: DataTable)(implicit timeProvider: C
 
   override def subscribe(key: String): Unit = {}
 
+  //def charMaker = ((48 to 57).toSeq ++ ( 65 to 90 )).map( i => i.toChar )
   def charMaker = ( 65 to 90 ).map( i => i.toChar )
-  //def charMaker = ( 65 to 66 ).map( i => i.toChar )
 
-  def suffixes  = List(".L", ".N", ".OQ", ".AS")
+  def suffixes  = List(".L", ".N", ".OQ", ".AS", ".OE", ".MI", ".A", ".PA", ".MC", ".DE")
 
   def ricBuilder = for( c1 <- charMaker; c2 <- charMaker ; c3 <- charMaker; suff <- suffixes) yield new String(Array(c1.toChar, c2.toChar, c3.toChar ) ) + suff
 
@@ -59,6 +60,9 @@ class SimulatedBigInstrumentsProvider(table: DataTable)(implicit timeProvider: C
   }
 
   def build() = {
+
+    Thread.sleep(5.seconds.toMillis)
+
     val rics = ricBuilder
 
     val rows = rics.map(mkRow(_))

diff --git a/vuu/src/main/scala/io/venuu/vuu/viewport/ViewPort.scala b/vuu/src/main/scala/io/venuu/vuu/viewport/ViewPort.scala
@@ -38,6 +38,24 @@ case class ViewPortRange(from: Int, to: Int){
   def contains(i: Int): Boolean = {
     i >= from && i < to
   }
+
+  def subtract(newRange: ViewPortRange): ViewPortRange ={
+        var from = newRange.from
+        var to = newRange.to
+
+        if(newRange.from > this.from && newRange.from < this.to){
+            from = this.to
+            to = newRange.to
+        }
+
+        if(newRange.from < this.from && newRange.to < this.to && newRange.to > this.from){
+            from = newRange.from
+            to   = this.from
+        }
+
+        ViewPortRange(from, to)
+  }
+
 }
 
 case class ViewPortUpdate(vp: ViewPort, table: RowSource, key: RowKeyUpdate, index: Int, vpUpdate: ViewPortUpdateType, ts: Long)
@@ -97,23 +115,23 @@ case class ViewPortImpl(id: String,
     structuralFields.set(newStructuralFields)
 
     if(!onlySortOrFilterChange)
-      sendUpdatesOnChange()
+      sendUpdatesOnChange(range.get())
   }
 
   def setRange(newRange: ViewPortRange): Unit = {
+    val oldRange = range.get()
     range.set(newRange)
-    sendUpdatesOnChange()
+    val diffRange = oldRange.subtract(newRange)
+    sendUpdatesOnChange(diffRange)
   }
 
   //def setColumns(columns: List[Column])
   override def filterSpec: FilterSpec = structuralFields.get().filterSpec
 
-  def sendUpdatesOnChange() = {
+  def sendUpdatesOnChange(currentRange: ViewPortRange) = {
 
     val currentKeys = keys.toArray
 
-    val currentRange = range.get()
-
     val from = currentRange.from
     val to = currentRange.to