Skip to content

Commit

Permalink
Merge branch 'STORM-2453' of https://github.com/HeartSaVioR/storm int…
Browse files Browse the repository at this point in the history
…o STORM-2453-merge
  • Loading branch information
HeartSaVioR committed Apr 5, 2017
2 parents e84b39d + e9d7833 commit 1dd4bbc
Show file tree
Hide file tree
Showing 209 changed files with 46 additions and 118 deletions.
2 changes: 1 addition & 1 deletion bin/storm-kafka-monitor
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,4 @@ if [ -z "$JAVA_HOME" ]; then
else
JAVA="$JAVA_HOME/bin/java"
fi
exec $JAVA $STORM_JAAS_CONF_PARAM -cp $STORM_BASE_DIR/toollib/storm-kafka-monitor-*.jar org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"
exec $JAVA $STORM_JAAS_CONF_PARAM -cp $STORM_BASE_DIR/lib-tools/storm-kafka-monitor/storm-kafka-monitor-*.jar org.apache.storm.kafka.monitor.KafkaOffsetLagUtil "$@"
9 changes: 4 additions & 5 deletions bin/storm.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ def init_storm_env():

STORM_LIB_DIR = os.path.join(STORM_DIR, "lib")
STORM_DRPC_LIB_DIR = os.path.join(STORM_DIR, "lib-drpc-server")
STORM_TOOLS_LIB_DIR = os.path.join(STORM_DIR, "lib-tools")
STORM_BIN_DIR = os.path.join(STORM_DIR, "bin")
STORM_LOG4J2_CONF_DIR = os.path.join(STORM_DIR, "log4j2")
STORM_SUPERVISOR_LOG_FILE = os.getenv('STORM_SUPERVISOR_LOG_FILE', "supervisor.log")
Expand Down Expand Up @@ -167,10 +168,8 @@ def resolve_dependencies(artifacts, artifact_repositories):
print("Resolving dependencies on demand: artifacts (%s) with repositories (%s)" % (artifacts, artifact_repositories))
sys.stdout.flush()

# TODO: should we move some external modules to outer place?

# storm-submit module doesn't rely on storm-core and relevant libs
extrajars = get_jars_full(STORM_DIR + "/external/storm-submit-tools")
extrajars = get_jars_full(os.path.join(STORM_TOOLS_LIB_DIR, "submit-tools"))
classpath = normclasspath(os.pathsep.join(extrajars))

command = [
Expand Down Expand Up @@ -343,8 +342,8 @@ def sql(sql_file, topology_name):
local_jars = DEP_JARS_OPTS
artifact_to_file_jars = resolve_dependencies(DEP_ARTIFACTS_OPTS, DEP_ARTIFACTS_REPOSITORIES_OPTS)

sql_core_jars = get_jars_full(STORM_DIR + "/external/sql/storm-sql-core")
sql_runtime_jars = get_jars_full(STORM_DIR + "/external/sql/storm-sql-runtime")
sql_core_jars = get_jars_full(os.path.join(STORM_TOOLS_LIB_DIR, "sql", "core"))
sql_runtime_jars = get_jars_full(os.path.join(STORM_TOOLS_LIB_DIR, "sql", "runtime"))

# include storm-sql-runtime jar(s) to local jar list
local_jars.extend(sql_runtime_jars)
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.Map;
import java.util.Properties;

/**
* Flux entry point.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@
*/
package org.apache.storm.flux.model;

import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
* A representation of a Java object that is uniquely identifyable, and given a className, constructor arguments,
* and properties, can be instantiated.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package org.apache.storm.flux.parser;

import org.apache.storm.flux.api.TopologySource;
import org.apache.storm.flux.model.BoltDef;
import org.apache.storm.flux.model.IncludeDef;
import org.apache.storm.flux.model.SpoutDef;
Expand All @@ -32,7 +31,6 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.Properties;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@
import org.apache.storm.flux.test.TestBolt;
import org.junit.Test;

import java.io.File;

import static org.junit.Assert.*;

public class TCKTest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import org.apache.storm.generated.StormTopology;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.flux.api.TopologySource;
import org.apache.storm.flux.wrappers.bolts.LogInfoBolt;
import org.apache.storm.flux.wrappers.spouts.FluxShellSpout;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
import org.apache.storm.flux.wrappers.bolts.LogInfoBolt;
import org.apache.storm.flux.wrappers.spouts.FluxShellSpout;

import java.util.Map;

/**
* Test topology source that does not implement TopologySource, but has the same
* `getTopology()` method.
Expand Down
File renamed without changes.
10 changes: 2 additions & 8 deletions examples/flux-examples/pom.xml → flux/flux-examples/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@
<modelVersion>4.0.0</modelVersion>

<parent>
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<artifactId>flux</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>flux-examples</artifactId>
Expand All @@ -31,12 +31,6 @@
<name>flux-examples</name>

<dependencies>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${project.version}</version>
<scope>${provided.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>flux-core</artifactId>
Expand Down
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion external/flux/pom.xml → flux/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>

<developers>
Expand Down
6 changes: 3 additions & 3 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,9 @@
<module>storm-drpc-server</module>
<module>storm-rename-hack</module>
<module>storm-clojure</module>
<module>storm-submit-tools</module>
<module>flux</module>
<module>sql</module>

<!-- externals -->
<module>external/storm-kafka</module>
Expand All @@ -332,8 +335,6 @@
<module>external/storm-jdbc</module>
<module>external/storm-redis</module>
<module>external/storm-eventhubs</module>
<module>external/flux</module>
<module>external/sql</module>
<module>external/storm-elasticsearch</module>
<module>external/storm-solr</module>
<module>external/storm-metrics</module>
Expand All @@ -345,7 +346,6 @@
<module>external/storm-kafka-monitor</module>
<module>external/storm-kinesis</module>
<module>external/storm-druid</module>
<module>external/storm-submit-tools</module>
<module>external/storm-jms</module>
<module>external/storm-pmml</module>

Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion external/sql/pom.xml → sql/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
<relativePath>../pom.xml</relativePath>
</parent>

<artifactId>sql</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
<relativePath>../../pom.xml</relativePath>
</parent>

<artifactId>storm-sql-core</artifactId>
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,10 @@

import org.apache.calcite.DataContext;
import org.apache.storm.sql.javac.CompilingClassLoader;
import org.apache.storm.sql.runtime.ISqlTridentDataSource;
import org.apache.storm.trident.Stream;
import org.apache.storm.trident.TridentTopology;

import java.util.List;
import java.util.Map;

public abstract class AbstractTridentProcessor {
protected Stream outputStream;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package org.apache.storm.sql.parser;

import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.SqlWriter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Calc;
import org.apache.calcite.rex.RexProgram;
import org.apache.storm.sql.planner.rel.StormRelNode;

public abstract class StormCalcRelBase extends Calc implements StormRelNode {
protected StormCalcRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexProgram program) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rex.RexNode;
import org.apache.storm.sql.planner.rel.StormRelNode;

public abstract class StormFilterRelBase extends Filter implements StormRelNode {
protected StormFilterRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rex.RexNode;
import org.apache.storm.sql.planner.rel.StormRelNode;

import java.util.Set;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexNode;
import org.apache.storm.sql.planner.rel.StormRelNode;

import java.util.List;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.TableModify;
import org.apache.calcite.rex.RexNode;
import org.apache.storm.sql.planner.rel.StormRelNode;

import java.util.List;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.calcite.plan.RelOptTable;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.core.TableScan;
import org.apache.storm.sql.planner.rel.StormRelNode;

public abstract class StormStreamScanRelBase extends TableScan implements StormRelNode {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import org.apache.calcite.DataContext;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.interpreter.Scalar;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
Expand All @@ -36,7 +35,6 @@
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package org.apache.storm.sql.planner.trident.rel;

import org.apache.calcite.plan.Convention;
import org.apache.storm.sql.planner.rel.StormRelNode;
import org.apache.storm.sql.planner.trident.TridentPlanCreator;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,8 @@
package org.apache.storm.sql.compiler.backends.trident;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.calcite.DataContext;
import org.apache.calcite.avatica.util.DateTimeUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.LocalCluster.LocalTopology;
Expand All @@ -45,7 +43,6 @@
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../../../pom.xml</relativePath>
<relativePath>../../../pom.xml</relativePath>
</parent>

<artifactId>storm-sql-hdfs</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../../../pom.xml</relativePath>
<relativePath>../../../pom.xml</relativePath>
</parent>

<artifactId>storm-sql-kafka</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../../../pom.xml</relativePath>
<relativePath>../../../pom.xml</relativePath>
</parent>

<artifactId>storm-sql-mongodb</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../../../pom.xml</relativePath>
<relativePath>../../../pom.xml</relativePath>
</parent>

<artifactId>storm-sql-redis</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<artifactId>storm</artifactId>
<groupId>org.apache.storm</groupId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
<relativePath>../../pom.xml</relativePath>
</parent>

<artifactId>storm-sql-runtime</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@
package org.apache.storm.sql.runtime;

import org.apache.storm.tuple.Values;
import org.apache.storm.sql.runtime.ChannelHandler;
import org.apache.storm.sql.runtime.DataSource;

import java.util.Map;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.apache.storm.sql.runtime.datasource.socket.trident.SocketState;
import org.apache.storm.sql.runtime.datasource.socket.trident.SocketStateUpdater;
import org.apache.storm.sql.runtime.datasource.socket.trident.TridentSocketSpout;
import org.apache.storm.sql.runtime.serde.json.JsonSerializer;
import org.apache.storm.sql.runtime.utils.FieldInfoUtils;
import org.apache.storm.sql.runtime.utils.SerdeUtils;
import org.apache.storm.trident.spout.ITridentDataSource;
Expand Down
Loading

0 comments on commit 1dd4bbc

Please sign in to comment.