Merge pull request running-elephant#174 from scottsut/master

chore: alpha.2 release
xieliuduo · Nov 8, 2021 · d1f593c · d1f593c
2 parents c2cb7f6 + ee748fd
commit d1f593c
Show file tree

Hide file tree

Showing 212 changed files with 2,422 additions and 2,765 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,6 @@
+.idea
+
+/static
+
+*.iml
+*.zip
diff --git a/Dockerfile b/Dockerfile
@@ -4,6 +4,8 @@ RUN mkdir /datart
 COPY ./bin/* /datart/bin/
 COPY ./config/* /datart/config/
 COPY ./lib/* /datart/lib/
+COPY static /datart/static
+ENV TZ=Asia/Shanghai
 EXPOSE 58080
 WORKDIR /datart
 ENTRYPOINT java -cp "lib/*" datart.DatartServerApplication
diff --git a/bin/migrations/source.1.0.0-alpha.2.sql b/bin/migrations/source.1.0.0-alpha.2.sql
@@ -0,0 +1,2 @@
+ALTER TABLE `source`
+DROP INDEX `prj_name`;
diff --git a/config/jdbc-driver-ext.yml b/config/jdbc-driver-ext.yml
@@ -9,7 +9,7 @@
 IMPALA:
   db-type: "impala"
   name: "impala"
-  driver-class: "com.mysql.cj.jdbc.Driver"
+  driver-class: "com.cloudera.impala.jdbc41.Driver"
   literal-quote: "'"
   identifier-quote: "`"
   adapter-class: "datart.data.provider.jdbc.adapters.ImpalaDataProviderAdapter"

diff --git a/core/pom.xml b/core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-alpha.1</version>
+        <version>1.0.0-alpha.2</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/core/src/main/java/datart/core/base/PageInfo.java b/core/src/main/java/datart/core/base/PageInfo.java
@@ -32,6 +32,8 @@ public class PageInfo implements Serializable {
 
     private long total;
 
+    private boolean countTotal;
+
     @Override
     public String toString() {
         return "PageInfo{" +

diff --git a/core/src/main/java/datart/core/base/consts/ValueType.java b/core/src/main/java/datart/core/base/consts/ValueType.java
@@ -16,4 +16,5 @@ public enum ValueType {
 
     SNIPPET, //will be parse to sql node
 
+    KEYWORD,
 }
diff --git a/core/src/main/java/datart/core/common/CSVParser.java b/core/src/main/java/datart/core/common/CSVParser.java
@@ -21,6 +21,7 @@
 import datart.core.base.consts.ValueType;
 import datart.core.base.exception.BaseException;
 import lombok.Data;
+import org.apache.commons.lang.math.NumberUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.util.CollectionUtils;
 
@@ -95,11 +96,9 @@ private ValueType[] inferDataType(String line) {
         String[] split = line.split(CSV_SPLIT);
         ValueType[] valueTypes = new ValueType[split.length];
         for (int i = 0; i < split.length; i++) {
-            try {
-                Double.parseDouble(split[i]);
+            if (NumberUtils.isNumber(split[i])) {
                 valueTypes[i] = ValueType.NUMERIC;
                 continue;
-            } catch (Exception ignore) {
             }
             try {
                 simpleDateFormat.parse(split[i]);
@@ -115,7 +114,7 @@ private ValueType[] inferDataType(String line) {
     private List<Object> extractValues(String line) {
         try {
             if (StringUtils.isEmpty(line)) {
-                return Collections.EMPTY_LIST;
+                return Collections.emptyList();
             }
             LinkedList<Object> values = new LinkedList<>();
             String[] split = line.split(CSV_SPLIT);
@@ -142,7 +141,11 @@ private Object parseValue(String val, ValueType valueType) throws ParseException
             case DATE:
                 return simpleDateFormat.parse(val);
             case NUMERIC:
-                return Double.parseDouble(val);
+                if (NumberUtils.isDigits(val)) {
+                    return Long.parseLong(val);
+                } else {
+                    return Double.parseDouble(val);
+                }
             default:
                 return val;
         }

diff --git a/core/src/main/java/datart/core/common/FileUtils.java b/core/src/main/java/datart/core/common/FileUtils.java
@@ -21,10 +21,8 @@
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.File;
-import java.io.FileFilter;
 import java.util.Collections;
 import java.util.LinkedHashSet;
-import java.util.LinkedList;
 import java.util.Set;
 
 public class FileUtils {
@@ -92,6 +90,4 @@ public static Set<String> walkDir(File file, String extension, boolean recursion
             return names;
         }
     }
-
-
 }
diff --git a/core/src/main/java/datart/core/common/POIUtils.java b/core/src/main/java/datart/core/common/POIUtils.java
@@ -108,20 +108,26 @@ public static List<List<Object>> loadExcel(String path) throws IOException {
             // 只处理第一个sheet
             Sheet sheet = workbook.getSheetAt(0);
             Iterator<Row> rowIterator = sheet.rowIterator();
+            Row row0 = sheet.getRow(0);
+            if (row0 == null) {
+                throw new RuntimeException("excel is empty");
+            }
+            int columns = row0.getPhysicalNumberOfCells();
             while (rowIterator.hasNext()) {
                 Row row = rowIterator.next();
-                Iterator<Cell> iterator = row.iterator();
                 LinkedList<Object> cellValues = new LinkedList<>();
+                for (int i = 0; i < columns; i++)
+                    cellValues.add(readCellValue(row.getCell(i)));
                 rows.add(cellValues);
-                while (iterator.hasNext()) {
-                    cellValues.add(readCellValue(iterator.next()));
-                }
             }
         }
         return rows;
     }
 
     private static Object readCellValue(Cell cell) {
+        if (cell == null) {
+            return null;
+        }
         switch (cell.getCellType()) {
             case NUMERIC:
                 return cell.getNumericCellValue();
@@ -132,5 +138,4 @@ private static Object readCellValue(Cell cell) {
         }
     }
 
-
 }
diff --git a/core/src/main/java/datart/core/common/WebUtils.java b/core/src/main/java/datart/core/common/WebUtils.java
@@ -56,7 +56,7 @@ private static WebDriver createWebDriver() throws Exception {
         }
     }
 
-    public static <T> T screenShot(String url, OutputType<T> outputType) throws Exception {
+    public static <T> T screenShot(String url, OutputType<T> outputType, int imageWidth) throws Exception {
         WebDriver webDriver = createWebDriver();
         webDriver.get(url);
 
@@ -71,19 +71,24 @@ public static <T> T screenShot(String url, OutputType<T> outputType) throws Exce
 
         int contentHeight = Integer.parseInt(webDriver.findElement(By.id("height")).getAttribute("value"));
 
-        // scale the window
-        webDriver.manage().window().setSize(new Dimension(contentWidth, contentHeight));
-        try {
-            Thread.sleep(2 * 1000);
-        } catch (InterruptedException ignored) {
+        if (imageWidth != contentWidth) {
+            // scale the window
+            webDriver.manage().window().setSize(new Dimension(imageWidth, contentHeight));
+            Thread.sleep(1000);
         }
+        // scale the window again
+        contentWidth = Integer.parseInt(webDriver.findElement(By.id("width")).getAttribute("value"));
+        contentHeight = Integer.parseInt(webDriver.findElement(By.id("height")).getAttribute("value"));
+        webDriver.manage().window().setSize(new Dimension(contentWidth, contentHeight));
+        Thread.sleep(1000);
+
         TakesScreenshot screenshot = (TakesScreenshot) webDriver;
         return screenshot.getScreenshotAs(outputType);
     }
 
-    public static File screenShot2File(String url, String path) throws Exception {
+    public static File screenShot2File(String url, String path, int imageWidth) throws Exception {
 
-        File temp = screenShot(url, OutputType.FILE);
+        File temp = screenShot(url, OutputType.FILE, imageWidth);
         path = FileUtils.concatPath(path, temp.getName());
         File file = new File(path);
 

diff --git a/core/src/main/java/datart/core/data/provider/DataProvider.java b/core/src/main/java/datart/core/data/provider/DataProvider.java
@@ -91,8 +91,6 @@ public String getType() throws IOException {
         return getBaseInfo().getType();
     }
 
-    public abstract List<DataProviderSubType> getSubTypes();
-
     public abstract String getConfigFile();
 
     @Override

diff --git a/core/src/main/java/datart/core/mappers/ext/DatachartMapperExt.java b/core/src/main/java/datart/core/mappers/ext/DatachartMapperExt.java
@@ -27,9 +27,17 @@ public interface DatachartMapperExt extends DatachartMapper {
     List<Datachart> listByIds(Set<String> datachartIds);
 
     @Select({
-            "SELECT COUNT(*) FROM rel_widget_element rwe WHERE rel_type='DATACHART' AND rel_id=#{datachartId}"
+            "SELECT",
+            "COUNT(*)",
+            "FROM",
+            "rel_widget_element rwe ",
+            "JOIN widget w ON w.id = rwe.widget_id ",
+            "JOIN dashboard d ON w.dashboard_id = d.id  ",
+            "AND d.`status` !=0 ",
+            "WHERE ",
+            "rwe.rel_type = 'DATACHART'  ",
+            "AND rwe.rel_id = #{datachartId}"
     })
     int countWidgetRels(String datachartId);
-
-
+
 }
diff --git a/data-providers/file-data-provider/pom.xml b/data-providers/file-data-provider/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-alpha.1</version>
+        <version>1.0.0-alpha.2</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/data-providers/file-data-provider/src/main/java/datart/data/provider/FileDataProvider.java b/data-providers/file-data-provider/src/main/java/datart/data/provider/FileDataProvider.java
@@ -48,41 +48,35 @@ public class FileDataProvider extends DefaultDataProvider {
 
     public static final String FILE_PATH = "path";
 
-
     @Override
-    public List<Dataframe> loadFullDataFromSource(DataProviderSource config) {
-        try {
-            Map<String, Object> properties = config.getProperties();
-            List<Map<String, Object>> schemas;
-            if (properties.containsKey(SCHEMAS)) {
-                schemas = (List<Map<String, Object>>) properties.get(SCHEMAS);
-            } else {
-                schemas = Collections.singletonList(properties);
-            }
-            LinkedList<Dataframe> dataframes = new LinkedList<>();
-            for (Map<String, Object> schema : schemas) {
-                String path = schema.get(FILE_PATH).toString();
-                FileFormat fileFormat = FileFormat.valueOf(schema.get(FILE_FORMAT).toString().toUpperCase());
-                List<Column> columns = null;
-                try {
-                    List<Map<String, String>> columnConfig = (List<Map<String, String>>) schema.get(COLUMNS);
-                    if (!CollectionUtils.isEmpty(columnConfig)) {
-                        columns = columnConfig
-                                .stream()
-                                .map(c -> new Column(c.get(COLUMN_NAME), ValueType.valueOf(c.get(COLUMN_TYPE))))
-                                .collect(Collectors.toList());
-                    }
-                } catch (ClassCastException ignored) {
+    public List<Dataframe> loadFullDataFromSource(DataProviderSource config) throws Exception {
+        Map<String, Object> properties = config.getProperties();
+        List<Map<String, Object>> schemas;
+        if (properties.containsKey(SCHEMAS)) {
+            schemas = (List<Map<String, Object>>) properties.get(SCHEMAS);
+        } else {
+            schemas = Collections.singletonList(properties);
+        }
+        LinkedList<Dataframe> dataframes = new LinkedList<>();
+        for (Map<String, Object> schema : schemas) {
+            String path = schema.get(FILE_PATH).toString();
+            FileFormat fileFormat = FileFormat.valueOf(schema.get(FILE_FORMAT).toString().toUpperCase());
+            List<Column> columns = null;
+            try {
+                List<Map<String, String>> columnConfig = (List<Map<String, String>>) schema.get(COLUMNS);
+                if (!CollectionUtils.isEmpty(columnConfig)) {
+                    columns = columnConfig
+                            .stream()
+                            .map(c -> new Column(c.get(COLUMN_NAME), ValueType.valueOf(c.get(COLUMN_TYPE))))
+                            .collect(Collectors.toList());
                 }
-                Dataframe dataframe = loadFromPath(FileUtils.withBasePath(path), fileFormat, columns);
-                dataframe.setName(StringUtils.isNoneBlank(schema.getOrDefault(TABLE, "").toString()) ? schema.get(TABLE).toString() : "TEST" + UUIDGenerator.generate());
-                dataframes.add(dataframe);
+            } catch (ClassCastException ignored) {
             }
-            return dataframes;
-
-        } catch (Exception e) {
-            throw new DataProviderException(e);
+            Dataframe dataframe = loadFromPath(FileUtils.withBasePath(path), fileFormat, columns);
+            dataframe.setName(StringUtils.isNoneBlank(schema.getOrDefault(TABLE, "").toString()) ? schema.get(TABLE).toString() : "TEST" + UUIDGenerator.generate());
+            dataframes.add(dataframe);
         }
+        return dataframes;
     }
 
     private Dataframe loadFromPath(String path, FileFormat format, List<Column> columns) throws IOException {
@@ -134,7 +128,7 @@ private List<Column> inferHeader(List<List<Object>> values) {
         if (isHeader) {
             for (Object value : typedValues) {
                 Column column = new Column();
-                ValueType valueType = DataTypeUtils.javaType2DataType(value.getClass().getSimpleName());
+                ValueType valueType = DataTypeUtils.javaType2DataType(value);
                 column.setType(valueType);
                 column.setName(value.toString());
                 columns.add(column);
@@ -143,7 +137,7 @@ private List<Column> inferHeader(List<List<Object>> values) {
         } else {
             for (int i = 0; i < typedValues.size(); i++) {
                 Column column = new Column();
-                ValueType valueType = DataTypeUtils.javaType2DataType(typedValues.get(i).getClass().getSimpleName());
+                ValueType valueType = DataTypeUtils.javaType2DataType(typedValues.get(i));
                 column.setType(valueType);
                 column.setName("column" + i);
                 columns.add(column);

diff --git a/data-providers/http-data-provider/pom.xml b/data-providers/http-data-provider/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-alpha.1</version>
+        <version>1.0.0-alpha.2</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/data-providers/http-data-provider/src/main/java/datart/data/provider/HttpDataProvider.java b/data-providers/http-data-provider/src/main/java/datart/data/provider/HttpDataProvider.java
@@ -98,7 +98,6 @@ public List<Dataframe> loadFullDataFromSource(DataProviderSource config) throws
         return dataframes;
     }
 
-
     @Override
     public String getConfigFile() {
         return "http-data-provider.json";

diff --git a/data-providers/http-data-provider/src/main/java/datart/data/provider/ResponseJsonParser.java b/data-providers/http-data-provider/src/main/java/datart/data/provider/ResponseJsonParser.java
@@ -93,7 +93,7 @@ private ArrayList<Column> getSchema(JSONObject jsonObject) {
                 if (val instanceof JSONObject || val instanceof JSONArray) {
                     val = val.toString();
                 }
-                column.setType(DataTypeUtils.javaType2DataType(val.getClass().getSimpleName()));
+                column.setType(DataTypeUtils.javaType2DataType(val));
             } else {
                 column.setType(ValueType.STRING);
             }

diff --git a/data-providers/http-data-provider/src/main/resources/http-data-provider.json b/data-providers/http-data-provider/src/main/resources/http-data-provider.json
@@ -54,7 +54,7 @@
           "type": "number"
         },
         {
-          "name": "ResponseParser",
+          "name": "responseParser",
           "defaultValue": "",
           "description": "Http Response Parser",
           "type": "string"

diff --git a/data-providers/jdbc-data-provider/pom.xml b/data-providers/jdbc-data-provider/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>datart-parent</artifactId>
         <groupId>datart</groupId>
-        <version>1.0.0-alpha.1</version>
+        <version>1.0.0-alpha.2</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-Original file line number
+Diff line change
@@ -0,0 +1,6 @@
+    .idea
+    /static
+    *.iml
+    *.zip
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,4 +16,5 @@ public enum ValueType {

		SNIPPET, //will be parse to sql node

		KEYWORD,
		}