forked from microsoft/Mobius
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request microsoft#175 from tawan0109/dataframe
DataFrame API Na(), Read(), JsonFile() and Map()
- Loading branch information
Showing
23 changed files
with
1,721 additions
and
33 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
21 changes: 21 additions & 0 deletions
21
csharp/Adapter/Microsoft.Spark.CSharp/Proxy/IDataFrameNaFunctionsProxy.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Microsoft.Spark.CSharp.Proxy | ||
| { | ||
| interface IDataFrameNaFunctionsProxy | ||
| { | ||
| IDataFrameProxy Drop(int minNonNulls, string[] cols); | ||
| IDataFrameProxy Fill(double value, string[] cols); | ||
| IDataFrameProxy Fill(string value, string[] cols); | ||
| IDataFrameProxy Fill(Dictionary<string, object> valueMap); | ||
| IDataFrameProxy Replace<T>(string col, Dictionary<T, T> replacement); | ||
| IDataFrameProxy Replace<T>(string[] cols, Dictionary<T, T> replacement); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
csharp/Adapter/Microsoft.Spark.CSharp/Proxy/IDataFrameReaderProxy.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Data; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Spark.CSharp.Sql; | ||
|
|
||
| namespace Microsoft.Spark.CSharp.Proxy | ||
| { | ||
| internal interface IDataFrameReaderProxy | ||
| { | ||
| void Format(string source); | ||
| void Schema(StructType schema); | ||
| void Options(Dictionary<string,string> options); | ||
| IDataFrameProxy Load(); | ||
| IDataFrameProxy Jdbc(string url, string table, string[] predicates, Dictionary<string,string> connectionProperties); | ||
| IDataFrameProxy Jdbc(string url, string table, Dictionary<String, String> properties); | ||
| IDataFrameProxy Jdbc(string url, string table, string columnName, string lowerBound, string upperBound, | ||
| int numPartitions, Dictionary<String, String> connectionProperties); | ||
| IDataFrameProxy Parquet(string[] paths); | ||
| IDataFrameProxy Table(string tableName); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
65 changes: 65 additions & 0 deletions
65
csharp/Adapter/Microsoft.Spark.CSharp/Proxy/Ipc/DataFrameNaFunctionsIpcProxy.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using Microsoft.Spark.CSharp.Interop.Ipc; | ||
|
|
||
| namespace Microsoft.Spark.CSharp.Proxy.Ipc | ||
| { | ||
| [ExcludeFromCodeCoverage] //IPC calls to JVM validated using validation-enabled samples - unit test coverage not reqiured | ||
| internal class DataFrameNaFunctionsIpcProxy : IDataFrameNaFunctionsProxy | ||
| { | ||
| private readonly JvmObjectReference jvmDataFrameNaFunctionsReference; | ||
| private readonly ISqlContextProxy sqlContextProxy; | ||
|
|
||
| internal DataFrameNaFunctionsIpcProxy(JvmObjectReference jvmDataFrameNaFunctionsReference, ISqlContextProxy sqlContextProxy) | ||
| { | ||
| this.jvmDataFrameNaFunctionsReference = jvmDataFrameNaFunctionsReference; | ||
| this.sqlContextProxy = sqlContextProxy; | ||
| } | ||
|
|
||
| public IDataFrameProxy Drop(int minNonNulls, string[] cols) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameNaFunctionsReference, "drop", minNonNulls, cols).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Fill(double value, string[] cols) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameNaFunctionsReference, "fill", value, cols).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Fill(string value, string[] cols) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameNaFunctionsReference, "fill", value, cols).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Fill(Dictionary<string, object> valueMap) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameNaFunctionsReference, "fill", valueMap).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Replace<T>(string col, Dictionary<T, T> replacement) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameNaFunctionsReference, "replace", col, replacement).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Replace<T>(string[] cols, Dictionary<T, T> replacement) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameNaFunctionsReference, "replace", cols, replacement).ToString()), sqlContextProxy); | ||
| } | ||
| } | ||
| } |
87 changes: 87 additions & 0 deletions
87
csharp/Adapter/Microsoft.Spark.CSharp/Proxy/Ipc/DataFrameReaderIpcProxy.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
| // Licensed under the MIT license. See LICENSE file in the project root for full license information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Linq; | ||
| using System.Text; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Spark.CSharp.Interop.Ipc; | ||
| using Microsoft.Spark.CSharp.Sql; | ||
|
|
||
| namespace Microsoft.Spark.CSharp.Proxy.Ipc | ||
| { | ||
| [ExcludeFromCodeCoverage] //IPC calls to JVM validated using validation-enabled samples - unit test coverage not reqiured | ||
| internal class DataFrameReaderIpcProxy : IDataFrameReaderProxy | ||
| { | ||
| private readonly JvmObjectReference jvmDataFrameReaderReference; | ||
| private readonly ISqlContextProxy sqlContextProxy; | ||
|
|
||
| internal DataFrameReaderIpcProxy(JvmObjectReference jvmDataFrameReaderReference, ISqlContextProxy sqlContextProxy) | ||
| { | ||
| this.jvmDataFrameReaderReference = jvmDataFrameReaderReference; | ||
| this.sqlContextProxy = sqlContextProxy; | ||
| } | ||
|
|
||
| public void Format(string source) | ||
| { | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDataFrameReaderReference, "format", new object[] { source }); | ||
| } | ||
|
|
||
| public void Schema(StructType schema) | ||
| { | ||
| var structTypeIpcProxy = schema.StructTypeProxy as StructTypeIpcProxy; | ||
| if (structTypeIpcProxy != null) | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDataFrameReaderReference, "schema", | ||
| new object[] { structTypeIpcProxy.JvmStructTypeReference }); | ||
| } | ||
|
|
||
| public void Options(Dictionary<string, string> options) | ||
| { | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDataFrameReaderReference, "options", new object[] { options }); | ||
| } | ||
|
|
||
| public IDataFrameProxy Load() | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmDataFrameReaderReference, "load").ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Jdbc(string url, string table, string[] predicates, Dictionary<string, string> connectionProperties) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameReaderReference, "jdbc", new object[] { url, table, predicates, connectionProperties }).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Jdbc(string url, string table, Dictionary<string, string> properties) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameReaderReference, "jdbc", new object[] { url, table, properties }).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Jdbc(string url, string table, string columnName, string lowerBound, string upperBound, int numPartitions, Dictionary<string, string> connectionProperties) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameReaderReference, "jdbc", new object[] { url, table, columnName, lowerBound, upperBound, numPartitions, connectionProperties }).ToString()), | ||
| sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Parquet(string[] paths) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameReaderReference, "parquet", new object[] { paths }).ToString()), sqlContextProxy); | ||
| } | ||
|
|
||
| public IDataFrameProxy Table(string tableName) | ||
| { | ||
| return new DataFrameIpcProxy(new JvmObjectReference( | ||
| SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod( | ||
| jvmDataFrameReaderReference, "table", new object[] { tableName }).ToString()), sqlContextProxy); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.