init experimental lib

OpenMined · Jul 16, 2020 · ec94caa · ec94caa
1 parent 506fb8b
commit ec94caa
Show file tree

Hide file tree

Showing 18 changed files with 503 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -1,2 +1,219 @@
-# syft_experimental
-Deliberate experimental Rust implementation of Syft 
+# PySyft
+
+This README.md is formatted with remark:
+mrmlnc.vscode-remark
+
+## Setup
+
+- python 3.7+ - https://www.python.org/
+- rustup - https://rustup.rs/
+- bloomrpc - https://github.com/uw-labs/bloomrpc
+- protoc - https://github.com/protocolbuffers/protobuf
+- vscode - https://github.com/microsoft/vscode
+
+### Linux
+
+### MacOS
+
+Python
+
+```
+$ brew install python
+```
+
+rustup
+
+```
+$ brew install rustup
+$ rustup-init
+```
+
+bloomrpc
+
+```
+$ brew cask install bloomrpc
+```
+
+protoc
+
+```
+$ brew install protobuf
+```
+
+### Windows
+
+## Rust Toolchain
+
+We are currently using nightly due to some rust dependencies.
+
+```
+$ rustup toolchain install nightly
+$ rustup default nightly
+```
+
+### Formatting and Linting
+
+Rust comes with an opinionated formatter and linter so we will mandate that these are used.
+
+Install Rust Format:
+
+```
+$ rustup component add rustfmt
+```
+
+Install Rust Language Server:
+
+```
+$ rustup component add rls
+```
+
+Install Rust Linting:
+
+```
+$ rustup component add clippy
+```
+
+### VSCode Configuration
+
+While VSCode is not required it is highly recommended.
+
+Install Rust VSCode Extension:
+https://marketplace.visualstudio.com/items?itemName=rust-lang.rust
+
+```
+$ code --install-extension rust-lang.rust
+```
+
+Install Even Better TOML Extension:
+https://marketplace.visualstudio.com/items?itemName=tamasfe.even-better-toml
+
+```
+$ code --install-extension tamasfe.even-better-toml
+```
+
+Add to settings:
+
+```
+{
+  "evenBetterToml.formatter.reorderKeys": false
+}
+```
+
+## Python
+
+### Setup
+
+Make sure you have `python3.7+`
+
+We use a virtual environment to isolate the syft core python wheel development and build process.
+
+We include support for Pipenv, Conda and pip with virtualenv.
+
+### Formatting and Linting
+
+To keep code clean and bug free we mandate all code inside syft core python, uses an agreed upon set of linting and formatting standards.
+
+- black - https://github.com/psf/black
+- isort - https://github.com/timothycrosley/isort
+- mypy - http://mypy-lang.org/
+
+```
+$ pip install black isort mypy
+```
+
+### VSCode Configuration
+
+Add these to your settings.json, making sure to update the paths as necessary to your platform.
+
+```
+{
+  "python.linting.enabled": true,
+  "python.linting.mypyEnabled": true,
+  "python.formatting.provider": "black",
+  "python.linting.mypyPath": "/usr/local/bin/mypy",
+  "python.formatting.blackPath": "/usr/local/bin/black"
+}
+```
+
+### Python Package Managers
+
+#### Pipenv
+
+Upgrade pip:
+
+```
+$ pip install --upgrade pip
+```
+
+Install pipenv:
+
+```
+$ pip install pipenv
+```
+
+Enter virtualenv:
+
+```
+$ cd platforms/python
+$ pipenv shell
+```
+
+Install packages:
+
+```
+$ pipenv install --dev --skip-lock
+```
+
+#### Conda
+
+Create your conda environment, navigate to the ./core/platforms/python directory:
+
+```
+conda create --name syft --file spec-file.txt
+```
+
+#### pip and virtualenv
+
+Create a virtualenv in the PySyft/core/platforms/python folder and install the packages inside requirements.txt
+
+## Python Development
+
+You can compile and install the python library from the virtualenv in one command:
+
+```
+$ maturin develop
+```
+
+## Python Tests
+
+To test it out try:
+
+```
+$ python tests/message.py
+```
+
+## Build Python Wheel
+During this step we:
+- Build and install the Python wheel
+- Compile the protos for Python use and output to the ./src/syft/protos directory
+
+
+If you are using pipenv:
+
+```
+$ cd platforms/python
+$ pipenv shell
+$ touch build.rs
+$ maturin build -i python
+$ pip install `find -L ./target/wheels -name "*.whl"`
+```
+
+If you are using conda and have you conda environment activated (```conda activate syft```):
+
+```
+$ cd platforms/python
+$ touch build.rs
+$ maturin build -i python
+$ pip install `find -L ./target/wheels -name "*.whl"`
+```
+
diff --git a/platforms/python/.gitignore b/platforms/python/.gitignore
@@ -0,0 +1,11 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
diff --git a/platforms/python/Cargo.toml b/platforms/python/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name    = "syft_python"
+version = "0.1.0"
+authors = ["Ben Szymkow <simcof@gmail.com>", "Madhava Jay <me@madhavajay.com>"]
+edition = "2018"
+
+[lib]
+name       = "syft"
+crate-type = ["cdylib"]
+
+[dependencies]
+syft    = { version = "^0", path = "../../syft" }
+pyo3    = { version = "0.11.0", features = ["extension-module"] }
+tokio   = { version = "0.2.21", features = ["full"] }
+futures = "0.3"
+prost   = "0.6.1"
diff --git a/platforms/python/Pipfile b/platforms/python/Pipfile
@@ -0,0 +1,7 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+[dev-packages]
+maturin = "*"
+protobuf = "*"
diff --git a/platforms/python/build.rs b/platforms/python/build.rs
@@ -0,0 +1,20 @@
+use std::process::Command;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let output = if cfg!(target_os = "windows") {
+        Command::new("cmd")
+            .args(&["/C", "echo hello"])
+            .output()
+            .expect("failed to execute process")
+    } else {
+        Command::new("sh")
+            .arg("-c")
+            .arg("protoc -I=../../protos --python_out=../../../src/syft/protos ../../protos/message.proto")
+            .output()
+            .expect("failed to execute process")
+    };
+
+    println!("out: {:?}", output);
+
+    Ok(())
+}
diff --git a/platforms/python/requirements.txt b/platforms/python/requirements.txt
@@ -0,0 +1 @@
+maturin
diff --git a/platforms/python/spec-file.txt b/platforms/python/spec-file.txt
@@ -0,0 +1,29 @@
+# This file may be used to create an environment using:
+# $ conda create --name <env> --file <this file>
+# platform: linux-64
+@EXPLICIT
+https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
+https://conda.anaconda.org/anaconda/linux-64/ca-certificates-2020.6.24-0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.34-h53a641e_5.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-9.2.0-hdf63c60_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgomp-9.2.0-h24d8f2e_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-0_gnu.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-9.2.0-h24d8f2e_2.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/libffi-3.2.1-he1b5a44_1007.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.1-hf484d3e_1002.tar.bz2
+https://conda.anaconda.org/anaconda/linux-64/openssl-1.1.1g-h7b6447c_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.5-h516909a_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h516909a_1006.tar.bz2
+https://conda.anaconda.org/anaconda/linux-64/libprotobuf-3.12.3-hd408876_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/readline-8.0-h46ee950_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.10-hed695b0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.32.3-hcee41ef_1.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/python-3.8.3-cpython_he5300dc_0.tar.bz2
+https://conda.anaconda.org/anaconda/linux-64/certifi-2020.6.20-py38_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.8-1_cp38.tar.bz2
+https://conda.anaconda.org/anaconda/noarch/six-1.15.0-py_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/maturin-0.8.2-py38h1e0a361_0.tar.bz2
+https://conda.anaconda.org/conda-forge/linux-64/setuptools-49.1.0-py38h32f6830_0.tar.bz2
+https://conda.anaconda.org/anaconda/linux-64/protobuf-3.12.3-py38he6710b0_0.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/wheel-0.34.2-py_1.tar.bz2
+https://conda.anaconda.org/conda-forge/noarch/pip-20.1.1-py_1.tar.bz2
diff --git a/platforms/python/src/ffi/ffi.rs b/platforms/python/src/ffi/ffi.rs
@@ -0,0 +1,58 @@
+use pyo3::prelude::*;
+use pyo3::types::{PyBytes, PyUnicode};
+use pyo3::{wrap_pyfunction, wrap_pymodule};
+use syft::message::SyftMessage;
+
+#[pymodule]
+fn syft(_py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_wrapped(wrap_pymodule!(message))?;
+    Ok(())
+}
+
+#[pymodule]
+fn message(_py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_wrapped(wrap_pyfunction!(run_class_method_message))?;
+    Ok(())
+}
+
+#[pyfunction]
+fn run_class_method_message(
+    target_addr: &PyUnicode,
+    capability_name: &PyUnicode,
+    py_bytes: &PyBytes,
+) -> PyResult<std::vec::Vec<u8>> {
+    println!(
+        "Rust got Python Request {:?} {:?} {:?}",
+        target_addr, capability_name, py_bytes
+    );
+
+    // deserialize
+    let request: SyftMessage;
+    request = from_bytes(py_bytes.as_bytes()).expect("Rust Failed to decode message");
+    println!("Rust deserialized message {:?}", request);
+
+    // serialize
+    let mut response_bytes = vec![];
+    to_bytes(&request, &mut response_bytes).expect("Rust Failed to encode message");
+
+    println!("Rust sending back message as bytes {:?}", request);
+
+    return Ok(response_bytes);
+}
+
+/// Encodes the message to a `Vec<u8>`.
+pub fn to_bytes<M: prost::Message>(
+    message: &M,
+    buf: &mut Vec<u8>,
+) -> Result<(), prost::EncodeError> {
+    buf.reserve(message.encoded_len());
+    return message.encode(buf);
+}
+
+// Decodes an message from the buffer.
+pub fn from_bytes<M: prost::Message + std::default::Default>(
+    buf: &[u8],
+) -> Result<M, prost::DecodeError> {
+    let msg = prost::Message::decode(buf);
+    return msg;
+}
diff --git a/platforms/python/src/ffi/mod.rs b/platforms/python/src/ffi/mod.rs
@@ -0,0 +1,3 @@
+mod ffi;
+
+pub use ffi::*;
diff --git a/platforms/python/src/lib.rs b/platforms/python/src/lib.rs
@@ -0,0 +1,3 @@
+mod ffi;
+
+pub use ffi::*;