Include core in Python distribution

yangyan · Jan 7, 2016 · 116982e · 116982e
1 parent bbaad2a
commit 116982e
Show file tree

Hide file tree

Showing 35 changed files with 563 additions and 440 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,8 +4,13 @@ gens
 libs
 objs
 
-# Python virtual environments
-python*_virtual_environment
+# Python items
+.coverage*
+.eggs
+.tox
+htmlcov/
+dist/
+*.egg
 
 # gcov coverage data
 reports

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,8 @@
+graft src/python/grpcio/grpc
+graft src/python/grpcio/tests
+graft src/core
+graft include/grpc
+include src/python/grpcio/commands.py
+include src/python/grpcio/grpc_core_dependencies.py
+include src/python/grpcio/README.rst
+include requirements.txt
diff --git a/build.yaml b/build.yaml
@@ -2474,3 +2474,7 @@ node_modules:
   - src/node/ext/server.cc
   - src/node/ext/server_credentials.cc
   - src/node/ext/timeval.cc
+python_dependencies:
+  deps:
+  - grpc
+  - gpr
diff --git a/src/python/grpcio/requirements.txt → requirements.txt b/src/python/grpcio/requirements.txt → requirements.txt
diff --git a/src/python/grpcio/setup.cfg → setup.cfg b/src/python/grpcio/setup.cfg → setup.cfg
diff --git a/src/python/grpcio/setup.py → setup.py b/src/python/grpcio/setup.py → setup.py
@@ -37,11 +37,16 @@
 from distutils import extension as _extension
 import setuptools
 
+PYTHON_STEM = './src/python/grpcio/'
+CORE_INCLUDE = ('./include', './',)
+
 # Ensure we're in the proper directory whether or not we're being used by pip.
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, PYTHON_STEM)
 
-# Break import-style to ensure we can actually find our commands module.
+# Break import-style to ensure we can actually find our in-repo dependencies.
 import commands
+import grpc_core_dependencies
 
 # Environment variable to determine whether or not the Cython extension should
 # *use* Cython or use the generated C files. Note that this requires the C files
@@ -59,23 +64,13 @@
 
 CYTHON_EXTENSION_PACKAGE_NAMES = ()
 
-CYTHON_EXTENSION_MODULE_NAMES = (
-    'grpc._cython.cygrpc',
-    'grpc._cython._cygrpc.call',
-    'grpc._cython._cygrpc.channel',
-    'grpc._cython._cygrpc.completion_queue',
-    'grpc._cython._cygrpc.credentials',
-    'grpc._cython._cygrpc.records',
-    'grpc._cython._cygrpc.server',
-)
+CYTHON_EXTENSION_MODULE_NAMES = ('grpc._cython.cygrpc',)
 
-EXTENSION_INCLUDE_DIRECTORIES = (
-    '.',
-)
+EXTENSION_INCLUDE_DIRECTORIES = (PYTHON_STEM,) + CORE_INCLUDE
 
 EXTENSION_LIBRARIES = (
-    'grpc',
-    'gpr',
+    'ssl',
+    'crypto',
 )
 if not "darwin" in sys.platform:
     EXTENSION_LIBRARIES += ('rt',)
@@ -84,11 +79,13 @@
 def cython_extensions(package_names, module_names, include_dirs, libraries,
                       build_with_cython=False):
   file_extension = 'pyx' if build_with_cython else 'c'
-  module_files = [name.replace('.', '/') + '.' + file_extension
+  module_files = [os.path.join(PYTHON_STEM,
+                               name.replace('.', '/') + '.' + file_extension)
                   for name in module_names]
   extensions = [
       _extension.Extension(
-          name=module_name, sources=[module_file],
+          name=module_name,
+          sources=[module_file] + grpc_core_dependencies.CORE_SOURCE_FILES,
           include_dirs=include_dirs, libraries=libraries,
           define_macros=[('CYTHON_TRACE_NOGIL', 1)] if ENABLE_CYTHON_TRACING else []
       ) for (module_name, module_file) in zip(module_names, module_files)
@@ -97,6 +94,7 @@ def cython_extensions(package_names, module_names, include_dirs, libraries,
     import Cython.Build
     return Cython.Build.cythonize(
         extensions,
+        include_path=include_dirs,
         compiler_directives={'linetrace': bool(ENABLE_CYTHON_TRACING)})
   else:
     return extensions
@@ -107,7 +105,7 @@ def cython_extensions(package_names, module_names, include_dirs, libraries,
     bool(BUILD_WITH_CYTHON))
 
 PACKAGE_DIRECTORIES = {
-    '': '.',
+    '': PYTHON_STEM,
 }
 
 INSTALL_REQUIRES = (
@@ -157,16 +155,18 @@ def cython_extensions(package_names, module_names, include_dirs, libraries,
 PACKAGE_DATA = {}
 if INSTALL_TESTS:
   PACKAGE_DATA = dict(PACKAGE_DATA, **TEST_PACKAGE_DATA)
-  PACKAGES = setuptools.find_packages('.')
+  PACKAGES = setuptools.find_packages(PYTHON_STEM)
 else:
-  PACKAGES = setuptools.find_packages('.', exclude=['tests', 'tests.*'])
+  PACKAGES = setuptools.find_packages(
+      PYTHON_STEM, exclude=['tests', 'tests.*'])
 
 setuptools.setup(
     name='grpcio',
-    version='0.12.0b0',
+    version='0.12.0b1',
     ext_modules=CYTHON_EXTENSION_MODULES,
     packages=list(PACKAGES),
     package_dir=PACKAGE_DIRECTORIES,
+    package_data=PACKAGE_DATA,
     install_requires=INSTALL_REQUIRES,
     setup_requires=SETUP_REQUIRES,
     cmdclass=COMMAND_CLASS,

diff --git a/src/python/grpcio/MANIFEST.in b/src/python/grpcio/MANIFEST.in
diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py
@@ -40,6 +40,8 @@
 from setuptools.command import build_py
 from setuptools.command import test
 
+PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
+
 CONF_PY_ADDENDUM = """
 extensions.append('sphinx.ext.napoleon')
 napoleon_google_docstring = True
@@ -68,7 +70,7 @@ def run(self):
     import sphinx.apidoc
     metadata = self.distribution.metadata
     src_dir = os.path.join(
-        os.getcwd(), self.distribution.package_dir[''], 'grpc')
+        PYTHON_STEM, self.distribution.package_dir[''], 'grpc')
     sys.path.append(src_dir)
     sphinx.apidoc.main([
         '', '--force', '--full', '-H', metadata.name, '-A', metadata.author,
@@ -104,7 +106,7 @@ def run(self):
     include_regex = re.compile(self.include)
     exclude_regex = re.compile(self.exclude) if self.exclude else None
     paths = []
-    root_directory = os.getcwd()
+    root_directory = PYTHON_STEM
     for walk_root, directories, filenames in os.walk(root_directory):
       for filename in filenames:
         path = os.path.join(walk_root, filename)
@@ -140,7 +142,7 @@ def finalize_options(self):
     pass
 
   def run(self):
-    with open('grpc/_grpcio_metadata.py', 'w') as module_file:
+    with open(os.path.join(PYTHON_STEM, 'grpc/_grpcio_metadata.py'), 'w') as module_file:
       module_file.write('__version__ = """{}"""'.format(
           self.distribution.get_version()))
 

diff --git a/...thon/grpcio/grpc/_cython/_cygrpc/call.pxd → .../grpcio/grpc/_cython/_cygrpc/call.pxd.pxi b/...thon/grpcio/grpc/_cython/_cygrpc/call.pxd → .../grpcio/grpc/_cython/_cygrpc/call.pxd.pxi
@@ -27,11 +27,9 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from grpc._cython._cygrpc cimport grpc
-
 
 cdef class Call:
 
-  cdef grpc.grpc_call *c_call
+  cdef grpc_call *c_call
   cdef list references
 
diff --git a/...thon/grpcio/grpc/_cython/_cygrpc/call.pyx → .../grpcio/grpc/_cython/_cygrpc/call.pyx.pxi b/...thon/grpcio/grpc/_cython/_cygrpc/call.pyx → .../grpcio/grpc/_cython/_cygrpc/call.pyx.pxi
@@ -29,10 +29,6 @@
 
 cimport cpython
 
-from grpc._cython._cygrpc cimport credentials
-from grpc._cython._cygrpc cimport grpc
-from grpc._cython._cygrpc cimport records
-
 
 cdef class Call:
 
@@ -44,50 +40,50 @@ cdef class Call:
   def start_batch(self, operations, tag):
     if not self.is_valid:
       raise ValueError("invalid call object cannot be used from Python")
-    cdef records.Operations cy_operations = records.Operations(operations)
-    cdef records.OperationTag operation_tag = records.OperationTag(tag)
+    cdef Operations cy_operations = Operations(operations)
+    cdef OperationTag operation_tag = OperationTag(tag)
     operation_tag.operation_call = self
     operation_tag.batch_operations = cy_operations
     cpython.Py_INCREF(operation_tag)
-    return grpc.grpc_call_start_batch(
+    return grpc_call_start_batch(
         self.c_call, cy_operations.c_ops, cy_operations.c_nops,
         <cpython.PyObject *>operation_tag, NULL)
 
   def cancel(
-      self, grpc.grpc_status_code error_code=grpc.GRPC_STATUS__DO_NOT_USE,
+      self, grpc_status_code error_code=GRPC_STATUS__DO_NOT_USE,
       details=None):
     if not self.is_valid:
       raise ValueError("invalid call object cannot be used from Python")
-    if (details is None) != (error_code == grpc.GRPC_STATUS__DO_NOT_USE):
+    if (details is None) != (error_code == GRPC_STATUS__DO_NOT_USE):
       raise ValueError("if error_code is specified, so must details "
                        "(and vice-versa)")
-    if error_code != grpc.GRPC_STATUS__DO_NOT_USE:
+    if error_code != GRPC_STATUS__DO_NOT_USE:
       if isinstance(details, bytes):
         pass
       elif isinstance(details, basestring):
         details = details.encode()
       else:
         raise TypeError("expected details to be str or bytes")
       self.references.append(details)
-      return grpc.grpc_call_cancel_with_status(
+      return grpc_call_cancel_with_status(
           self.c_call, error_code, details, NULL)
     else:
-      return grpc.grpc_call_cancel(self.c_call, NULL)
+      return grpc_call_cancel(self.c_call, NULL)
 
   def set_credentials(
-      self, credentials.CallCredentials call_credentials not None):
-    return grpc.grpc_call_set_credentials(
+      self, CallCredentials call_credentials not None):
+    return grpc_call_set_credentials(
         self.c_call, call_credentials.c_credentials)
 
   def peer(self):
-    cdef char *peer = grpc.grpc_call_get_peer(self.c_call)
+    cdef char *peer = grpc_call_get_peer(self.c_call)
     result = <bytes>peer
-    grpc.gpr_free(peer)
+    gpr_free(peer)
     return result
 
   def __dealloc__(self):
     if self.c_call != NULL:
-      grpc.grpc_call_destroy(self.c_call)
+      grpc_call_destroy(self.c_call)
 
   # The object *should* always be valid from Python. Used for debugging.
   @property

diff --git a/...n/grpcio/grpc/_cython/_cygrpc/channel.pxd → ...pcio/grpc/_cython/_cygrpc/channel.pxd.pxi b/...n/grpcio/grpc/_cython/_cygrpc/channel.pxd → ...pcio/grpc/_cython/_cygrpc/channel.pxd.pxi
@@ -27,10 +27,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from grpc._cython._cygrpc cimport grpc
-
 
 cdef class Channel:
 
-  cdef grpc.grpc_channel *c_channel
+  cdef grpc_channel *c_channel
   cdef list references
diff --git a/...n/grpcio/grpc/_cython/_cygrpc/channel.pyx → ...pcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/...n/grpcio/grpc/_cython/_cygrpc/channel.pyx → ...pcio/grpc/_cython/_cygrpc/channel.pyx.pxi
@@ -29,18 +29,12 @@
 
 cimport cpython
 
-from grpc._cython._cygrpc cimport call
-from grpc._cython._cygrpc cimport completion_queue
-from grpc._cython._cygrpc cimport credentials
-from grpc._cython._cygrpc cimport grpc
-from grpc._cython._cygrpc cimport records
-
 
 cdef class Channel:
 
-  def __cinit__(self, target, records.ChannelArgs arguments=None,
-                credentials.ChannelCredentials channel_credentials=None):
-    cdef grpc.grpc_channel_args *c_arguments = NULL
+  def __cinit__(self, target, ChannelArgs arguments=None,
+                ChannelCredentials channel_credentials=None):
+    cdef grpc_channel_args *c_arguments = NULL
     self.c_channel = NULL
     self.references = []
     if arguments is not None:
@@ -52,18 +46,18 @@ cdef class Channel:
     else:
       raise TypeError("expected target to be str or bytes")
     if channel_credentials is None:
-      self.c_channel = grpc.grpc_insecure_channel_create(target, c_arguments,
+      self.c_channel = grpc_insecure_channel_create(target, c_arguments,
                                                          NULL)
     else:
-      self.c_channel = grpc.grpc_secure_channel_create(
+      self.c_channel = grpc_secure_channel_create(
           channel_credentials.c_credentials, target, c_arguments, NULL)
       self.references.append(channel_credentials)
     self.references.append(target)
     self.references.append(arguments)
 
-  def create_call(self, call.Call parent, int flags,
-                  completion_queue.CompletionQueue queue not None,
-                  method, host, records.Timespec deadline not None):
+  def create_call(self, Call parent, int flags,
+                  CompletionQueue queue not None,
+                  method, host, Timespec deadline not None):
     if queue.is_shutting_down:
       raise ValueError("queue must not be shutting down or shutdown")
     if isinstance(method, bytes):
@@ -82,36 +76,36 @@ cdef class Channel:
       host_c_string = host
     else:
       raise TypeError("expected host to be str, bytes, or None")
-    cdef call.Call operation_call = call.Call()
+    cdef Call operation_call = Call()
     operation_call.references = [self, method, host, queue]
-    cdef grpc.grpc_call *parent_call = NULL
+    cdef grpc_call *parent_call = NULL
     if parent is not None:
       parent_call = parent.c_call
-    operation_call.c_call = grpc.grpc_channel_create_call(
+    operation_call.c_call = grpc_channel_create_call(
         self.c_channel, parent_call, flags,
         queue.c_completion_queue, method, host_c_string, deadline.c_time,
         NULL)
     return operation_call
 
   def check_connectivity_state(self, bint try_to_connect):
-    return grpc.grpc_channel_check_connectivity_state(self.c_channel,
+    return grpc_channel_check_connectivity_state(self.c_channel,
                                                       try_to_connect)
 
   def watch_connectivity_state(
-      self, last_observed_state, records.Timespec deadline not None,
-      completion_queue.CompletionQueue queue not None, tag):
-    cdef records.OperationTag operation_tag = records.OperationTag(tag)
+      self, last_observed_state, Timespec deadline not None,
+      CompletionQueue queue not None, tag):
+    cdef OperationTag operation_tag = OperationTag(tag)
     cpython.Py_INCREF(operation_tag)
-    grpc.grpc_channel_watch_connectivity_state(
+    grpc_channel_watch_connectivity_state(
         self.c_channel, last_observed_state, deadline.c_time,
         queue.c_completion_queue, <cpython.PyObject *>operation_tag)
 
   def target(self):
-    cdef char * target = grpc.grpc_channel_get_target(self.c_channel)
+    cdef char * target = grpc_channel_get_target(self.c_channel)
     result = <bytes>target
-    grpc.gpr_free(target)
+    gpr_free(target)
     return result
 
   def __dealloc__(self):
     if self.c_channel != NULL:
-      grpc.grpc_channel_destroy(self.c_channel)
+      grpc_channel_destroy(self.c_channel)
diff --git a/...grpc/_cython/_cygrpc/completion_queue.pxd → .../_cython/_cygrpc/completion_queue.pxd.pxi b/...grpc/_cython/_cygrpc/completion_queue.pxd → .../_cython/_cygrpc/completion_queue.pxd.pxi
@@ -27,15 +27,13 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from grpc._cython._cygrpc cimport grpc
-
 
 cdef class CompletionQueue:
 
-  cdef grpc.grpc_completion_queue *c_completion_queue
+  cdef grpc_completion_queue *c_completion_queue
   cdef object poll_condition
   cdef bint is_polling
   cdef bint is_shutting_down
   cdef bint is_shutdown
 
-  cdef _interpret_event(self, grpc.grpc_event event)
+  cdef _interpret_event(self, grpc_event event)