Skip to content

Commit

Permalink
Organize Python tests to use grpcio-tools directly
Browse files Browse the repository at this point in the history 
Moves all tests into a separate package. This does not change existing
supported means of running tests (e.g. through run_tests.py).
  • Loading branch information
@soltanmm-google
soltanmm-google committed Jul 1, 2016
1 parent b926ef2 commit 1ff429d
Show file tree
Hide file tree
Showing 134 changed files with 787 additions and 611 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ python_build/
htmlcov/
dist/
*.egg
py27/
py34/

# Node installation output
^node_modules
Expand Down
44 changes: 1 addition & 43 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,6 @@
ENABLE_CYTHON_TRACING = os.environ.get(
'GRPC_PYTHON_ENABLE_CYTHON_TRACING', False)

# Environment variable to determine whether or not to include the test files in
# the installation.
INSTALL_TESTS = os.environ.get('GRPC_PYTHON_INSTALL_TESTS', False)

CYTHON_EXTENSION_PACKAGE_NAMES = ()

CYTHON_EXTENSION_MODULE_NAMES = ('grpc._cython.cygrpc',)
Expand Down Expand Up @@ -183,13 +179,10 @@ def cython_extensions(module_names, extra_sources, include_dirs,

COMMAND_CLASS = {
'doc': commands.SphinxDocumentation,
'build_proto_modules': commands.BuildProtoModules,
'build_project_metadata': commands.BuildProjectMetadata,
'build_py': commands.BuildPy,
'build_ext': commands.BuildExt,
'gather': commands.Gather,
'run_interop': commands.RunInterop,
'test_lite': commands.TestLite
}

# Ensure that package data is copied over before any commands have been run:
Expand All @@ -200,32 +193,6 @@ def cython_extensions(module_names, extra_sources, include_dirs,
pass
shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem'))

TEST_PACKAGE_DATA = {
'tests.interop': [
'credentials/ca.pem',
'credentials/server1.key',
'credentials/server1.pem',
],
'tests.protoc_plugin': [
'protoc_plugin_test.proto',
],
'tests.unit': [
'credentials/ca.pem',
'credentials/server1.key',
'credentials/server1.pem',
],
}

TESTS_REQUIRE = (
'oauth2client>=2.1.0',
'protobuf>=3.0.0a3',
'coverage>=4.0',
) + INSTALL_REQUIRES

TEST_SUITE = 'tests'
TEST_LOADER = 'tests:Loader'
TEST_RUNNER = 'tests:Runner'

PACKAGE_DATA = {
# Binaries that may or may not be present in the final installation, but are
# mentioned here for completeness.
Expand All @@ -235,12 +202,7 @@ def cython_extensions(module_names, extra_sources, include_dirs,
'_windows/grpc_c.64.python',
],
}
if INSTALL_TESTS:
PACKAGE_DATA = dict(PACKAGE_DATA, **TEST_PACKAGE_DATA)
PACKAGES = setuptools.find_packages(PYTHON_STEM)
else:
PACKAGES = setuptools.find_packages(
PYTHON_STEM, exclude=['tests', 'tests.*'])
PACKAGES = setuptools.find_packages(PYTHON_STEM)

setuptools.setup(
name='grpcio',
Expand All @@ -253,8 +215,4 @@ def cython_extensions(module_names, extra_sources, include_dirs,
install_requires=INSTALL_REQUIRES,
setup_requires=SETUP_REQUIRES,
cmdclass=COMMAND_CLASS,
tests_require=TESTS_REQUIRE,
test_suite=TEST_SUITE,
test_loader=TEST_LOADER,
test_runner=TEST_RUNNER,
)
146 changes: 0 additions & 146 deletions src/python/grpcio/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,75 +134,6 @@ def run(self):
sphinx.main(['', os.path.join('doc', 'src'), os.path.join('doc', 'build')])


class BuildProtoModules(setuptools.Command):
"""Command to generate project *_pb2.py modules from proto files."""

description = 'build protobuf modules'
user_options = [
('include=', None, 'path patterns to include in protobuf generation'),
('exclude=', None, 'path patterns to exclude from protobuf generation')
]

def initialize_options(self):
self.exclude = None
self.include = r'.*\.proto$'
self.protoc_command = None
self.grpc_python_plugin_command = None

def finalize_options(self):
self.protoc_command = distutils.spawn.find_executable('protoc')
self.grpc_python_plugin_command = distutils.spawn.find_executable(
'grpc_python_plugin')

def run(self):
if not self.protoc_command:
raise CommandError('could not find protoc')
if not self.grpc_python_plugin_command:
raise CommandError('could not find grpc_python_plugin '
'(protoc plugin for GRPC Python)')

if not os.path.exists(PROTO_GEN_STEM):
os.makedirs(PROTO_GEN_STEM)

include_regex = re.compile(self.include)
exclude_regex = re.compile(self.exclude) if self.exclude else None
paths = []
for walk_root, directories, filenames in os.walk(PROTO_STEM):
for filename in filenames:
path = os.path.join(walk_root, filename)
if include_regex.match(path) and not (
exclude_regex and exclude_regex.match(path)):
paths.append(path)

# TODO(kpayson): It would be nice to do this in a batch command,
# but we currently have name conflicts in src/proto
for path in paths:
command = [
self.protoc_command,
'--plugin=protoc-gen-python-grpc={}'.format(
self.grpc_python_plugin_command),
'-I {}'.format(GRPC_STEM),
'-I .',
'-I {}/third_party/protobuf/src'.format(GRPC_STEM),
'--python_out={}'.format(PROTO_GEN_STEM),
'--python-grpc_out={}'.format(PROTO_GEN_STEM),
] + [path]
try:
subprocess.check_output(' '.join(command), cwd=PYTHON_STEM, shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
sys.stderr.write(
'warning: Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
command, str(e), e.output))

# Generated proto directories dont include __init__.py, but
# these are needed for python package resolution
for walk_root, _, _ in os.walk(PROTO_GEN_STEM):
if walk_root != PROTO_GEN_STEM:
path = os.path.join(walk_root, '__init__.py')
open(path, 'a').close()


class BuildProjectMetadata(setuptools.Command):
"""Command to generate project metadata in a module."""

Expand All @@ -225,10 +156,6 @@ class BuildPy(build_py.build_py):
"""Custom project build command."""

def run(self):
try:
self.run_command('build_proto_modules')
except CommandError as error:
sys.stderr.write('warning: %s\n' % error.message)
self.run_command('build_project_metadata')
build_py.build_py.run(self)

Expand Down Expand Up @@ -281,76 +208,3 @@ def run(self):
self.distribution.fetch_build_eggs(self.distribution.install_requires)
if self.test and self.distribution.tests_require:
self.distribution.fetch_build_eggs(self.distribution.tests_require)


class TestLite(setuptools.Command):
"""Command to run tests without fetching or building anything."""

description = 'run tests without fetching or building anything.'
user_options = []

def initialize_options(self):
pass

def finalize_options(self):
# distutils requires this override.
pass

def run(self):
self._add_eggs_to_path()

import tests
loader = tests.Loader()
loader.loadTestsFromNames(['tests'])
runner = tests.Runner()
result = runner.run(loader.suite)
if not result.wasSuccessful():
sys.exit('Test failure')

def _add_eggs_to_path(self):
"""Fetch install and test requirements"""
self.distribution.fetch_build_eggs(self.distribution.install_requires)
self.distribution.fetch_build_eggs(self.distribution.tests_require)


class RunInterop(test.test):

description = 'run interop test client/server'
user_options = [
('args=', 'a', 'pass-thru arguments for the client/server'),
('client', 'c', 'flag indicating to run the client'),
('server', 's', 'flag indicating to run the server')
]

def initialize_options(self):
self.args = ''
self.client = False
self.server = False

def finalize_options(self):
if self.client and self.server:
raise DistutilsOptionError('you may only specify one of client or server')

def run(self):
if self.distribution.install_requires:
self.distribution.fetch_build_eggs(self.distribution.install_requires)
if self.distribution.tests_require:
self.distribution.fetch_build_eggs(self.distribution.tests_require)
if self.client:
self.run_client()
elif self.server:
self.run_server()

def run_server(self):
# We import here to ensure that our setuptools parent has had a chance to
# edit the Python system path.
from tests.interop import server
sys.argv[1:] = self.args.split()
server.serve()

def run_client(self):
# We import here to ensure that our setuptools parent has had a chance to
# edit the Python system path.
from tests.interop import client
sys.argv[1:] = self.args.split()
client.test_interoperability()
47 changes: 2 additions & 45 deletions src/python/grpcio_health_checking/health_commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,51 +39,17 @@

import setuptools
from setuptools.command import build_py
from setuptools.command import sdist

ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
HEALTH_PROTO = os.path.join(ROOT_DIR, '../../proto/grpc/health/v1/health.proto')


class BuildProtoModules(setuptools.Command):
"""Command to generate project *_pb2.py modules from proto files."""
class CopyProtoModules(setuptools.Command):
"""Command to copy proto modules from grpc/src/proto."""

description = ''
user_options = []

def initialize_options(self):
pass

def finalize_options(self):
self.protoc_command = distutils.spawn.find_executable('protoc')
self.grpc_python_plugin_command = distutils.spawn.find_executable(
'grpc_python_plugin')

def run(self):
paths = []
root_directory = os.getcwd()
for walk_root, directories, filenames in os.walk(root_directory):
for filename in filenames:
if filename.endswith('.proto'):
paths.append(os.path.join(walk_root, filename))
command = [
self.protoc_command,
'--plugin=protoc-gen-python-grpc={}'.format(
self.grpc_python_plugin_command),
'-I {}'.format(root_directory),
'--python_out={}'.format(root_directory),
'--python-grpc_out={}'.format(root_directory),
] + paths
try:
subprocess.check_output(' '.join(command), cwd=root_directory, shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise Exception('{}\nOutput:\n{}'.format(e.message, e.output))


class CopyProtoModules(setuptools.Command):
"""Command to copy proto modules from grpc/src/proto."""

def initialize_options(self):
pass

Expand All @@ -101,14 +67,5 @@ class BuildPy(build_py.build_py):
"""Custom project build command."""

def run(self):
self.run_command('copy_proto_modules')
self.run_command('build_proto_modules')
build_py.build_py.run(self)


class SDist(sdist.sdist):
"""Custom project build command."""

def run(self):
self.run_command('copy_proto_modules')
sdist.sdist.run(self)
34 changes: 21 additions & 13 deletions src/python/grpcio_health_checking/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,36 +36,44 @@
from distutils import core as _core
import setuptools

import grpc.tools.command

# Ensure we're in the proper directory whether or not we're being used by pip.
os.chdir(os.path.dirname(os.path.abspath(__file__)))

# Break import-style to ensure we can actually find our commands module.
import health_commands

_PACKAGES = (
PACKAGES = (
setuptools.find_packages('.')
)

_PACKAGE_DIRECTORIES = {
PACKAGE_DIRECTORIES = {
'': '.',
}

_INSTALL_REQUIRES = (
SETUP_REQUIRES = (
'grpcio-tools>=0.14.0',
)

INSTALL_REQUIRES = (
'grpcio>=0.13.1',
)

_COMMAND_CLASS = {
'copy_proto_modules': health_commands.CopyProtoModules,
'build_proto_modules': health_commands.BuildProtoModules,
COMMAND_CLASS = {
# Run preprocess from the repository *before* doing any packaging!
'preprocess': health_commands.CopyProtoModules,

'build_proto_modules': grpc.tools.command.BuildProtoModules,
'build_py': health_commands.BuildPy,
'sdist': health_commands.SDist,
}

setuptools.setup(
name='grpcio_health_checking',
version='0.14.0b0',
packages=list(_PACKAGES),
package_dir=_PACKAGE_DIRECTORIES,
install_requires=_INSTALL_REQUIRES,
cmdclass=_COMMAND_CLASS
name='grpcio-health-checking',
version='0.14.0',
packages=list(PACKAGES),
package_dir=PACKAGE_DIRECTORIES,
install_requires=INSTALL_REQUIRES,
setup_requires=SETUP_REQUIRES,
cmdclass=COMMAND_CLASS
)
4 changes: 4 additions & 0 deletions src/python/grpcio_tests/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
proto/
src/
*_pb2.py
*.egg-info/
Loading

0 comments on commit 1ff429d

Please sign in to comment.