Skip to content

Commit

Permalink
Revert "[core] Support encrypted redis connection." (ray-project#28991)
Browse files Browse the repository at this point in the history 
Reverts ray-project#28628

Looks like this fails the build in the master. I verified it works after reverting this PR

```
ERROR: /Users/sangbincho/work/ray/BUILD.bazel:2764:18 Executing genrule //:cp_redis failed: (Exit 64): bash failed: error executing command 
  (cd /private/var/tmp/_bazel_sangbincho/f7d6f25d281df169000c577629db7adf/sandbox/darwin-sandbox/1985/execroot/com_github_ray_project_ray && \
  exec env - \
    PATH=/Users/sangbincho/google-cloud-sdk/bin:/Users/sangbincho/anaconda3/envs/core/bin:/Users/sangbincho/anaconda3/condabin:/Users/sangbincho/.nvm/versions/node/v14.19.2/bin:/Library/Frameworks/Python.framework/Versions/3.8/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/Users/sangbincho/.cargo/bin:/Users/sangbincho/bin \
  /bin/bash -c 'source external/bazel_tools/tools/genrule/genrule-setup.sh; 
    mkdir tmp-redis-bin
    cp bazel-out/darwin-opt/bin/external/com_github_antirez_redis/copy_redis/redis bazel-out/darwin-opt/bin/external/com_github_antirez_redis/redis/bin/redis-cli bazel-out/darwin-opt/bin/external/com_github_antirez_redis/redis/bin/redis-server bazel-out/darwin-opt/bin/external/com_github_antirez_redis/redis/include ./tmp-redis-bin/ -rf
    cp tmp-redis-bin/redis-server bazel-out/darwin-opt/bin/external/com_github_antirez_redis/redis-server
    chmod +x bazel-out/darwin-opt/bin/external/com_github_antirez_redis/redis-server
    cp tmp-redis-bin/redis-cli bazel-out/darwin-opt/bin/external/com_github_antirez_redis/redis-cli
    chmod +x bazel-out/darwin-opt/bin/external/com_github_antirez_redis/redis-cli
    rm -rf tmp-redis-bin
    ')
Execution platform: @local_config_platform//:host
```
  • Loading branch information
@rkooo567
rkooo567 authored Oct 3, 2022
1 parent 7167f9c commit d48e06b
Show file tree
Hide file tree
Showing 13 changed files with 62 additions and 294 deletions.
7 changes: 2 additions & 5 deletions bazel/BUILD.hiredis
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
COPTS = ["-DUSE_SSL=1"] + select({
COPTS = [] + select({
"@bazel_tools//src/conditions:windows": [
"-D_CRT_DECLARE_NONSTDC_NAMES=0", # don't define off_t, to avoid conflicts
"-DWIN32",
"-DWIN32_LEAN_AND_MEAN"
],
"//conditions:default": [
],
Expand Down Expand Up @@ -34,6 +32,7 @@ cc_library(
],
exclude =
[
"ssl.c",
"test.c",
],
),
Expand All @@ -45,8 +44,6 @@ cc_library(
include_prefix = "hiredis",
deps = [
":_hiredis",
"@boringssl//:ssl",
"@boringssl//:crypto"
],
visibility = ["//visibility:public"],
)
91 changes: 49 additions & 42 deletions bazel/BUILD.redis
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
load("@rules_foreign_cc//foreign_cc:defs.bzl", "make")

exports_files(
[
"redis-server.exe",
Expand All @@ -8,53 +6,62 @@ exports_files(
visibility = ["//visibility:public"],
)

filegroup(
name = "all_srcs",
srcs = glob(
include = ["**"],
exclude = ["*.bazel"],
),
)


make(
name = "redis",
args = [
"BUILD_TLS=yes",
"-s",
],
copts = [
"-DLUA_USE_MKSTEMP",
"-Wno-pragmas",
"-Wno-empty-body",
"-fPIC",
],
visibility = ["//visibility:public"],
lib_source = ":all_srcs",
deps = [
"@openssl//:openssl",
],
out_binaries = [
"redis-server",
"redis-cli"
]
)

genrule(
name = "bin",
srcs = [":redis"],
srcs = glob(["**"]),
outs = [
"redis-server",
"redis-cli",
],
cmd = """
mkdir tmp-redis-bin
cp $(locations :redis) ./tmp-redis-bin/ -rf
cp tmp-redis-bin/redis-server $(location redis-server)
chmod +x $(location redis-server)
cp tmp-redis-bin/redis-cli $(location redis-cli)
chmod +x $(location redis-cli)
rm -rf tmp-redis-bin
unset CC LDFLAGS CXX CXXFLAGS
tmpdir="redis.tmp"
p=$(location Makefile)
cp -p -L -R -- "$${p%/*}" "$${tmpdir}"
chmod +x "$${tmpdir}"/deps/jemalloc/configure
parallel="$$(getconf _NPROCESSORS_ONLN || echo 1)"
make -s -C "$${tmpdir}" -j"$${parallel}" V=0 CFLAGS="$${CFLAGS-} -DLUA_USE_MKSTEMP -Wno-pragmas -Wno-empty-body"
mv "$${tmpdir}"/src/redis-server $(location redis-server)
chmod +x $(location redis-server)
mv "$${tmpdir}"/src/redis-cli $(location redis-cli)
chmod +x $(location redis-cli)
rm -r -f -- "$${tmpdir}"
""",
visibility = ["//visibility:public"],
tags = ["local"],
)

# This library is for internal hiredis use, because hiredis assumes a
# different include prefix for itself than external libraries do.
cc_library(
name = "_hiredis",
hdrs = [
"deps/hiredis/dict.c",
"deps/hiredis/dict.h",
"deps/hiredis/fmacros.h",
],
strip_include_prefix = "deps/hiredis",
)

cc_library(
name = "hiredis",
srcs = glob(
[
"deps/hiredis/*.c",
"deps/hiredis/*.h",
],
exclude =
[
"deps/hiredis/test.c",
],
),
hdrs = glob([
"deps/hiredis/*.h",
"deps/hiredis/adapters/*.h",
]),
strip_include_prefix = "deps",
deps = [
":_hiredis",
],
visibility = ["//visibility:public"],
)
5 changes: 0 additions & 5 deletions bazel/ray_deps_build_all.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_confi
load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains")
load("@com_github_johnynek_bazel_jar_jar//:jar_jar.bzl", "jar_jar_repositories")
load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
load("@rules_foreign_cc_thirdparty//openssl:openssl_setup.bzl", "openssl_setup")



def ray_deps_build_all():
Expand All @@ -20,5 +17,3 @@ def ray_deps_build_all():
grpc_deps()
rules_proto_grpc_toolchains()
jar_jar_repositories()
rules_foreign_cc_dependencies()
openssl_setup()
31 changes: 0 additions & 31 deletions bazel/ray_deps_setup.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ def ray_deps_setup():
auto_http_archive(
name = "com_github_antirez_redis",
build_file = "@com_github_ray_project_ray//bazel:BUILD.redis",
patch_args = ["-p1"],
url = "https://github.com/redis/redis/archive/refs/tags/7.0.5.tar.gz",
sha256 = "40827fcaf188456ad9b3be8e27a4f403c43672b6bb6201192dc15756af6f1eae",
patches = [
Expand Down Expand Up @@ -244,36 +243,6 @@ def ray_deps_setup():
"@com_github_ray_project_ray//thirdparty/patches:grpc-python.patch",
],
)

http_archive(
name = "openssl",
strip_prefix = "openssl-1.1.1f",
sha256 = "186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35",
urls = [
"https://www.openssl.org/source/openssl-1.1.1f.tar.gz",
],
build_file = "@rules_foreign_cc_thirdparty//openssl:BUILD.openssl.bazel",
)

http_archive(
name = "rules_foreign_cc",
sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
strip_prefix = "rules_foreign_cc-0.9.0",
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
)

git_repository(
name = "rules_perl",
remote = "https://github.com/bazelbuild/rules_perl.git",
commit = "022b8daf2bb4836ac7a50e4a1d8ea056a3e1e403",
)

http_archive(
name = "rules_foreign_cc_thirdparty",
sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
strip_prefix = "rules_foreign_cc-0.9.0/examples/third_party",
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
)

http_archive(
# This rule is used by @com_github_grpc_grpc, and using a GitHub mirror
Expand Down
29 changes: 2 additions & 27 deletions python/ray/_private/services.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import ray
import ray._private.ray_constants as ray_constants
from ray._private.gcs_utils import GcsClient
from ray._raylet import GcsClientOptions, Config
from ray._raylet import GcsClientOptions
from ray.core.generated.common_pb2 import Language

resource = None
Expand Down Expand Up @@ -1107,36 +1107,11 @@ def _start_redis_instance(
if " " in password:
raise ValueError("Spaces not permitted in redis password.")
command += ["--requirepass", password]

if not Config.REDIS_ENABLE_SSL():
command += ["--port", str(port), "--loglevel", "warning"]
else:
import socket

with socket.socket() as s:
s.bind(("", 0))
free_port = s.getsockname()[1]
command += [
"--tls-port",
str(port),
"--loglevel",
"warning",
"--port",
str(free_port),
]

command += ["--port", str(port), "--loglevel", "warning"]
if listen_to_localhost_only:
command += ["--bind", "127.0.0.1"]
pidfile = os.path.join(session_dir_path, "redis-" + uuid.uuid4().hex + ".pid")
command += ["--pidfile", pidfile]
if Config.REDIS_ENABLE_SSL():
if Config.REDIS_CA_CERT():
command += ["--tls-ca-cert-file", Config.REDIS_CA_CERT()]
if Config.REDIS_CLIENT_CERT():
command += ["--tls-cert-file", Config.REDIS_CLIENT_CERT()]
if Config.REDIS_CLIENT_KEY():
command += ["--tls-key-file", Config.REDIS_CLIENT_KEY()]
command += ["--tls-replication", "yes"]
if sys.platform != "win32":
command += ["--save", "", "--appendonly", "no"]
process_info = start_ray_process(
Expand Down
12 changes: 0 additions & 12 deletions python/ray/includes/ray_config.pxd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,3 @@ cdef extern from "ray/common/ray_config.h" nogil:
c_bool start_python_importer_thread() const

c_bool use_ray_syncer() const

c_bool REDIS_ENABLE_SSL() const

c_string REDIS_CA_CERT() const

c_string REDIS_CA_PATH() const

c_string REDIS_CLIENT_CERT() const

c_string REDIS_CLIENT_KEY() const

c_string REDIS_SERVER_NAME() const
24 changes: 0 additions & 24 deletions python/ray/includes/ray_config.pxi
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,27 +116,3 @@ cdef class Config:
@staticmethod
def use_ray_syncer():
return RayConfig.instance().use_ray_syncer()

@staticmethod
def REDIS_ENABLE_SSL():
return RayConfig.instance().REDIS_ENABLE_SSL()

@staticmethod
def REDIS_CA_CERT():
return RayConfig.instance().REDIS_CA_CERT()

@staticmethod
def REDIS_CA_PATH():
return RayConfig.instance().REDIS_CA_PATH()

@staticmethod
def REDIS_CLIENT_CERT():
return RayConfig.instance().REDIS_CLIENT_CERT()

@staticmethod
def REDIS_CLIENT_KEY():
return RayConfig.instance().REDIS_CLIENT_KEY()

@staticmethod
def REDIS_SERVER_NAME():
return RayConfig.instance().REDIS_SERVER_NAME()
1 change: 0 additions & 1 deletion python/ray/tests/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,6 @@ py_test_module_list(
"test_raylet_output.py",
"test_scheduling_performance.py",
"test_get_or_create_actor.py",
"test_redis_tls.py",
],
size = "small",
tags = ["exclusive", "small_size_python_tests", "team:core"],
Expand Down
62 changes: 0 additions & 62 deletions python/ray/tests/test_redis_tls.py
View file

This file was deleted.

10 changes: 0 additions & 10 deletions src/ray/common/ray_config_def.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -669,16 +669,6 @@ RAY_CONFIG(std::string, TLS_SERVER_CERT, "")
RAY_CONFIG(std::string, TLS_SERVER_KEY, "")
RAY_CONFIG(std::string, TLS_CA_CERT, "")

/// Location of Redis TLS credentials
/// https://github.com/redis/hiredis/blob/c78d0926bf169670d15cfc1214e4f5d21673396b/README.md#hiredis-openssl-wrappers
RAY_CONFIG(bool, REDIS_ENABLE_SSL, false)
RAY_CONFIG(std::string, REDIS_CA_CERT, "")
RAY_CONFIG(std::string, REDIS_CA_PATH, "")

RAY_CONFIG(std::string, REDIS_CLIENT_CERT, "")
RAY_CONFIG(std::string, REDIS_CLIENT_KEY, "")
RAY_CONFIG(std::string, REDIS_SERVER_NAME, "")

/// grpc delay testing flags
/// To use this, simply do
/// export RAY_testing_asio_delay_us="method1=min_val:max_val,method2=20:100"
Expand Down
Loading

0 comments on commit d48e06b

Please sign in to comment.