Skip to content

Commit

Permalink
Factored conformance tests so they can run in-process.
Browse files Browse the repository at this point in the history 
This is necessary for testing on iOS.

Change-Id: I54ec1e3aa2e9fbfff9a6cd6580920a6a62069b63
  • Loading branch information
@haberman
haberman committed Apr 15, 2015
1 parent 8fe135d commit 4e63b52
Show file tree
Hide file tree
Showing 5 changed files with 415 additions and 199 deletions.
14 changes: 7 additions & 7 deletions conformance/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ protoc_outputs = \
conformance.pb.cc \
conformance.pb.h

bin_PROGRAMS = conformance-test conformance-cpp
bin_PROGRAMS = conformance-test-runner conformance-cpp

conformance_test_LDADD = $(top_srcdir)/src/libprotobuf.la
conformance_test_SOURCES = conformance_test.cc
nodist_conformance_test_SOURCES = conformance.pb.cc
conformance_test_CPPFLAGS = -I$(top_srcdir)/src
conformance_test_runner_LDADD = $(top_srcdir)/src/libprotobuf.la
conformance_test_runner_SOURCES = conformance_test.cc conformance_test_runner.cc
nodist_conformance_test_runner_SOURCES = conformance.pb.cc
conformance_test_runner_CPPFLAGS = -I$(top_srcdir)/src

conformance_cpp_LDADD = $(top_srcdir)/src/libprotobuf.la
conformance_cpp_SOURCES = conformance_cpp.cc
Expand Down Expand Up @@ -46,5 +46,5 @@ MAINTAINERCLEANFILES = \
Makefile.in

# Targets for actually running tests.
test_cpp: unittest_proto_middleman conformance-test conformance-cpp
./conformance-test ./conformance-cpp
test_cpp: unittest_proto_middleman conformance-test-runner conformance-cpp
./conformance-test-runner ./conformance-cpp
30 changes: 13 additions & 17 deletions conformance/conformance.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,27 +32,23 @@ syntax = "proto3";
package conformance;

// This defines the conformance testing protocol. This protocol exists between
// the conformance tester process (the "tester") and the process whose protobuf
// implemention is being tested (the "testee"). The tester forks the testee and
// communicates with it over its stdin/stdout:
// the conformance test suite itself and the code being tested. For each test,
// the suite will send a ConformanceRequest message and expect a
// ConformanceResponse message.
//
// +--------+ pipe +----------+
// | tester | <------> | testee |
// | | | |
// | C++ | | any lang |
// +--------+ +----------+
// You can either run the tests in two different ways:
//
// The tester contains all of the test cases and their expected output.
// The testee is a simple program written in the target language that reads
// each test case and attempts to produce acceptable output for it.
// 1. in-process (using the interface in conformance_test.h).
//
// Every test consists of a ConformanceRequest/ConformanceResponse
// request/reply pair. The protocol on the pipe is simply:
// 2. as a sub-process communicating over a pipe. Information about how to
// do this is in conformance_test_runner.cc.
//
// 1. tester sends 4-byte length N
// 2. tester sends N bytes representing a ConformanceRequest proto
// 3. testee sends 4-byte length M
// 4. testee sends M bytes representing a ConformanceResponse proto
// Pros/cons of the two approaches:
//
// - running as a sub-process is much simpler for languages other than C/C++.
//
// - running as a sub-process may be more tricky in unusual environments like
// iOS apps, where fork/stdin/stdout are not available.

// Represents a single test case's input. The testee should:
//
Expand Down
265 changes: 90 additions & 175 deletions conformance/conformance_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <errno.h>
#include <stdarg.h>
#include <unistd.h>
#include <string>

#include "conformance.pb.h"
#include "conformance_test.h"
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/stringprintf.h>
#include <google/protobuf/wire_format_lite.h>

using conformance::ConformanceRequest;
Expand All @@ -45,167 +45,7 @@ using google::protobuf::FieldDescriptor;
using google::protobuf::internal::WireFormatLite;
using std::string;

int write_fd;
int read_fd;
int successes;
int failures;
bool verbose = false;

string Escape(const string& str) {
// TODO.
return str;
}

#define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x)
#define CHECK_SYSCALL(call) \
if (call < 0) { \
perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
exit(1); \
}

// TODO(haberman): make this work on Windows, instead of using these
// UNIX-specific APIs.
//
// There is a platform-agnostic API in
// src/google/protobuf/compiler/subprocess.h
//
// However that API only supports sending a single message to the subprocess.
// We really want to be able to send messages and receive responses one at a
// time:
//
// 1. Spawning a new process for each test would take way too long for thousands
// of tests and subprocesses like java that can take 100ms or more to start
// up.
//
// 2. Sending all the tests in one big message and receiving all results in one
// big message would take away our visibility about which test(s) caused a
// crash or other fatal error. It would also give us only a single failure
// instead of all of them.
void SpawnTestProgram(char *executable) {
int toproc_pipe_fd[2];
int fromproc_pipe_fd[2];
if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
perror("pipe");
exit(1);
}

pid_t pid = fork();
if (pid < 0) {
perror("fork");
exit(1);
}

if (pid) {
// Parent.
CHECK_SYSCALL(close(toproc_pipe_fd[0]));
CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
write_fd = toproc_pipe_fd[1];
read_fd = fromproc_pipe_fd[0];
} else {
// Child.
CHECK_SYSCALL(close(STDIN_FILENO));
CHECK_SYSCALL(close(STDOUT_FILENO));
CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));

CHECK_SYSCALL(close(toproc_pipe_fd[0]));
CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
CHECK_SYSCALL(close(toproc_pipe_fd[1]));
CHECK_SYSCALL(close(fromproc_pipe_fd[0]));

char *const argv[] = {executable, NULL};
CHECK_SYSCALL(execv(executable, argv)); // Never returns.
}
}

/* Invoking of tests **********************************************************/

void ReportSuccess() {
successes++;
}

void ReportFailure(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
failures++;
}

void CheckedWrite(int fd, const void *buf, size_t len) {
if (write(fd, buf, len) != len) {
GOOGLE_LOG(FATAL) << "Error writing to test program: " << strerror(errno);
}
}

void CheckedRead(int fd, void *buf, size_t len) {
size_t ofs = 0;
while (len > 0) {
ssize_t bytes_read = read(fd, (char*)buf + ofs, len);

if (bytes_read == 0) {
GOOGLE_LOG(FATAL) << "Unexpected EOF from test program";
} else if (bytes_read < 0) {
GOOGLE_LOG(FATAL) << "Error reading from test program: " << strerror(errno);
}

len -= bytes_read;
ofs += bytes_read;
}
}

void RunTest(const ConformanceRequest& request, ConformanceResponse* response) {
string serialized;
request.SerializeToString(&serialized);
uint32_t len = serialized.size();
CheckedWrite(write_fd, &len, sizeof(uint32_t));
CheckedWrite(write_fd, serialized.c_str(), serialized.size());
CheckedRead(read_fd, &len, sizeof(uint32_t));
serialized.resize(len);
CheckedRead(read_fd, (void*)serialized.c_str(), len);
if (!response->ParseFromString(serialized)) {
GOOGLE_LOG(FATAL) << "Could not parse response proto from tested process.";
}

if (verbose) {
fprintf(stderr, "conformance_test: request=%s, response=%s\n",
request.ShortDebugString().c_str(),
response->ShortDebugString().c_str());
}
}

void DoExpectParseFailureForProto(const string& proto, int line) {
ConformanceRequest request;
ConformanceResponse response;
request.set_protobuf_payload(proto);

// We don't expect output, but if the program erroneously accepts the protobuf
// we let it send its response as this. We must not leave it unspecified.
request.set_requested_output(ConformanceRequest::PROTOBUF);

RunTest(request, &response);
if (response.result_case() == ConformanceResponse::kParseError) {
ReportSuccess();
} else {
ReportFailure("Should have failed, but didn't. Line: %d, Request: %s, "
"response: %s\n",
line,
request.ShortDebugString().c_str(),
response.ShortDebugString().c_str());
}
}

// Expect that this precise protobuf will cause a parse error.
#define ExpectParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)

// Expect that this protobuf will cause a parse error, even if it is followed
// by valid protobuf data. We can try running this twice: once with this
// data verbatim and once with this data followed by some valid data.
//
// TODO(haberman): implement the second of these.
#define ExpectHardParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)

namespace {

/* Routines for building arbitrary protos *************************************/

Expand Down Expand Up @@ -299,7 +139,78 @@ uint32_t GetFieldNumberForType(WireFormatLite::FieldType type, bool repeated) {
return 0;
}

void TestPrematureEOFForType(WireFormatLite::FieldType type) {
} // anonymous namespace

namespace google {
namespace protobuf {

void ConformanceTestSuite::ReportSuccess() {
successes_++;
}

void ConformanceTestSuite::ReportFailure(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
StringAppendV(&output_, fmt, args);
va_end(args);
failures_++;
}

void ConformanceTestSuite::RunTest(const ConformanceRequest& request,
ConformanceResponse* response) {
string serialized_request;
string serialized_response;
request.SerializeToString(&serialized_request);

runner_->RunTest(serialized_request, &serialized_response);

if (!response->ParseFromString(serialized_response)) {
response->Clear();
response->set_runtime_error("response proto could not be parsed.");
}

if (verbose_) {
StringAppendF(&output_, "conformance test: request=%s, response=%s\n",
request.ShortDebugString().c_str(),
response->ShortDebugString().c_str());
}
}

void ConformanceTestSuite::DoExpectParseFailureForProto(const string& proto,
int line) {
ConformanceRequest request;
ConformanceResponse response;
request.set_protobuf_payload(proto);

// We don't expect output, but if the program erroneously accepts the protobuf
// we let it send its response as this. We must not leave it unspecified.
request.set_requested_output(ConformanceRequest::PROTOBUF);

RunTest(request, &response);
if (response.result_case() == ConformanceResponse::kParseError) {
ReportSuccess();
} else {
ReportFailure("Should have failed, but didn't. Line: %d, Request: %s, "
"response: %s\n",
line,
request.ShortDebugString().c_str(),
response.ShortDebugString().c_str());
}
}

// Expect that this precise protobuf will cause a parse error.
#define ExpectParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)

// Expect that this protobuf will cause a parse error, even if it is followed
// by valid protobuf data. We can try running this twice: once with this
// data verbatim and once with this data followed by some valid data.
//
// TODO(haberman): implement the second of these.
#define ExpectHardParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)


void ConformanceTestSuite::TestPrematureEOFForType(
WireFormatLite::FieldType type) {
// Incomplete values for each wire type.
static const string incompletes[6] = {
string("\x80"), // VARINT
Expand Down Expand Up @@ -376,20 +287,24 @@ void TestPrematureEOFForType(WireFormatLite::FieldType type) {
}
}


int main(int argc, char *argv[]) {
if (argc < 2) {
fprintf(stderr, "Usage: conformance_test <test-program>\n");
exit(1);
}

SpawnTestProgram(argv[1]);
void ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
std::string* output) {
runner_ = runner;
output_.clear();
successes_ = 0;
failures_ = 0;

for (int i = 1; i <= FieldDescriptor::MAX_TYPE; i++) {
TestPrematureEOFForType(static_cast<WireFormatLite::FieldType>(i));
}

fprintf(stderr, "conformance_test: completed %d tests for %s, %d successes, "
"%d failures.\n", successes + failures, argv[1], successes,
failures);
StringAppendF(&output_,
"CONFORMANCE SUITE FINISHED: completed %d tests, %d successes, "
"%d failures.\n",
successes_ + failures_, successes_, failures_);

output->assign(output_);
}

} // namespace protobuf
} // namespace google
Loading

0 comments on commit 4e63b52

Please sign in to comment.