Port FlatBuffers to Rust: generator/runtime/tests.

google · aardappel · Sep 3, 2018 · Aug 29, 2018 · Aug 31, 2018 · Aug 31, 2018
commit 93f91622561907e2e51662be897631b5e0ed5634
diff --git a/.gitignore b/.gitignore
@@ -16,6 +16,7 @@
 *.keystore
 **/.vs/**
 **/bin/**
+!tests/rust_usage_test/bin/**
 **/gen/**
 **/libs/**
 **/obj/**

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -54,6 +54,7 @@ set(FlatBuffers_Compiler_SRCS
   src/idl_gen_python.cpp
   src/idl_gen_lobster.cpp
   src/idl_gen_lua.cpp
+  src/idl_gen_rust.cpp
   src/idl_gen_fbs.cpp
   src/idl_gen_grpc.cpp
   src/idl_gen_json_schema.cpp

diff --git a/docs/source/Compiler.md b/docs/source/Compiler.md
@@ -43,6 +43,8 @@ For any schema input files, one or more generators can be specified:
 
 -   `--lobster`: Generate Lobster code.
 
+-   `--rust`, `-r` : Generate Rust code.
+
 For any data input files:
 
 -   `--binary`, `-b` : If data is contained in this file, generate a

diff --git a/docs/source/RustUsage.md b/docs/source/RustUsage.md
@@ -0,0 +1,162 @@
+Use in Rust    {#flatbuffers_guide_use_rust}
+==========
+
+## Before you get started
+
+Before diving into the FlatBuffers usage in Rust, it should be noted that
+the [Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide
+to general FlatBuffers usage in all of the supported languages (including Rust).
+This page is designed to cover the nuances of FlatBuffers usage, specific to
+Rust.
+
+#### Prerequisites
+
+This page assumes you have written a FlatBuffers schema and compiled it
+with the Schema Compiler. If you have not, please see
+[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler)
+and [Writing a schema](@ref flatbuffers_guide_writing_schema).
+
+Assuming you wrote a schema, say `mygame.fbs` (though the extension doesn't
+matter), you've generated a Rust file called `mygame_generated.rs` using the
+compiler (e.g. `flatc --rust mygame.fbs`), you can now start using this in
+your program by including the file. As noted, this header relies on the crate
+`flatbuffers`, which should be in your include `Cargo.toml`.
+
+## FlatBuffers Rust library code location
+
+The code for the FlatBuffers Rust library can be found at
+`flatbuffers/rust`. You can browse the library code on the
+[FlatBuffers GitHub page](https://github.com/google/flatbuffers/tree/master/rust).
+
+## Testing the FlatBuffers Rust library
+
+The code to test the Rust library can be found at `flatbuffers/tests/rust_usage_test`.
+The test code itself is located in
+[integration_test.rs](https://github.com/google/flatbuffers/blob/master/tests/rust_usage_test/tests/integration_test.rs)
+
+This test file requires `flatc` to be present. To review how to build the project,
+please read the [Building](@ref flatbuffers_guide_building) documenation.
+
+To run the tests, execute `RustTest.sh` from the `flatbuffers/tests` directory.
+For example, on [Linux](https://en.wikipedia.org/wiki/Linux), you would simply
+run: `cd tests && ./RustTest.sh`.
+
+*Note: The shell script requires [Rust](https://www.rust-lang.org) to
+be installed.*
+
+## Using the FlatBuffers Rust library
+
+*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth
+example of how to use FlatBuffers in Rust.*
+
+FlatBuffers supports both reading and writing FlatBuffers in Rust.
+
+To use FlatBuffers in your code, first generate the Rust modules from your
+schema with the `--rust` option to `flatc`. Then you can import both FlatBuffers
+and the generated code to read or write FlatBuffers.
+
+For example, here is how you would read a FlatBuffer binary file in Rust:
+First, include the library and generated code. Then read the file into
+a `u8` vector, which you pass, as a byte slice, to `get_root_as_monster()`.
+
+This full example program is available in the Rust test suite:
+[monster_example.rs](https://github.com/google/flatbuffers/blob/master/tests/rust_usage_test/bin/monster_example.rs)
+
+It can be run by `cd`ing to the `rust_usage_test` directory and executing: `cargo run monster_example`.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.rs}
+    extern crate flatbuffers;
+
+    #[path = "../../monster_test_generated.rs"]
+    mod monster_test_generated;
+    pub use monster_test_generated::my_game;
+
+    use std::io::Read;
+
+    fn main() {
+        let mut f = std::fs::File::open("../monsterdata_test.mon").unwrap();
+        let mut buf = Vec::new();
+        f.read_to_end(&mut buf).expect("file reading failed");
+
+        let monster = my_game::example::get_root_as_monster(&buf[..]);
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`monster` is of type `Monster`, and points to somewhere *inside* your
+buffer (root object pointers are not the same as `buffer_pointer` !).
+If you look in your generated header, you'll see it has
+convenient accessors for all fields, e.g. `hp()`, `mana()`, etc:
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.rs}
+        println!("{}", monster.hp());     // `80`
+        println!("{}", monster.mana());   // default value of `150`
+        println!("{:?}", monster.name()); // Some("MyMonster")
+    }
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*Note: That we never stored a `mana` value, so it will return the default.*
+
+## Direct memory access
+
+As you can see from the above examples, all elements in a buffer are
+accessed through generated accessors. This is because everything is
+stored in little endian format on all platforms (the accessor
+performs a swap operation on big endian machines), and also because
+the layout of things is generally not known to the user.
+
+For structs, layout is deterministic and guaranteed to be the same
+across platforms (scalars are aligned to their
+own size, and structs themselves to their largest member), and you
+are allowed to access this memory directly by using `safe_slice` and
+on the reference to a struct, or even an array of structs.
+
+To compute offsets to sub-elements of a struct, make sure they
+are structs themselves, as then you can use the pointers to
+figure out the offset without having to hardcode it. This is
+handy for use of arrays of structs with calls like `glVertexAttribPointer`
+in OpenGL or similar APIs.
+
+It is important to note is that structs are still little endian on all
+machines, so only use tricks like this if you can guarantee you're not
+shipping on a big endian machine (using an `#[cfg(target_endian = "little")]`
+attribute would be wise).
+
+The special function `safe_slice` is implemented on Vector objects that are
+represented in memory the same way as they are represented on the wire. This
+function is always available on vectors of struct, bool, u8, and i8. It is
+conditionally-compiled on little-endian systems for all the remaining scalar
+types.
+
+The FlatBufferBuilder function `create_vector_direct` is implemented for all
+types that are endian-safe to write with a `memcpy`. It is the write-equivalent
+of `safe_slice`.
+
+## Access of untrusted buffers
+
+The generated accessor functions access fields over offsets, which is
+very quick. These offsets are not verified at run-time, so a malformed
+buffer could cause a program to crash by accessing random memory. (We try to
+prevent this in Rust by using safe slice accesses, instead of unsafe pointer
+dereferencing, but you should not rely on its correctness.)
+
+When you're processing large amounts of data from a source you know (e.g.
+your own generated data on disk), this is acceptable, but when reading
+data from the network that can potentially have been modified by an
+attacker, this is undesirable.
+
+The C++ port provides a buffer verifier, but, at this time, Rust does not.
+
+## Threading
+
+Reading a FlatBuffer does not touch any memory outside the original buffer,
+and is entirely read-only (all immutable), so is safe to access from multiple
+threads even without synchronisation primitives.
+
+Creating a FlatBuffer is not thread safe. All state related to building
+a FlatBuffer is contained in a FlatBufferBuilder instance, and no memory
+outside of it is touched. To make this thread safe, either do not
+share instances of FlatBufferBuilder between threads (recommended), or
+manually wrap it in synchronisation primitives. There's no automatic way to
+accomplish this, by design, as we feel multithreaded construction
+of a single buffer will be rare, and synchronisation overhead would be costly.
+
+<br>
diff --git a/docs/source/doxyfile b/docs/source/doxyfile
@@ -760,6 +760,7 @@ INPUT = "FlatBuffers.md" \
         "PythonUsage.md" \
         "LuaUsage.md" \
         "LobsterUsage.md" \
+        "RustUsage.md" \
         "Support.md" \
         "Benchmarks.md" \
         "WhitePaper.md" \
@@ -778,6 +779,7 @@ INPUT = "FlatBuffers.md" \
         "../../net/FlatBuffers/FlatBufferBuilder.cs" \
         "../../include/flatbuffers/flatbuffers.h" \
         "../../go/builder.go"
+        "../../rust/flatbuffers/src/builder.rs"
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h
@@ -41,24 +41,24 @@ namespace flatbuffers {
 // of type tokens.
 // clang-format off
 #define FLATBUFFERS_GEN_TYPES_SCALAR(TD) \
-  TD(NONE,   "",       uint8_t,  byte,   byte,    byte,   uint8) \
-  TD(UTYPE,  "",       uint8_t,  byte,   byte,    byte,   uint8) /* begin scalar/int */ \
-  TD(BOOL,   "bool",   uint8_t,  boolean,byte,    bool,   bool) \
-  TD(CHAR,   "byte",   int8_t,   byte,   int8,    sbyte,  int8) \
-  TD(UCHAR,  "ubyte",  uint8_t,  byte,   byte,    byte,   uint8) \
-  TD(SHORT,  "short",  int16_t,  short,  int16,   short,  int16) \
-  TD(USHORT, "ushort", uint16_t, short,  uint16,  ushort, uint16) \
-  TD(INT,    "int",    int32_t,  int,    int32,   int,    int32) \
-  TD(UINT,   "uint",   uint32_t, int,    uint32,  uint,   uint32) \
-  TD(LONG,   "long",   int64_t,  long,   int64,   long,   int64) \
-  TD(ULONG,  "ulong",  uint64_t, long,   uint64,  ulong,  uint64) /* end int */ \
-  TD(FLOAT,  "float",  float,    float,  float32, float,  float32) /* begin float */ \
-  TD(DOUBLE, "double", double,   double, float64, double, float64) /* end float/scalar */
+  TD(NONE,   "",       uint8_t,  byte,   byte,    byte,   uint8,   u8) \
+  TD(UTYPE,  "",       uint8_t,  byte,   byte,    byte,   uint8,   u8) /* begin scalar/int */ \
+  TD(BOOL,   "bool",   uint8_t,  boolean,byte,    bool,   bool,    bool) \
+  TD(CHAR,   "byte",   int8_t,   byte,   int8,    sbyte,  int8,    i8) \
+  TD(UCHAR,  "ubyte",  uint8_t,  byte,   byte,    byte,   uint8,   u8) \
+  TD(SHORT,  "short",  int16_t,  short,  int16,   short,  int16,   i16) \
+  TD(USHORT, "ushort", uint16_t, short,  uint16,  ushort, uint16,  u16) \
+  TD(INT,    "int",    int32_t,  int,    int32,   int,    int32,   i32) \
+  TD(UINT,   "uint",   uint32_t, int,    uint32,  uint,   uint32,  u32) \
+  TD(LONG,   "long",   int64_t,  long,   int64,   long,   int64,   i64) \
+  TD(ULONG,  "ulong",  uint64_t, long,   uint64,  ulong,  uint64,  u64) /* end int */ \
+  TD(FLOAT,  "float",  float,    float,  float32, float,  float32, f32) /* begin float */ \
+  TD(DOUBLE, "double", double,   double, float64, double, float64, f64) /* end float/scalar */
 #define FLATBUFFERS_GEN_TYPES_POINTER(TD) \
-  TD(STRING, "string", Offset<void>, int, int, StringOffset, int) \
-  TD(VECTOR, "",       Offset<void>, int, int, VectorOffset, int) \
-  TD(STRUCT, "",       Offset<void>, int, int, int, int) \
-  TD(UNION,  "",       Offset<void>, int, int, int, int)
+  TD(STRING, "string", Offset<void>, int, int, StringOffset, int, unused) \
+  TD(VECTOR, "",       Offset<void>, int, int, VectorOffset, int, unused) \
+  TD(STRUCT, "",       Offset<void>, int, int, int,          int, unused) \
+  TD(UNION,  "",       Offset<void>, int, int, int,          int, unused)
 
 // The fields are:
 // - enum
@@ -68,12 +68,14 @@ namespace flatbuffers {
 // - Go type.
 // - C# / .Net type.
 // - Python type.
+// - Rust type.
 
 // using these macros, we can now write code dealing with types just once, e.g.
 
 /*
 switch (type) {
-  #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
+  #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \
+                         RTYPE) \
     case BASE_TYPE_ ## ENUM: \
       // do something specific to CTYPE here
     FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
@@ -90,13 +92,15 @@ switch (type) {
 __extension__  // Stop GCC complaining about trailing comma with -Wpendantic.
 #endif
 enum BaseType {
-  #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
+  #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \
+                         RTYPE) \
       BASE_TYPE_ ## ENUM,
     FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
   #undef FLATBUFFERS_TD
 };
 
-#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
+#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \
+                       RTYPE) \
     static_assert(sizeof(CTYPE) <= sizeof(largest_scalar_t), \
                   "define largest_scalar_t as " #CTYPE);
   FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
@@ -111,6 +115,8 @@ inline bool IsFloat  (BaseType t) { return t == BASE_TYPE_FLOAT ||
 inline bool IsLong   (BaseType t) { return t == BASE_TYPE_LONG ||
                                            t == BASE_TYPE_ULONG; }
 inline bool IsBool   (BaseType t) { return t == BASE_TYPE_BOOL; }
+inline bool IsOneByte(BaseType t) { return t >= BASE_TYPE_UTYPE &&
+                                           t <= BASE_TYPE_UCHAR; }
 // clang-format on
 
 extern const char *const kTypeNames[];
@@ -410,6 +416,7 @@ struct IDLOptions {
     kDart = 1 << 11,
     kLua = 1 << 12,
     kLobster = 1 << 13,
+    kRust = 1 << 14,
     kMAX
   };
 
@@ -834,6 +841,12 @@ extern bool GenerateLua(const Parser &parser,
 	                    const std::string &path,
 	                    const std::string &file_name);
 
+// Generate Rust files from the definitions in the Parser object.
+// See idl_gen_rust.cpp.
+extern bool GenerateRust(const Parser &parser,
+                         const std::string &path,
+                         const std::string &file_name);
+
 // Generate Json schema file
 // See idl_gen_json_schema.cpp.
 extern bool GenerateJsonSchema(const Parser &parser,
@@ -872,6 +885,12 @@ extern std::string DartMakeRule(const Parser &parser,
                                 const std::string &path,
                                 const std::string &file_name);
 
+// Generate a make rule for the generated Rust code.
+// See idl_gen_rust.cpp.
+extern std::string RustMakeRule(const Parser &parser,
+                                const std::string &path,
+                                const std::string &file_name);
+
 // Generate a make rule for the generated Java/C#/... files.
 // See idl_gen_general.cpp.
 extern std::string GeneralMakeRule(const Parser &parser,

diff --git a/rust/flatbuffers/Cargo.lock b/rust/flatbuffers/Cargo.lock
diff --git a/rust/flatbuffers/Cargo.toml b/rust/flatbuffers/Cargo.toml
@@ -0,0 +1,7 @@
+[package]
+name = "flatbuffers"
+version = "0.1.0"
+authors = ["Robert Winslow <hello@rwinslow.com>", "FlatBuffers Maintainers"]
+
+[dependencies]
+smallvec = "0.6"