Switch parser to be a function

oliver-ni · Feb 6, 2021 · 214cbb9 · 214cbb9
1 parent 223da9f
commit 214cbb9
Show file tree

Hide file tree

Showing 15 changed files with 972 additions and 539 deletions.
diff --git a/fluent-syntax/Cargo.toml b/fluent-syntax/Cargo.toml
@@ -27,12 +27,14 @@ include = [
 [dependencies]
 serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
+thiserror = "1.0"
 
 [dev-dependencies]
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 glob = "0.3"
 criterion = "0.3"
+iai = "0.1"
 
 [features]
 default = []

diff --git a/fluent-syntax/benches/parser.rs b/fluent-syntax/benches/parser.rs
@@ -4,22 +4,18 @@ use criterion::Criterion;
 use std::collections::HashMap;
 use std::fs;
 use std::io;
-use std::io::Read;
 
-use fluent_syntax::parser::Parser;
+use fluent_syntax::parser::{parse, parse_runtime};
 use fluent_syntax::unicode::{unescape_unicode, unescape_unicode_to_string};
 
 fn read_file(path: &str) -> Result<String, io::Error> {
-    let mut f = fs::File::open(path)?;
-    let mut s = String::new();
-    f.read_to_string(&mut s)?;
-    Ok(s)
+    fs::read_to_string(path)
 }
 
-fn get_strings(tests: &[&'static str]) -> HashMap<&'static str, String> {
+fn get_resources(tests: &[&'static str]) -> HashMap<&'static str, String> {
     let mut ftl_strings = HashMap::new();
     for test in tests {
-        let path = format!("./benches/{}.ftl", test);
+        let path = format!("./benches/{}", test);
         ftl_strings.insert(*test, read_file(&path).expect("Couldn't load file"));
     }
     return ftl_strings;
@@ -42,25 +38,53 @@ fn get_ctxs(tests: &[&'static str]) -> HashMap<&'static str, Vec<String>> {
     return ftl_strings;
 }
 
-fn parser_bench(c: &mut Criterion) {
-    let tests = &["simple", "preferences", "menubar"];
-    let ftl_strings = get_strings(tests);
+fn parse_bench(c: &mut Criterion) {
+    let tests = &["simple.ftl", "preferences.ftl", "menubar.ftl"];
 
-    c.bench_function_over_inputs(
-        "parse",
-        move |b, &&name| {
-            let source = &ftl_strings[name];
+    let mut group = c.benchmark_group("parse_resource");
+
+    for (name, resource) in get_resources(tests) {
+        group.bench_with_input(name, &resource, |b, source| {
+            b.iter(|| parse(source.as_str()).expect("Parsing of the FTL failed."))
+        });
+    }
+
+    group.finish();
+}
+
+fn parse_ctx_bench(c: &mut Criterion) {
+    let tests = &["browser", "preferences"];
+
+    let mut group = c.benchmark_group("parse_ctx");
+
+    for (name, ctx) in get_ctxs(tests) {
+        group.bench_with_input(name, &ctx, |b, ctx| {
             b.iter(|| {
-                Parser::new(source.as_str())
-                    .parse()
-                    .expect("Parsing of the FTL failed.")
+                for source in ctx {
+                    parse(source.as_str()).expect("Parsing of the FTL failed.");
+                }
             })
-        },
-        tests,
-    );
+        });
+    }
+
+    group.finish();
+
+    let mut group = c.benchmark_group("parse_ctx_runtime");
+
+    for (name, ctx) in get_ctxs(tests) {
+        group.bench_with_input(name, &ctx, |b, ctx| {
+            b.iter(|| {
+                for source in ctx {
+                    parse_runtime(source.as_str()).expect("Parsing of the FTL failed.");
+                }
+            })
+        });
+    }
+
+    group.finish();
 }
 
-fn unicode_unescape_bench(c: &mut Criterion) {
+fn unicode_bench(c: &mut Criterion) {
     let strings = &[
         "foo",
         "This is an example value",
@@ -78,7 +102,10 @@ fn unicode_unescape_bench(c: &mut Criterion) {
         "\\u0041\\u0064\\u0064\\u0069\\u0074\\u0069\\u006f\\u006e\\u0061\\u006c \\u0063\\u006f\\u006e\\u0074\\u0065\\u006e\\u0074 \\u0070\\u0072\\u006f\\u0063\\u0065\\u0073\\u0073\\u0065\\u0073 \\u0063\\u0061\\u006e \\u0069\\u006d\\u0070\\u0072\\u006f\\u0076\\u0065 \\u0070\\u0065\\u0072\\u0066\\u006f\\u0072\\u006d\\u0061\\u006e\\u0063\\u0065 \\u0077\\u0068\\u0065\\u006e \\u0075\\u0073\\u0069\\u006e\\u0067 \\u006d\\u0075\\u006c\\u0074\\u0069\\u0070\\u006c\\u0065 \\u0074\\u0061\\u0062\\u0073\\u002c \\u0062\\u0075\\u0074 \\u0077\\u0069\\u006c\\u006c \\u0061\\u006c\\u0073\\u006f \\u0075\\u0073\\u0065 \\u006d\\u006f\\u0072\\u0065 \\u006d\\u0065\\u006d\\u006f\\u0072\\u0079\\u002e",
         "Additional content processes can improve performance when using multiple tabs, but will also use more memory.",
     ];
-    c.bench_function("unicode", move |b| {
+
+    let mut group = c.benchmark_group("unicode");
+
+    group.bench_function("writer", |b| {
         b.iter(|| {
             let mut result = String::new();
             for s in strings {
@@ -87,56 +114,18 @@ fn unicode_unescape_bench(c: &mut Criterion) {
             }
         })
     });
-    c.bench_function("unicode_to_string", move |b| {
+
+    group.bench_function("to_string", |b| {
         b.iter(|| {
             for s in strings {
                 let _ = unescape_unicode_to_string(s);
             }
         })
     });
-}
-
-fn parser_ctx_bench(c: &mut Criterion) {
-    let tests = &["browser", "preferences"];
-    let ftl_strings = get_ctxs(tests);
-
-    c.bench_function_over_inputs(
-        "parse_ctx",
-        move |b, &&name| {
-            let sources = &ftl_strings[name];
-            b.iter(|| {
-                for source in sources {
-                    Parser::new(source.as_str())
-                        .parse()
-                        .expect("Parsing of the FTL failed.");
-                }
-            })
-        },
-        tests,
-    );
-
-    let ftl_strings = get_ctxs(tests);
 
-    c.bench_function_over_inputs(
-        "parse_ctx_runtime",
-        move |b, &&name| {
-            let sources = &ftl_strings[name];
-            b.iter(|| {
-                for source in sources {
-                    Parser::new(source.as_str())
-                        .parse_runtime()
-                        .expect("Parsing of the FTL failed.");
-                }
-            })
-        },
-        tests,
-    );
+    group.finish();
 }
 
-criterion_group!(
-    benches,
-    parser_bench,
-    unicode_unescape_bench,
-    parser_ctx_bench
-);
+criterion_group!(benches, parse_bench, parse_ctx_bench, unicode_bench,);
+
 criterion_main!(benches);
diff --git a/fluent-syntax/src/ast/mod.rs b/fluent-syntax/src/ast/mod.rs
@@ -1,3 +1,86 @@
+//! Abstract Syntax Tree representation of the Fluent Translation List.
+//!
+//! The AST of Fluent contains all nodes structures to represent a complete
+//! representation of the FTL resource.
+//!
+//! The AST is intended to preserve all semantic information and allow for round-trip
+//! of a canonically written FTL resource.
+//!
+//! # Example
+//!
+//! ```
+//! use fluent_syntax::parser;
+//! use fluent_syntax::ast;
+//!
+//! let ftl = r#"
+//!
+//! ## This is a message comment
+//! hello-world = Hello World!
+//!     .tooltip = Tooltip for you, { $userName }.
+//!
+//! "#;
+//!
+//! let resource = parser::parse(ftl)
+//!     .expect("Failed to parse an FTL resource.");
+//!
+//! assert_eq!(
+//!     resource.body[0],
+//!     ast::Entry::Message(
+//!         ast::Message {
+//!             id: ast::Identifier {
+//!                 name: "hello-world"
+//!             },
+//!             value: Some(ast::Pattern {
+//!                 elements: vec![
+//!                     ast::PatternElement::TextElement {
+//!                         value: "Hello World!"
+//!                     },
+//!                 ]
+//!             }),
+//!             attributes: vec![
+//!                 ast::Attribute {
+//!                     id: ast::Identifier {
+//!                         name: "tooltip"
+//!                     },
+//!                     value: ast::Pattern {
+//!                         elements: vec![
+//!                             ast::PatternElement::TextElement {
+//!                                 value: "Tooltip for you, "
+//!                             },
+//!                             ast::PatternElement::Placeable {
+//!                                 expression: ast::Expression::InlineExpression(
+//!                                     ast::InlineExpression::VariableReference {
+//!                                         id: ast::Identifier {
+//!                                             name: "userName"
+//!                                         }
+//!                                     }
+//!                                 )
+//!                             },
+//!                             ast::PatternElement::TextElement {
+//!                                 value: "."
+//!                             },
+//!                         ]
+//!                     }
+//!                 }
+//!             ],
+//!             comment: Some(
+//!                 ast::Comment {
+//!                     content: vec!["This is a message comment"]
+//!                 }
+//!             )
+//!         }
+//!     ),
+//! );
+//! ```
+//!
+//! ## Errors
+//!
+//! AST does preserve blocks containing invaid syntax as [`Entry::Junk`].
+//!
+//! ## White space
+//!
+//! At the moment, AST does not preserve white space. In result only a
+//! canonical form of the AST is suitable for a round-trip.
 mod helper;
 
 #[cfg(feature = "serde")]

diff --git a/fluent-syntax/src/bin/parser.rs b/fluent-syntax/src/bin/parser.rs
@@ -1,4 +1,4 @@
-use fluent_syntax::parser::Parser;
+use fluent_syntax::parser::parse;
 use std::env;
 use std::fs::File;
 use std::io;
@@ -15,7 +15,7 @@ fn main() {
     let args: Vec<String> = env::args().collect();
     let source = read_file(args.get(1).expect("Pass an argument")).expect("Failed to fetch file");
 
-    let (ast, errors) = match Parser::new(source.as_str()).parse() {
+    let (ast, errors) = match parse(source.as_str()) {
         Ok(ast) => (ast, None),
         Err((ast, err)) => (ast, Some(err)),
     };

diff --git a/fluent-syntax/src/bin/update_fixtures.rs b/fluent-syntax/src/bin/update_fixtures.rs
@@ -1,7 +1,7 @@
 use std::fs;
 use std::io;
 
-use fluent_syntax::parser::Parser;
+use fluent_syntax::parser::parse;
 
 fn read_file(path: &str) -> Result<String, io::Error> {
     fs::read_to_string(path)
@@ -18,7 +18,7 @@ fn main() {
     for sample in samples {
         let path = format!("./benches/{}.ftl", sample);
         let source = read_file(&path).unwrap();
-        let ast = Parser::new(source).parse().unwrap();
+        let ast = parse(source).unwrap();
         let target_json = serde_json::to_string_pretty(&ast).unwrap();
         let new_path = format!("./tests/fixtures/benches/{}.json", sample);
         write_file(&new_path, &target_json).unwrap();
@@ -31,7 +31,7 @@ fn main() {
             let file_name = p.file_name().unwrap().to_str().unwrap();
             let path = p.to_str().unwrap();
             let source = read_file(path).unwrap();
-            let ast = Parser::new(source).parse().unwrap();
+            let ast = parse(source).unwrap();
             let target_json = serde_json::to_string_pretty(&ast).unwrap();
             let new_path = format!(
                 "./tests/fixtures/benches/contexts/{}/{}",

diff --git a/fluent-syntax/src/lib.rs b/fluent-syntax/src/lib.rs
@@ -1,3 +1,49 @@
+//! Fluent is a modern localization system designed to improve how software is translated.
+//!
+//! `fluent-syntax` is a lowest level component of the system which exposes components
+//! necessary for parsing and operating on Fluent Translation Lists ("FTL").
+//!
+//! The crate provides the `parser` module which allows for parsing of an
+//! input string to an Abstract Syntax Tree defined in the `ast` module.
+//!
+//! The `unicode` module exposes a set of helper functions used to decode
+//! escaped unicode literals according to Fluent specification.
+//!
+//! # Example
+//!
+//! ```
+//! use fluent_syntax::parser;
+//! use fluent_syntax::ast;
+//!
+//! let ftl = r#"
+//!
+//! hello-world = Hello World!
+//!
+//! "#;
+//!
+//! let resource = parser::parse(ftl)
+//!     .expect("Failed to parse an FTL resource.");
+//!
+//! assert_eq!(
+//!     resource.body[0],
+//!     ast::Entry::Message(
+//!         ast::Message {
+//!             id: ast::Identifier {
+//!                 name: "hello-world"
+//!             },
+//!             value: Some(ast::Pattern {
+//!                 elements: vec![
+//!                     ast::PatternElement::TextElement {
+//!                         value: "Hello World!"
+//!                     },
+//!                 ]
+//!             }),
+//!             attributes: vec![],
+//!             comment: None,
+//!         }
+//!     ),
+//! );
+//! ```
 pub mod ast;
 pub mod parser;
 pub mod unicode;
diff --git a/fluent-syntax/src/parser/comment.rs b/fluent-syntax/src/parser/comment.rs
@@ -1,4 +1,4 @@
-use super::{Parser, Result, Slice};
+use super::{core::Parser, core::Result, Slice};
 use crate::ast;
 
 #[derive(Debug, PartialEq, Clone, Copy)]