Add C++11 token support to all lexers

Support was already in slex, though it wasn't covered by the token test. This updates the tests while adding virtually identical code to lexertl and xlex.
boostorg · jefftrull · Aug 16, 2020 · Aug 10, 2020 · Aug 10, 2020 · Aug 10, 2020
commit 35fc5aa38116ced3e1a34d2e4779327678a53a3a
diff --git a/samples/list_includes/lexertl/lexertl_lexer.hpp b/samples/list_includes/lexertl/lexertl_lexer.hpp
@@ -57,7 +57,8 @@ namespace boost { namespace wave { namespace cpplexer { namespace lexertl
 #endif
 #define INIT_DATA_CPP_SIZE          15
 #define INIT_DATA_PP_NUMBER_SIZE    2
-#define INIT_MACRO_DATA_SIZE        27
+#define INIT_DATA_CPP0X_SIZE        15
+#define INIT_MACRO_DATA_SIZE        28
 #endif // #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0
 
 //  this is just a hack to have a unique token id not otherwise used by Wave
@@ -117,6 +118,7 @@ class lexertl
     static lexer_data const init_data[INIT_DATA_SIZE];              // common patterns
     static lexer_data const init_data_cpp[INIT_DATA_CPP_SIZE];      // C++ only patterns
     static lexer_data const init_data_pp_number[INIT_DATA_PP_NUMBER_SIZE];  // pp-number only patterns
+    static lexer_data const init_data_cpp0x[INIT_DATA_CPP0X_SIZE];  // C++0X only patterns
 
 // helper for calculation of the time of last compilation
     static boost::wave::util::time_conversion_helper compilation_time;
@@ -167,6 +169,7 @@ lexertl<Iterator, Position>::init_macro_data[INIT_MACRO_DATA_SIZE] =
 #endif
     MACRO_DATA("FLOAT_SUFFIX", "(" "[fF][lL]?" OR "[lL][fF]?" ")"),
     MACRO_DATA("CHAR_SPEC", "L?"),
+    MACRO_DATA("EXTCHAR_SPEC", "(" "[uU]" OR "u8" ")"),
     MACRO_DATA("BACKSLASH", "(" Q("\\") OR TRI(Q("/")) ")"),
     MACRO_DATA("ESCAPESEQ", "{BACKSLASH}([abfnrtv?'\"]|{BACKSLASH}|x{HEXDIGIT}+|{OCTALDIGIT}{1,3})"),
     MACRO_DATA("HEXQUAD", "{HEXDIGIT}{4}"),
@@ -413,6 +416,37 @@ lexertl<Iterator, Position>::init_data_pp_number[INIT_DATA_PP_NUMBER_SIZE] =
     { token_id(0) }       // this should be the last entry
 };
 
+// C++11 specific token definitions
+
+#define T_EXTCHARLIT      token_id(T_CHARLIT|AltTokenType)
+#define T_EXTSTRINGLIT    token_id(T_STRINGLIT|AltTokenType)
+#define T_EXTRAWSTRINGLIT token_id(T_RAWSTRINGLIT|AltTokenType)
+
+template <typename Iterator, typename Position>
+typename lexertl<Iterator, Position>::lexer_data const
+lexertl<Iterator, Position>::init_data_cpp0x[INIT_DATA_CPP0X_SIZE] =
+{
+    TOKEN_DATA(T_EXTCHARLIT, "{EXTCHAR_SPEC}" "'"
+                "(" "{ESCAPESEQ}" OR "{UNIVERSALCHAR}" OR "[^\\n\\r\\\\']" ")+" "'"),
+    TOKEN_DATA(T_EXTSTRINGLIT, "{EXTCHAR_SPEC}" Q("\"")
+                "(" "{ESCAPESEQ}" OR "{UNIVERSALCHAR}" OR "[^\\n\\r\\\\\"]" ")*" Q("\"")),
+    TOKEN_DATA(T_RAWSTRINGLIT, "{CHAR_SPEC}" "R" Q("\"")
+                "(" "{ESCAPESEQ}" OR "{UNIVERSALCHAR}" OR "[^\\\\\"]" ")*" Q("\"")),
+    TOKEN_DATA(T_EXTRAWSTRINGLIT, "{EXTCHAR_SPEC}" "R" Q("\"")
+                "(" "{ESCAPESEQ}" OR "{UNIVERSALCHAR}" OR "[^\\\\\"]" ")*" Q("\"")),
+    TOKEN_DATA(T_ALIGNAS, "alignas"),
+    TOKEN_DATA(T_ALIGNOF, "alignof"),
+    TOKEN_DATA(T_CHAR16_T, "char16_t"),
+    TOKEN_DATA(T_CHAR32_T, "char32_t"),
+    TOKEN_DATA(T_CONSTEXPR, "constexpr"),
+    TOKEN_DATA(T_DECLTYPE, "decltype"),
+    TOKEN_DATA(T_NOEXCEPT, "noexcept"),
+    TOKEN_DATA(T_NULLPTR, "nullptr"),
+    TOKEN_DATA(T_STATICASSERT, "static_assert"),
+    TOKEN_DATA(T_THREADLOCAL, "thread_local"),
+    { token_id(0) }       // this should be the last entry
+};
+
 #undef MACRO_DATA
 #undef TOKEN_DATA
 #undef OR
@@ -459,6 +493,17 @@ std::ifstream dfa_in("wave_lexertl_lexer.dfa", std::ios::in|std::ios::binary);
             }
         }
 
+    // if in C++0x mode, add all new keywords
+#if BOOST_WAVE_SUPPORT_CPP0X != 0
+        if (wave::need_cpp0x(lang)) {
+            for (int j = 0; 0 != init_data_cpp0x[j].tokenid; ++j) {
+                rules.add(init_data_cpp0x[j].tokenregex,
+                          init_data_cpp0x[j].tokenid);
+        }
+    }
+#endif
+
+
         for (int i = 0; 0 != init_data[i].tokenid; ++i) {
             rules.add(init_data[i].tokenregex, init_data[i].tokenid);
         }
@@ -764,6 +809,10 @@ lexer::lexertl<
 #undef INIT_MACRO_DATA_SIZE
 #undef T_ANYCTRL
 
+#undef T_EXTCHARLIT
+#undef T_EXTSTRINGLIT
+#undef T_EXTRAWSTRINGLIT
+
 ///////////////////////////////////////////////////////////////////////////////
 //
 //  The new_lexer_gen<>::new_lexer function (declared in lexertl_interface.hpp)

diff --git a/samples/token_statistics/xlex/xlex_lexer.hpp b/samples/token_statistics/xlex/xlex_lexer.hpp
@@ -104,6 +104,7 @@ class lexer
 
     static lexer_data const init_data[];        // common patterns
     static lexer_data const init_data_cpp[];    // C++ only patterns
+    static lexer_data const init_data_cpp0x[];  // C++11 only patterns
 
 #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0
     boost::wave::cpplexer::include_guards<token_type> guards;
@@ -154,6 +155,7 @@ class lexer
 #endif
 #define FLOAT_SUFFIX        "(" "[fF][lL]?|[lL][fF]?" ")"
 #define CHAR_SPEC           "L?"
+#define EXTCHAR_SPEC        "(" "[uU]" OR "u8" ")"
 
 #define BACKSLASH           "(" Q("\\") OR TRI(Q("/")) ")"
 #define ESCAPESEQ           BACKSLASH "(" \
@@ -397,6 +399,37 @@ lexer<Iterator, Position>::init_data_cpp[] =
     { token_id(0) }       // this should be the last entry
 };
 
+///////////////////////////////////////////////////////////////////////////////
+// C++11 only token definitions
+#define T_EXTCHARLIT      token_id(T_CHARLIT|AltTokenType)
+#define T_EXTSTRINGLIT    token_id(T_STRINGLIT|AltTokenType)
+#define T_EXTRAWSTRINGLIT token_id(T_RAWSTRINGLIT|AltTokenType)
+
+template <typename Iterator, typename Position>
+typename lexer<Iterator, Position>::lexer_data const
+lexer<Iterator, Position>::init_data_cpp0x[] =
+{
+    TOKEN_DATA(T_EXTCHARLIT, EXTCHAR_SPEC "'"
+                "(" ESCAPESEQ OR UNIVERSALCHAR OR "[^\\n\\r\\\\']" ")+" "'"),
+    TOKEN_DATA(T_EXTSTRINGLIT, EXTCHAR_SPEC Q("\"")
+                "(" ESCAPESEQ OR UNIVERSALCHAR OR "[^\\n\\r\\\\\"]" ")*" Q("\"")),
+    TOKEN_DATA(T_RAWSTRINGLIT, CHAR_SPEC "R" Q("\"")
+                "(" ESCAPESEQ OR UNIVERSALCHAR OR "[^\\\\\"]" ")*" Q("\"")),
+    TOKEN_DATA(T_EXTRAWSTRINGLIT, EXTCHAR_SPEC "R" Q("\"")
+                "(" ESCAPESEQ OR UNIVERSALCHAR OR "[^\\\\\"]" ")*" Q("\"")),
+    TOKEN_DATA(T_ALIGNAS, "alignas"),
+    TOKEN_DATA(T_ALIGNOF, "alignof"),
+    TOKEN_DATA(T_CHAR16_T, "char16_t"),
+    TOKEN_DATA(T_CHAR32_T, "char32_t"),
+    TOKEN_DATA(T_CONSTEXPR, "constexpr"),
+    TOKEN_DATA(T_DECLTYPE, "decltype"),
+    TOKEN_DATA(T_NOEXCEPT, "noexcept"),
+    TOKEN_DATA(T_NULLPTR, "nullptr"),
+    TOKEN_DATA(T_STATICASSERT, "static_assert"),
+    TOKEN_DATA(T_THREADLOCAL, "thread_local"),
+    { token_id(0) }       // this should be the last entry
+};
+
 ///////////////////////////////////////////////////////////////////////////////
 //  undefine macros, required for regular expression definitions
 #undef INCLUDEDEF
@@ -413,6 +446,7 @@ lexer<Iterator, Position>::init_data_cpp[] =
 #undef INTEGER
 #undef FLOAT_SUFFIX
 #undef CHAR_SPEC
+#undef EXTCHAR_SPEC
 #undef BACKSLASH    
 #undef ESCAPESEQ    
 #undef HEXQUAD      
@@ -425,6 +459,9 @@ lexer<Iterator, Position>::init_data_cpp[] =
 #undef TOKEN_DATA
 #undef TOKEN_DATA_EX
 
+#undef T_EXTCHARLIT
+#undef T_EXTSTRINGLIT
+#undef T_EXTRAWSTRINGLIT
 ///////////////////////////////////////////////////////////////////////////////
 // initialize cpp lexer 
 template <typename Iterator, typename Position>
@@ -444,6 +481,15 @@ lexer<Iterator, Position>::lexer(Iterator const &first,
         }
     }
 
+#if BOOST_WAVE_SUPPORT_CPP0X != 0
+    if (boost::wave::need_cpp0x(language)) {
+        for (int j = 0; 0 != init_data_cpp0x[j].tokenid; ++j) {
+            xlexer.register_regex(init_data_cpp0x[j].tokenregex,
+                init_data_cpp0x[j].tokenid, init_data_cpp[j].tokencb);
+        }
+    }
+#endif
+
 // tokens valid for C++ and C99    
     for (int i = 0; 0 != init_data[i].tokenid; ++i) {
         xlexer.register_regex(init_data[i].tokenregex, init_data[i].tokenid, 

diff --git a/test/testlexers/cpp_tokens.hpp b/test/testlexers/cpp_tokens.hpp
@@ -46,6 +46,7 @@ lexem const lexems[] =
     { "'\\U12345678'", boost::wave::T_CHARLIT },
     { "\"string literal\"", boost::wave::T_STRINGLIT },
     { "\"string literal \\n \\u1234 \\U12345678\"", boost::wave::T_STRINGLIT },
+    { "R\"foo(string literal)foo\"", boost::wave::T_RAWSTRINGLIT },
     { "&&", boost::wave::T_ANDAND },
     { "&=", boost::wave::T_ANDASSIGN },
     { "==", boost::wave::T_EQUAL },
@@ -108,6 +109,8 @@ lexem const lexems[] =
     { "*", boost::wave::T_STAR },
     { "~", boost::wave::T_COMPL },
     { "\?\?-", boost::wave::T_COMPL_TRIGRAPH },
+    { "alignas", boost::wave::T_ALIGNAS },
+    { "alignof", boost::wave::T_ALIGNOF },
     { "asm", boost::wave::T_ASM },
     { "auto", boost::wave::T_AUTO },
     { "bool", boost::wave::T_BOOL },
@@ -117,10 +120,14 @@ lexem const lexems[] =
     { "case", boost::wave::T_CASE },
     { "catch", boost::wave::T_CATCH },
     { "char", boost::wave::T_CHAR },
+    { "char16_t", boost::wave::T_CHAR16_T },
+    { "char32_t", boost::wave::T_CHAR32_T },
     { "class", boost::wave::T_CLASS },
     { "const_cast", boost::wave::T_CONSTCAST },
     { "const", boost::wave::T_CONST },
+    { "constexpr", boost::wave::T_CONSTEXPR },
     { "continue", boost::wave::T_CONTINUE },
+    { "decltype", boost::wave::T_DECLTYPE },
     { "default", boost::wave::T_DEFAULT },
     { "delete", boost::wave::T_DELETE },
     { "do", boost::wave::T_DO },
@@ -145,6 +152,8 @@ lexem const lexems[] =
     { "mutable", boost::wave::T_MUTABLE },
     { "namespace", boost::wave::T_NAMESPACE },
     { "new", boost::wave::T_NEW },
+    { "noexcept", boost::wave::T_NOEXCEPT },
+    { "nullptr", boost::wave::T_NULLPTR },
     { "operator", boost::wave::T_OPERATOR },
     { "private", boost::wave::T_PRIVATE },
     { "protected", boost::wave::T_PROTECTED },
@@ -161,6 +170,7 @@ lexem const lexems[] =
     { "switch", boost::wave::T_SWITCH },
     { "template", boost::wave::T_TEMPLATE },
     { "this", boost::wave::T_THIS },
+    { "thread_local", boost::wave::T_THREADLOCAL },
     { "throw", boost::wave::T_THROW },
     { "try", boost::wave::T_TRY },
     { "typedef", boost::wave::T_TYPEDEF },

diff --git a/test/testlexers/test_lexertl_lexer.cpp b/test/testlexers/test_lexertl_lexer.cpp
@@ -50,7 +50,7 @@ main(int argc, char *argv[])
         token_type::string_type instr(data->token);
 
         lexer_type it = lexer_type(instr.begin(), instr.end(), pos, 
-            boost::wave::support_option_long_long);
+                                   boost::wave::support_cpp0x);
         lexer_type end = lexer_type();
 
         // verify the correct outcome of the tokenization

diff --git a/test/testlexers/test_re2c_lexer.cpp b/test/testlexers/test_re2c_lexer.cpp
@@ -56,7 +56,7 @@ main(int argc, char *argv[])
         token_type::string_type instr(data->token);
 
         lexer_type it = lexer_type(instr.begin(), instr.end(), pos, 
-            boost::wave::support_option_long_long);
+            boost::wave::support_cpp0x);
         lexer_type end = lexer_type();
 
         // verify the correct outcome of the tokenization

diff --git a/test/testlexers/test_slex_lexer.cpp b/test/testlexers/test_slex_lexer.cpp
@@ -59,7 +59,7 @@ main(int argc, char *argv[])
         token_type::string_type instr(data->token);
 
         lexer_type it = lexer_type(instr.begin(), instr.end(), pos, 
-            boost::wave::support_option_long_long);
+            boost::wave::support_cpp0x);
         lexer_type end = lexer_type();
 
         // verify the correct outcome of the tokenization

diff --git a/test/testlexers/test_xlex_lexer.cpp b/test/testlexers/test_xlex_lexer.cpp
@@ -57,7 +57,7 @@ main(int argc, char *argv[])
         token_type::string_type instr(data->token);
 
         lexer_type it = lexer_type(instr.begin(), instr.end(), pos, 
-            boost::wave::support_option_long_long);
+                                   boost::wave::support_cpp0x);
         lexer_type end = lexer_type();
 
         // verify the correct outcome of the tokenisation