Merge pull request #106 from jefftrull/bugfix/tricky-line-file

Track the location tokens were expanded from, for __LINE__ and __FILE__
boostorg · Aug 16, 2020 · 7c92c31 · 7c92c31
2 parents 8536634 + 9d68ca3
commit 7c92c31
Show file tree

Hide file tree

Showing 8 changed files with 225 additions and 15 deletions.
diff --git a/include/boost/wave/cpplexer/cpp_lex_token.hpp b/include/boost/wave/cpplexer/cpp_lex_token.hpp
@@ -24,6 +24,7 @@
 #include <boost/throw_exception.hpp>
 #include <boost/pool/singleton_pool.hpp>
 #include <boost/detail/atomic_count.hpp>
+#include <boost/optional.hpp>
 
 // this must occur after all of the includes and before any code appears
 #ifdef BOOST_HAS_ABI_HEADERS
@@ -54,12 +55,14 @@ class token_data
     :   id(T_UNKNOWN), refcnt(1)
     {}
 
-    token_data(token_id id_, string_type const &value_, position_type const &pos_)
-    :   id(id_), value(value_), pos(pos_), refcnt(1)
+    token_data(token_id id_, string_type const &value_,
+               position_type const &pos_,
+               optional<position_type> const & expand_pos_ = boost::none)
+    :   id(id_), value(value_), pos(pos_), expand_pos(expand_pos_), refcnt(1)
     {}
 
     token_data(token_data const& rhs)
-    :   id(rhs.id), value(rhs.value), pos(rhs.pos), refcnt(1)
+    :   id(rhs.id), value(rhs.value), pos(rhs.pos), expand_pos(rhs.expand_pos), refcnt(1)
     {}
 
     ~token_data()
@@ -73,10 +76,18 @@ class token_data
     operator token_id() const { return id; }
     string_type const &get_value() const { return value; }
     position_type const &get_position() const { return pos; }
+    position_type const &get_expand_position() const
+    {
+        if (expand_pos)
+            return *expand_pos;
+        else
+            return pos;
+    }
 
     void set_token_id (token_id id_) { id = id_; }
     void set_value (string_type const &value_) { value = value_; }
     void set_position (position_type const &pos_) { pos = pos_; }
+    void set_expand_position (position_type const & pos_) { expand_pos = pos_; }
 
     friend bool operator== (token_data const& lhs, token_data const& rhs)
     {
@@ -138,6 +149,7 @@ class token_data
     token_id id;                // the token id
     string_type value;          // the text, which was parsed into this token
     position_type pos;          // the original file position
+    boost::optional<position_type> expand_pos;    // where this token was expanded
     boost::detail::atomic_count refcnt;
 };
 
@@ -241,12 +253,14 @@ class lex_token
     operator token_id() const { return 0 != data ? token_id(*data) : T_EOI; }
     string_type const &get_value() const { return data->get_value(); }
     position_type const &get_position() const { return data->get_position(); }
+    position_type const &get_expand_position() const { return data->get_expand_position(); }
     bool is_eoi() const { return 0 == data || token_id(*data) == T_EOI; }
     bool is_valid() const { return 0 != data && token_id(*data) != T_UNKNOWN; }
 
     void set_token_id (token_id id_) { make_unique(); data->set_token_id(id_); }
     void set_value (string_type const &value_) { make_unique(); data->set_value(value_); }
     void set_position (position_type const &pos_) { make_unique(); data->set_position(pos_); }
+    void set_expand_position (position_type const &pos_) { make_unique(); data->set_expand_position(pos_); }
 
     friend bool operator== (lex_token const& lhs, lex_token const& rhs)
     {

diff --git a/include/boost/wave/util/cpp_macromap.hpp b/include/boost/wave/util/cpp_macromap.hpp
@@ -33,6 +33,7 @@
 
 #include <boost/filesystem/path.hpp>
 #include <boost/lexical_cast.hpp>
+#include <boost/optional.hpp>
 
 #include <boost/wave/util/time_conversion_helper.hpp>
 #include <boost/wave/util/unput_queue_iterator.hpp>
@@ -175,7 +176,8 @@ class macromap {
         unput_queue_iterator<IteratorT, token_type, ContainerT> &first,
         unput_queue_iterator<IteratorT, token_type, ContainerT> const &last,
         bool& seen_newline, bool expand_operator_defined,
-        bool expand_operator_has_include);
+        bool expand_operator_has_include,
+        boost::optional<position_type> expanding_pos);
 
 //  Collect all arguments supplied to a macro invocation
 #if BOOST_WAVE_USE_DEPRECIATED_PREPROCESSING_HOOKS != 0
@@ -199,6 +201,7 @@ class macromap {
         IteratorT &first, IteratorT const &last,
         bool& seen_newline, bool expand_operator_defined,
         bool expand_operator_has_include,
+        boost::optional<position_type> expanding_pos,
         defined_macros_type *scope = 0, ContainerT *queue_symbol = 0);
 
 //  Expand a predefined macro (__LINE__, __FILE__ and __INCLUDE_LEVEL__)
@@ -259,6 +262,10 @@ class macromap {
 
     static bool is_space(char);
 
+// batch update tokens with a single expand position
+    template <typename ContainerT>
+    static void set_expand_positions(ContainerT &tokens, position_type pos);
+
 #if BOOST_WAVE_SERIALIZATION != 0
 public:
     BOOST_STATIC_CONSTANT(unsigned int, version = 0x10);
@@ -695,7 +702,8 @@ macromap<ContextT>::expand_tokensequence(IteratorT &first,
 on_exit::assign<IteratorT, iterator_type> on_exit(first, first_it);
 
     return expand_tokensequence_worker(pending, first_it, last_it,
-        seen_newline, expand_operator_defined, expand_operator_has_include);
+        seen_newline, expand_operator_defined, expand_operator_has_include,
+        boost::none);
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -718,7 +726,8 @@ macromap<ContextT>::expand_tokensequence_worker(
     unput_queue_iterator<IteratorT, token_type, ContainerT> &first,
     unput_queue_iterator<IteratorT, token_type, ContainerT> const &last,
     bool& seen_newline, bool expand_operator_defined,
-    bool expand_operator_has_include)
+    bool expand_operator_has_include,
+    boost::optional<position_type> expanding_pos)
 {
 // if there exist pending tokens (tokens, which are already preprocessed), then
 // return the next one from there
@@ -791,7 +800,8 @@ macromap<ContextT>::expand_tokensequence_worker(
             // defined as a macro
                 if (expand_macro(pending, name_token, it, first, last,
                                  seen_newline, expand_operator_defined,
-                                 expand_operator_has_include))
+                                 expand_operator_has_include,
+                                 expanding_pos))
                 {
                 // the tokens returned by expand_macro should be rescanned
                 // beginning at the last token of the returned replacement list
@@ -818,10 +828,17 @@ macromap<ContextT>::expand_tokensequence_worker(
                 }
 
             // return the next preprocessed token
-                return expand_tokensequence_worker(
-                    pending, first, last,
-                    seen_newline, expand_operator_defined,
-                    expand_operator_has_include);
+                if (!expanding_pos)
+                    expanding_pos = name_token.get_expand_position();
+
+                typename ContextT::token_type const & result =
+                    expand_tokensequence_worker(
+                        pending, first, last,
+                        seen_newline, expand_operator_defined,
+                        expand_operator_has_include,
+                        expanding_pos);
+
+                return result;
             }
             else {
                 act_token = name_token;
@@ -1032,7 +1049,8 @@ macromap<ContextT>::expand_whole_tokensequence(ContainerT &expanded,
             expand_tokensequence_worker(
                 pending_queue, first_it,
                 last_it, seen_newline, expand_operator_defined,
-                expand_operator_has_include)
+                expand_operator_has_include,
+                boost::none)
         );
     }
 
@@ -1436,6 +1454,7 @@ macromap<ContextT>::expand_macro(ContainerT &expanded,
     IteratorT &first, IteratorT const &last,
     bool& seen_newline, bool expand_operator_defined,
     bool expand_operator_has_include,
+    boost::optional<position_type> expanding_pos,
     defined_macros_type *scope, ContainerT *queue_symbol)
 {
     using namespace boost::wave;
@@ -1589,6 +1608,10 @@ ContainerT replacement_list;
                 arguments, expand_operator_defined,
                 expand_operator_has_include,
                 replacement_list);
+
+            if (!expanding_pos)
+                expanding_pos = curr_token.get_expand_position();
+            set_expand_positions(replacement_list, *expanding_pos);
         }
         else {
         // defined as an object-like macro
@@ -1683,6 +1706,14 @@ ContainerT expanded_list;
 #else
     ctx.get_hooks().rescanned_macro(ctx.derived(), expanded_list);
 #endif
+
+    if (!expanding_pos)
+        // set the expanding position for rescan
+        expanding_pos = curr_token.get_expand_position();
+
+    // record the location where all the tokens were expanded from
+    set_expand_positions(expanded_list, *expanding_pos);
+
     expanded.splice(expanded.end(), expanded_list);
     return true;        // rescan is required
 }
@@ -1712,7 +1743,7 @@ string_type const &value = curr_token.get_value();
 
 #if BOOST_WAVE_USE_DEPRECIATED_PREPROCESSING_HOOKS != 0
     ctx.get_hooks().expanding_object_like_macro(
-        deftoken, Container(), curr_token);    // BOZO check params
+        deftoken, Container(), curr_token);
 #else
     if (ctx.get_hooks().expanding_object_like_macro(ctx.derived(),
             deftoken, ContainerT(), curr_token))
@@ -1727,7 +1758,7 @@ string_type const &value = curr_token.get_value();
 
     if (value == "__LINE__") {
     // expand the __LINE__ macro
-        std::string buffer = lexical_cast<std::string>(main_pos.get_line());
+        std::string buffer = lexical_cast<std::string>(curr_token.get_expand_position().get_line());
 
         replacement = token_type(T_INTLIT, buffer.c_str(), curr_token.get_position());
     }
@@ -1736,7 +1767,8 @@ string_type const &value = curr_token.get_value();
         namespace fs = boost::filesystem;
 
     std::string file("\"");
-    fs::path filename(wave::util::create_path(main_pos.get_file().c_str()));
+    fs::path filename(
+        wave::util::create_path(curr_token.get_expand_position().get_file().c_str()));
 
         using boost::wave::util::impl::escape_lit;
         file += escape_lit(wave::util::native_file_string(filename)) + "\"";
@@ -2054,6 +2086,25 @@ macromap<ContextT>::is_valid_concat(string_type new_value,
     return 1 == rescanned.size();
 }
 
+///////////////////////////////////////////////////////////////////////////////
+//
+//  Bulk update expand positions
+//
+///////////////////////////////////////////////////////////////////////////////
+// batch update tokens with a single expand position
+template <typename ContextT>
+template <typename ContainerT>
+void macromap<ContextT>::set_expand_positions(ContainerT &tokens, position_type pos)
+{
+    typename ContainerT::iterator ex_end = tokens.end();
+    for (typename ContainerT::iterator it = tokens.begin();
+         it != ex_end; ++it) {
+        // expand positions are only used for __LINE__, __FILE__, and macro names
+        if (token_id(*it) == T_IDENTIFIER)
+            it->set_expand_position(pos);
+    }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 //
 //  Handle all occurrences of the concatenation operator '##' inside the given

diff --git a/samples/cpp_tokens/slex_token.hpp b/samples/cpp_tokens/slex_token.hpp
@@ -20,6 +20,7 @@
 #include <boost/wave/token_ids.hpp>  
 #include <boost/wave/language_support.hpp>
 #include <boost/wave/util/file_position.hpp>
+#include <boost/optional.hpp>
 
 ///////////////////////////////////////////////////////////////////////////////
 namespace boost {
@@ -61,12 +62,19 @@ class slex_token
     operator token_id() const { return id; }
     string_type const &get_value() const { return value; }
     position_type const &get_position() const { return pos; }
+    position_type const &get_expand_position() const {
+        if (expand_pos)
+            return *expand_pos;
+        else
+            return pos;
+    }
     bool is_eoi() const { return id == T_EOI; }
     bool is_valid() const { return id != T_UNKNOWN; }
 
     void set_token_id (token_id id_) { id = id_; }
     void set_value (string_type const &newval) { value = newval; }
     void set_position (position_type const &pos_) { pos = pos_; }
+    void set_expand_position (position_type const &pos_) { expand_pos = pos_; }
 
     friend bool operator== (slex_token const& lhs, slex_token const& rhs)
     {
@@ -115,6 +123,7 @@ class slex_token
     boost::wave::token_id id;   // the token id
     string_type value;             // the text, which was parsed into this token
     PositionT pos;              // the original file position
+    boost::optional<PositionT> expand_pos;
 };
 
 template <typename PositionT>

diff --git a/samples/real_positions/real_position_token.hpp b/samples/real_positions/real_position_token.hpp
@@ -18,6 +18,7 @@
 #include <boost/wave/token_ids.hpp>  
 #include <boost/wave/language_support.hpp>
 #include <boost/detail/atomic_count.hpp>
+#include <boost/optional.hpp>
 
 ///////////////////////////////////////////////////////////////////////////////
 namespace impl {
@@ -61,6 +62,13 @@ class token_data
     position_type const &get_position() const { return pos; }
     position_type const &get_corrected_position() const 
         { return corrected_pos; }
+    position_type const &get_expand_position() const
+    {
+        if (expand_pos)
+            return *expand_pos;
+        else
+            return pos;
+    }
     bool is_eoi() const
     {
         return id == boost::wave::T_EOI;
@@ -71,6 +79,8 @@ class token_data
     void set_position (position_type const &pos_) { pos = pos_; }
     void set_corrected_position (position_type const &pos_) 
         { corrected_pos = pos_; }
+    void set_expand_position (position_type const & pos_)
+        { expand_pos = pos_; }
 
     friend bool operator== (token_data const& lhs, token_data const& rhs)
     {
@@ -84,6 +94,7 @@ class token_data
     string_type value;            // the text, which was parsed into this token
     position_type pos;            // the original file position
     position_type corrected_pos;  // the original file position
+    boost::optional<position_type> expand_pos;  // where this token was expanded
     boost::detail::atomic_count refcnt;
 };
 
@@ -157,6 +168,8 @@ class lex_token
         { return data->get_position(); }
     position_type const &get_corrected_position() const 
         { return data->get_corrected_position(); }
+    position_type const &get_expand_position() const
+        { return data->get_expand_position(); }
     bool is_valid() const
     {
         using namespace boost::wave;
@@ -171,6 +184,8 @@ class lex_token
         { make_unique(); data->set_position(pos_); }
     void set_corrected_position (position_type const &pos_) 
         { make_unique(); data->set_corrected_position(pos_); }
+    void set_expand_position (position_type const &pos_)
+        { make_unique(); data->set_expand_position(pos_); }
 
     friend bool operator== (lex_token const& lhs, lex_token const& rhs)
     {

diff --git a/test/testwave/testfiles/t_5_036.cpp b/test/testwave/testfiles/t_5_036.cpp
@@ -0,0 +1,28 @@
+/*=============================================================================
+    Boost.Wave: A Standard compliant C++ preprocessor library
+    http://www.boost.org/
+
+    Copyright (c) 2020 Jeff Trull. Distributed under the Boost
+    Software License, Version 1.0. (See accompanying file
+    LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+    Part of the test included in this file was taken from a tweet by
+    Eric Niebler: https://twitter.com/ericniebler/status/1252660334545866752
+=============================================================================*/
+
+// Test __LINE__ and __FILE__ in a context where the invocation of the macro
+// that uses them is split across two lines
+
+// __LINE__ should reflect the position where the macros are expanded
+#define FOO(X) __LINE__ __FILE__ BAR
+#define BAR(X) __LINE__ __FILE__
+FOO(X)
+  (Y)
+//R #line 19 "t_5_036.cpp"
+//R 19 "$F" 19 "$F"
+
+// now use those same macros in a different file -
+// __FILE__ should report that.
+#include "t_5_036.hpp"
+//R #line 11 "t_5_036.hpp"
+//R 11 "$P(t_5_036.hpp)" 11 "$P(t_5_036.hpp)"
diff --git a/test/testwave/testfiles/t_5_036.hpp b/test/testwave/testfiles/t_5_036.hpp
@@ -0,0 +1,12 @@
+/*=============================================================================
+    Boost.Wave: A Standard compliant C++ preprocessor library
+    http://www.boost.org/
+
+    Copyright (c) 2020 Jeff Trull. Distributed under the Boost
+    Software License, Version 1.0. (See accompanying file
+    LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+
+// use macro defined in the parent file
+FOO(X)
+  (Y)