lregex: add {postrun} regex flag

Signed-off-by: Masatake YAMATO <yamato@redhat.com> (cherry picked from commit 6c774ab)
universal-ctags · Dec 11, 2023 · 123383f · 123383f
1 parent f9065da
commit 123383f
Show file tree

Hide file tree

Showing 11 changed files with 151 additions and 12 deletions.
diff --git a/Tmain/list-regex-flags.d/stdout-expected.txt b/Tmain/list-regex-flags.d/stdout-expected.txt
@@ -6,6 +6,7 @@ p       pcre2                                         use pcre2 regex engine
 x       exclusive                                     skip testing the other patterns if a line is matched to this pattern
 -       fatal="MESSAGE"                               print the given MESSAGE and exit
 -       placeholder                                   don't put this tag to tags file.
+-       postrun                                       run after parsing with built-in code, multline regex patterns, and multitable regex patterns
 -       scope=ACTION                                  use scope stack: ACTION = ref|push|pop|clear|set|replace
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX

diff --git a/Units/regex-flag-postrun.d/args.ctags b/Units/regex-flag-postrun.d/args.ctags
@@ -0,0 +1,7 @@
+--sort=no
+--fields=+ln
+
+--langdef=TESTC{base=C}
+--kinddef-TESTC=d,def,definitions
+--regex-TESTC=/.*def:([0-za-zA-Z]+).*/inline-\1/d/
+--regex-TESTC=/.*def:([0-za-zA-Z]+).*/post-\1/d/{postrun}
diff --git a/Units/regex-flag-postrun.d/expected.tags b/Units/regex-flag-postrun.d/expected.tags
@@ -0,0 +1,3 @@
+inline-x	input.c	/^\/* def:x *\/$/;"	d	line:1	language:TESTC
+f	input.c	/^int f(void)$/;"	f	line:2	language:C	typeref:typename:int
+post-x	input.c	/^\/* def:x *\/$/;"	d	line:1	language:TESTC
diff --git a/Units/regex-flag-postrun.d/input.c b/Units/regex-flag-postrun.d/input.c
@@ -0,0 +1,5 @@
+/* def:x */
+int f(void)
+{
+	return 0;
+}
diff --git a/docs/man/ctags-optlib.7.rst b/docs/man/ctags-optlib.7.rst
@@ -334,6 +334,34 @@ representation. ``--list-regex-flags`` lists all the flags.
 	can be an empty string.  See the following description of
 	``scope=...`` flag about how this is useful.
 
+``postrun``
+	Match the pattern at the end of all the parsing processes,
+	including running build-in parsers, applying
+	``--mline-regex-<LANG>`` patterns, and applying
+	``--_mtable-regex-<LANG>`` patterns. This flag is useful for
+	extending a built-in parser with the ``--regex-<LANG>=`` option.
+
+	A built-in parser processes source files line by line provided by
+	the main part of ctags.  The main part of
+	ctags applies ``--regex-<LANG>=`` patterns to a
+	line when providing the line to the built-in parser. Both the
+	built-in parser for ``<LANG>`` and the patterns of
+	``--regex-<LANG>=`` without ``postrun`` run side-by-side.
+
+	Thus, ``--regex-<LANG>=`` patterns without ``postrun`` cannot use
+	the tags information finally extracted by the built-in parser for
+	``<LANG>``.  This is where the ``postrun`` comes into play. The
+	main part never applies patterns with ``postrun`` when providing
+	lines to built-in parsers. Instead, it applies the patterns after
+	running the built-in parser.  In other words, it applies the
+	patterns to a source file after providing all the file lines. The
+	patterns can utilize the tags information completely extracted by
+	the built-in parser.
+
+	``--mline-regex-<LANG>`` and ``--_mtable-regex-<LANG>`` have no
+	``{postrun}`` flag because the main part always applies the patterns
+	specified with the options after running a built-in parser.
+
 ``scope=(ref|push|pop|clear|set|replace)``
 
 	Specify what to do with the internal scope stack.

diff --git a/main/lregex.c b/main/lregex.c
@@ -142,6 +142,7 @@ typedef struct {
 	enum pType type;
 	bool exclusive;
 	bool accept_empty_name;
+	bool postrun;
 	union {
 		struct {
 			int kindIndex;
@@ -596,9 +597,17 @@ static void pre_ptrn_flag_exclusive_long (const char* const s CTAGS_ATTR_UNUSED,
 	pre_ptrn_flag_exclusive_short ('x', data);
 }
 
+static void pre_ptrn_flag_postrun_long (const char* const s CTAGS_ATTR_UNUSED, const char* const unused CTAGS_ATTR_UNUSED, void* data)
+{
+	regexPattern * ptrn = data;
+	ptrn->postrun = true;
+}
+
 static flagDefinition prePtrnFlagDef[] = {
 	{ 'x',  "exclusive", pre_ptrn_flag_exclusive_short, pre_ptrn_flag_exclusive_long ,
 	  NULL, "skip testing the other patterns if a line is matched to this pattern"},
+	{ '\0', "postrun",   NULL, pre_ptrn_flag_postrun_long,
+	  NULL, "run after parsing with built-in code, multline regex patterns, and multitable regex patterns" },
 };
 
 static void scope_ptrn_flag_eval (const char* const f  CTAGS_ATTR_UNUSED,
@@ -689,6 +698,7 @@ static regexPattern * newPattern (regexCompiledCode* const pattern,
 	ptrn->pattern.code = pattern->code;
 
 	ptrn->exclusive = false;
+	ptrn->postrun = false;
 	ptrn->accept_empty_name = false;
 	ptrn->regptype = regptype;
 	ptrn->xtagType = XTAG_UNKNOWN;
@@ -2128,7 +2138,21 @@ static bool matchMultilineRegexPattern (struct lregexControlBlock *lcb,
 /* Match against all patterns for specified language. Returns true if at least
  * on pattern matched.
  */
-extern bool matchRegex (struct lregexControlBlock *lcb, const vString* const line)
+extern bool regexIsPostRun (struct lregexControlBlock *lcb)
+{
+	for (unsigned int i = 0  ;  i < ptrArrayCount(lcb->entries[REG_PARSER_SINGLE_LINE])  ;  ++i)
+	{
+		regexTableEntry *entry = ptrArrayItem(lcb->entries[REG_PARSER_SINGLE_LINE], i);
+		regexPattern *ptrn = entry->pattern;
+
+		if (ptrn->postrun)
+			return true;
+	}
+
+	return false;
+}
+
+extern bool matchRegex (struct lregexControlBlock *lcb, const vString* const line, bool postrun)
 {
 	bool result = false;
 	unsigned int i;
@@ -2139,6 +2163,9 @@ extern bool matchRegex (struct lregexControlBlock *lcb, const vString* const lin
 
 		Assert (ptrn);
 
+		if (postrun != ptrn->postrun)
+			continue;
+
 		if ((ptrn->xtagType != XTAG_UNKNOWN)
 			&& (!isXtagEnabled (ptrn->xtagType)))
 				continue;

diff --git a/main/lregex_p.h b/main/lregex_p.h
@@ -89,7 +89,8 @@ extern void addTagMultiTableRegex(struct lregexControlBlock *lcb,
 								  const char* const name, const char* const kinds, const char* const flags,
 								  bool *disabled);
 
-extern bool matchRegex (struct lregexControlBlock *lcb, const vString* const line);
+extern bool regexIsPostRun (struct lregexControlBlock *lcb);
+extern bool matchRegex (struct lregexControlBlock *lcb, const vString* const line, bool postrun);
 extern bool doesExpectCorkInRegex (struct lregexControlBlock *lcb);
 extern void addCallbackRegex (struct lregexControlBlock *lcb,
 							  const char* const regex,

diff --git a/main/parse.c b/main/parse.c
@@ -4580,6 +4580,11 @@ static bool lregexQueryParserAndSubparsers (const langType language, bool (* pre
 	return r;
 }
 
+extern bool hasLanguagePostRunRegexPatterns (const langType language)
+{
+	return lregexQueryParserAndSubparsers (language, regexIsPostRun);
+}
+
 extern bool hasLanguageMultilineRegexPatterns (const langType language)
 {
 	return lregexQueryParserAndSubparsers (language, regexNeedsMultilineBuffer);
@@ -4603,16 +4608,16 @@ extern bool doesLanguageExpectCorkInRegex (const langType language)
 	return hasScopeAction;
 }
 
-extern void matchLanguageRegex (const langType language, const vString* const line)
+extern void matchLanguageRegex (const langType language, const vString* const line, bool postrun)
 {
 	subparser *tmp;
 
-	matchRegex ((LanguageTable + language)->lregexControlBlock, line);
+	matchRegex ((LanguageTable + language)->lregexControlBlock, line, postrun);
 	foreachSubparser(tmp, true)
 	{
 		langType t = getSubparserLanguage (tmp);
 		enterSubparser (tmp);
-		matchLanguageRegex (t, line);
+		matchLanguageRegex (t, line, postrun);
 		leaveSubparser ();
 	}
 }

diff --git a/main/parse_p.h b/main/parse_p.h
@@ -145,7 +145,8 @@ extern bool processLanguageRegexOption (langType language, enum regexParserType
 extern void notifyLanguageRegexInputStart (langType language);
 extern void notifyLanguageRegexInputEnd (langType language);
 
-extern void matchLanguageRegex (const langType language, const vString* const line);
+extern bool hasLanguagePostRunRegexPatterns (const langType language);
+extern void matchLanguageRegex (const langType language, const vString* const line, bool postrun);
 extern void freeRegexResources (void);
 extern bool checkRegex (void);
 extern void useRegexMethod (const langType language);

diff --git a/main/read.c b/main/read.c
@@ -69,6 +69,7 @@ typedef struct sComputPos {
 	long    offset;
 	bool open;
 	int crAdjustment;
+	size_t posInAllLines;
 } compoundPos;
 
 typedef struct sInputLineFposMap {
@@ -337,7 +338,7 @@ static void allocLineFposMap (inputLineFposMap *lineFposMap)
 }
 
 static void appendLineFposMap (inputLineFposMap *lineFposMap, compoundPos *pos,
-							   bool crAdjustment)
+							   bool crAdjustment, size_t posInAllLines)
 {
 	int lastCrAdjustment = 0;
 
@@ -359,6 +360,7 @@ static void appendLineFposMap (inputLineFposMap *lineFposMap, compoundPos *pos,
 	lineFposMap->pos [lineFposMap->count].open = true;
 	lineFposMap->pos [lineFposMap->count].crAdjustment
 		= lastCrAdjustment + ((crAdjustment)? 1: 0);
+	lineFposMap->pos [lineFposMap->count].posInAllLines = posInAllLines;
 	lineFposMap->count++;
 }
 
@@ -801,7 +803,8 @@ extern void resetInputFile (const langType language)
 	vStringClear (File.line);
 	File.ungetchIdx = 0;
 
-	if (hasLanguageMultilineRegexPatterns (language))
+	if (hasLanguageMultilineRegexPatterns (language)
+		|| hasLanguagePostRunRegexPatterns (language))
 		File.allLines = vStringNew ();
 
 	resetLangOnStack (& inputLang, language);
@@ -837,13 +840,13 @@ extern void *getInputFileUserData(void)
 
 /*  Action to take for each encountered input newline.
  */
-static void fileNewline (bool crAdjustment)
+static void fileNewline (bool crAdjustment, size_t posInAllLines)
 {
 	File.filePosition = StartOfLine;
 
 	if (BackupFile.mio == NULL)
 		appendLineFposMap (&File.lineFposMap, &File.filePosition,
-						   crAdjustment);
+						   crAdjustment, posInAllLines);
 
 	File.input.lineNumber++;
 	File.source.lineNumber++;
@@ -927,7 +930,7 @@ static vString *iFileGetLine (bool chop_newline)
 	if (vStringLength (File.line) > 0)
 	{
 		/* Use StartOfLine from previous iFileGetLine() call */
-		fileNewline (eol == eol_cr_nl);
+		fileNewline (eol == eol_cr_nl, File.allLines? vStringLength(File.allLines): 0);
 		/* Store StartOfLine for the next iFileGetLine() call */
 		mio_getpos (File.mio, &StartOfLine.pos);
 		StartOfLine.offset = mio_tell (File.mio);
@@ -940,7 +943,7 @@ static vString *iFileGetLine (bool chop_newline)
 
 		bool chopped = vStringStripNewline (File.line);
 
-		matchLanguageRegex (lang, File.line);
+		matchLanguageRegex (lang, File.line, false);
 
 		if (chopped && !chop_newline)
 			vStringPutNewlinAgainUnsafe (File.line);
@@ -954,6 +957,36 @@ static vString *iFileGetLine (bool chop_newline)
 			matchLanguageMultilineRegex (lang, File.allLines);
 			matchLanguageMultitableRegex (lang, File.allLines);
 
+			if (hasLanguagePostRunRegexPatterns (lang))
+			{
+
+				unsigned input_ln = File.input.lineNumber;
+				unsigned source_ln = File.source.lineNumber;
+				MIOPos pos = File.filePosition.pos;
+
+				vString *line = vStringNew();
+				for (size_t i = 0; i < File.lineFposMap.count; i++)
+				{
+					File.input.lineNumber = i + 1;
+					File.source.lineNumber = File.input.lineNumber;
+					File.filePosition.pos = File.lineFposMap.pos[i].pos;
+
+					vStringNCopySUnsafe(line,
+										vStringValue(File.allLines) +
+										File.lineFposMap.pos[i].posInAllLines,
+										(((i + 1) < File.lineFposMap.count)
+										 ? File.lineFposMap.pos[i+1].posInAllLines
+										 : vStringLength (File.allLines))
+										- File.lineFposMap.pos[i].posInAllLines);
+					matchLanguageRegex (lang, line, true);
+				}
+				vStringDelete(line);
+
+				File.filePosition.pos = pos;
+				File.input.lineNumber = input_ln;
+				File.source.lineNumber = source_ln;
+			}
+
 			/* To limit the execution of multiline/multitable parser(s) only
 			   ONCE, clear File.allLines field. */
 			vStringDelete (File.allLines);

diff --git a/man/ctags-optlib.7.rst.in b/man/ctags-optlib.7.rst.in
@@ -334,6 +334,34 @@ representation. ``--list-regex-flags`` lists all the flags.
 	can be an empty string.  See the following description of
 	``scope=...`` flag about how this is useful.
 
+``postrun``
+	Match the pattern at the end of all the parsing processes,
+	including running build-in parsers, applying
+	``--mline-regex-<LANG>`` patterns, and applying
+	``--_mtable-regex-<LANG>`` patterns. This flag is useful for
+	extending a built-in parser with the ``--regex-<LANG>=`` option.
+
+	A built-in parser processes source files line by line provided by
+	the main part of @CTAGS_NAME_EXECUTABLE@.  The main part of
+	@CTAGS_NAME_EXECUTABLE@ applies ``--regex-<LANG>=`` patterns to a
+	line when providing the line to the built-in parser. Both the
+	built-in parser for ``<LANG>`` and the patterns of
+	``--regex-<LANG>=`` without ``postrun`` run side-by-side.
+
+	Thus, ``--regex-<LANG>=`` patterns without ``postrun`` cannot use
+	the tags information finally extracted by the built-in parser for
+	``<LANG>``.  This is where the ``postrun`` comes into play. The
+	main part never applies patterns with ``postrun`` when providing
+	lines to built-in parsers. Instead, it applies the patterns after
+	running the built-in parser.  In other words, it applies the
+	patterns to a source file after providing all the file lines. The
+	patterns can utilize the tags information completely extracted by
+	the built-in parser.
+
+	``--mline-regex-<LANG>`` and ``--_mtable-regex-<LANG>`` have no
+	``{postrun}`` flag because the main part always applies the patterns
+	specified with the options after running a built-in parser.
+
 ``scope=(ref|push|pop|clear|set|replace)``
 
 	Specify what to do with the internal scope stack.