main: add nulltag/z, a new extra

Close #4151. Consider a parser attempting to emit a null tag, a tag whose name is the empty string '\0'. Original Behavior: It warns "ignoring null tag..." if both parserDefinition::allowNullTag and tagEntryInfo::allowNullTag are unset. It does not warn if either parserDefinition::allowNullTag or tagEntryInfo::allowNullTag is set. It does not emit the null tag, even if allowNullTag is set. With This Change: The code now emits the null tag if: Either parserDefinition::allowNullTag or tagEntryInfo::allowNullTag is set, The --extras=+z (or --extras=+{nulltag}) option is specified, and The output format supports to emit null tags. The xref and json output formats support to emit null tags. Signed-off-by: Masatake YAMATO <yamato@redhat.com>
universal-ctags · Dec 19, 2024 · bddcc32 · bddcc32
1 parent fed6414
commit bddcc32
Show file tree

Hide file tree

Showing 20 changed files with 160 additions and 2 deletions.
diff --git a/Tmain/extras-long.d/stdout-expected.txt b/Tmain/extras-long.d/stdout-expected.txt
@@ -7,6 +7,7 @@ p       pseudo              no      NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           no      NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -25,6 +26,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           no      NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -43,6 +45,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           no      NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -61,6 +64,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           no      NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           yes     NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -79,6 +83,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           yes     NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           yes     NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             no      NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class

diff --git a/Tmain/json-output-format.d/stdout-expected.txt b/Tmain/json-output-format.d/stdout-expected.txt
@@ -20,6 +20,7 @@
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "fileScope", "pattern": "Include tags of file scope"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "guest", "pattern": "Include tags generated by guest parsers"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "inputFile", "pattern": "Include an entry for the base file name of every input file"}
+{"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "nulltag", "pattern": "Include tags with empty strings as their names"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "pseudo", "pattern": "Include pseudo tags"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "qualified", "pattern": "Include an extra class-qualified tag entry for each tag"}
 {"_type": "ptag", "name": "TAG_EXTRA_DESCRIPTION", "path": "reference", "pattern": "Include reference tags"}

diff --git a/Tmain/kind-abnormal-spec.d/stdout-expected.txt b/Tmain/kind-abnormal-spec.d/stdout-expected.txt
@@ -12,6 +12,8 @@ r  emit a tag with multi roles
 R  emit a tag with multi roles(disabled by default) [off]
 f  tag for testing field:
 n  trigger notice output
+z  emit a tag having an empty string
+Z  don't emit a tag having an empty string
 
 # list kinds-full
 #LETTER NAME                                      ENABLED REFONLY NROLES MASTER DESCRIPTION
@@ -22,12 +24,14 @@ L       ThisShouldNotBePrintedKindNameMustBeGiven yes     no      0      NONE
 N       nothingSpecial                            yes     no      0      NONE   emit a normal tag
 Q       quit                                      yes     no      0      NONE   stop the parsing
 R       rolesDisabled                             no      yes     2      NONE   emit a tag with multi roles(disabled by default)
+Z       dontEmitNullTag                           yes     no      0      NONE   don't emit a tag having an empty string
 b       broken tag                                yes     no      1      NONE   name with unwanted characters
 d       disabled                                  no      no      2      NONE   a kind disabled by default
 e       enabled                                   yes     no      2      NONE   a kind enabled by default
 f       fieldMaker                                yes     no      0      NONE   tag for testing field:
 n       triggerNotice                             yes     no      0      NONE   trigger notice output
 r       roles                                     yes     yes     4      NONE   emit a tag with multi roles
+z       emitNullTag                               yes     no      0      NONE   emit a tag having an empty string
 
 # +K
 abnormal kindDefinition testing (no letter)	input.x	/^@$/;"	no letter

diff --git a/Tmain/list-extras.d/stdout-expected.txt b/Tmain/list-extras.d/stdout-expected.txt
@@ -7,6 +7,7 @@ p       pseudo              yes     NONE        no    Include pseudo tags
 q       qualified           yes     NONE        no    Include an extra class-qualified tag entry for each tag
 r       reference           yes     NONE        no    Include reference tags
 s       subparser           yes     NONE        no    Include tags generated by subparsers
+z       nulltag             yes     NONE        no    Include tags with empty strings as their names
 -       canonicalizedName   yes     Automake    no    Include canonicalized object name like libctags_a
 -       linkName            no      Fortran     no    Linking name used in foreign languages
 -       implicitClass       no      GDScript    no    Include tag for the implicitly defined unnamed class
@@ -25,6 +26,7 @@ p	pseudo	yes	NONE	no	Include pseudo tags
 q	qualified	yes	NONE	no	Include an extra class-qualified tag entry for each tag
 r	reference	yes	NONE	no	Include reference tags
 s	subparser	yes	NONE	no	Include tags generated by subparsers
+z	nulltag	yes	NONE	no	Include tags with empty strings as their names
 -	canonicalizedName	yes	Automake	no	Include canonicalized object name like libctags_a
 -	linkName	no	Fortran	no	Linking name used in foreign languages
 -	implicitClass	no	GDScript	no	Include tag for the implicitly defined unnamed class
@@ -43,3 +45,4 @@ p       pseudo    no      NONE     no    Include pseudo tags
 q       qualified no      NONE     no    Include an extra class-qualified tag entry for each tag
 r       reference no      NONE     no    Include reference tags
 s       subparser no      NONE     no    Include tags generated by subparsers
+z       nulltag   no      NONE     no    Include tags with empty strings as their names
diff --git a/Tmain/nulltag-extra.d/input.cst b/Tmain/nulltag-extra.d/input.cst
@@ -0,0 +1,3 @@
+Z
+z
+
diff --git a/Tmain/nulltag-extra.d/run.sh b/Tmain/nulltag-extra.d/run.sh
@@ -0,0 +1,30 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+. ../utils.sh
+
+# The order of stdout and stderr lines is not stable.
+exit_if_win32 "$CTAGS"
+
+# is_feature_available $CTAGS json
+
+O="--options=NONE --language-force=CTagsSelfTest"
+
+for fmt in xref; do
+	echo "# no extra ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" input.cst 2>&1
+
+	echo "# drop '0' extra ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" --extras=-z input.cst 2>&1
+
+	echo "# drop '{nulltag}' extra ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" --extras=-'{nulltag}' input.cst 2>&1
+
+	echo '# with --extras=+0 ($fmt)'
+	${CTAGS} $O -o - --output-format="$fmt" --extras=+z input.cst 2>&1
+
+	echo "# with --extras=+{nulltag}' ($fmt)"
+	${CTAGS} $O -o - --output-format="$fmt" --extras=+'{nulltag}' input.cst 2>&1
+done | sed -e 's/\.exe//'
diff --git a/Tmain/nulltag-extra.d/stderr-expected.txt b/Tmain/nulltag-extra.d/stderr-expected.txt
diff --git a/Tmain/nulltag-extra.d/stdout-expected.txt b/Tmain/nulltag-extra.d/stdout-expected.txt
@@ -0,0 +1,17 @@
+# no extra (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+# drop '0' extra (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+# drop '{nulltag}' extra (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+# with --extras=+0 ($fmt)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+                 emitNullTag    2 input.cst        z
+# with --extras=+{nulltag}' (xref)
+ctags: Notice: No options will be read from files or environment
+ctags: Notice: ignoring null tag in input.cst(line: 1, language: CTagsSelfTest)
+                 emitNullTag    2 input.cst        z
diff --git a/docs/man/ctags.1.rst b/docs/man/ctags.1.rst
@@ -1835,6 +1835,8 @@ roles of tags to include in the output file for a particular language.
 Inquire the output of "``ctags --list-roles``" for the list of
 roles.
 
+.. _extras:
+
 Extras
 ~~~~~~
 
@@ -1927,6 +1929,24 @@ The meaning of major extras is as follows (long-name flag/one-letter flag):
 
 	The etags mode enables the ``Unknown`` parser implicitly.
 
+``nulltag``/``z``
+	Include tags (*null tags*) having empty strings as their names.
+	Generally speaking, trying to make a null tag is a sign of a parser bug
+	or broken input. ctags warns such trying or throws the
+	null tag away. To suppress the warnings, use ``--quiet`` option.
+
+	On the other hand, null tags are valid in some languages.
+	Consider ``{"":  val}`` in a JavaScript sourece code. The empty string is
+	valid as a key. If a parser intentionally makes a null tag (a valid null tag),
+	ctags doesn't warn but discard it by default.
+
+	The discards are due because some output formats may not consider null tags.
+
+	With ``nulltag``/``z`` extra, you can force ctags to emit the nulltags. This extra
+	is effective only if the output format supports null tags.
+
+	(since version 6.2.0)
+
 ``pseudo``/``p``
 	Include pseudo-tags. Enabled by default unless the tag file is
 	written to standard output. See :ref:`ctags-client-tools(7) <ctags-client-tools(7)>` about

diff --git a/docs/news/HEAD.rst b/docs/news/HEAD.rst
@@ -5,6 +5,16 @@ Changes in 6.?.0
 New and extended options and their flags
 ---------------------------------------------------------------------
 
+``nulltag``/``z`` extra
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Universal Ctags now supports tags (*null tags*) having empty strings as their names.
+See :ref:`extras` in :ref:`ctags(1) <ctags(1)>`.
+
+.. note::
+
+   * ``libreadtags`` and ``readtags`` do not support the null tags yet.
+   * ``json`` and ``xref`` output formats support the null tags.
+
 Incompatible changes
 ---------------------------------------------------------------------
 

diff --git a/main/entry.c b/main/entry.c
@@ -1770,6 +1770,15 @@ static void writeTagEntry (tagEntryInfo *const tag)
 
 	DebugStatement ( debugEntry (tag); )
 
+	if (isTagExtraBitMarked(tag, XTAG_NULLTAG))
+	{
+		if (!writerCanPrintNullTag())
+			return;
+
+		if (!isXtagEnabled(XTAG_NULLTAG))
+			return;
+	}
+
 #ifdef _WIN32
 	if (getFilenameSeparator(Option.useSlashAsFilenameSeparator) == FILENAME_SEP_USE_SLASH)
 	{
@@ -1939,10 +1948,17 @@ extern int makeTagEntry (tagEntryInfo *const tag)
 	if (tag->name [0] == '\0' && (!tag->placeholder))
 	{
 		if (! tag->allowNullTag)
+		{
 			error (NOTICE, "ignoring null tag in %s(line: %lu, language: %s)",
 				   getInputFileName (), tag->lineNumber,
 				   getLanguageName (tag->langType));
-		goto out;
+			goto out;
+		}
+
+		/* writeTagEntry decides whether ctags emits this tag or not.
+		 * At this point, we just mark the tag as a null tag. */
+		if (! tag->placeholder)
+			markTagExtraBit(tag, XTAG_NULLTAG);
 	}
 
 	if (TagFile.cork)

diff --git a/main/parse.c b/main/parse.c
@@ -5550,6 +5550,8 @@ typedef enum {
 	K_ROLES_DISABLED,
 	K_FIELD_TESTING,
 	K_TRIGGER_NOTICE,
+	K_EMIT_NULL_TAG,
+	K_DONT_EMIT_NULL_TAG,
 	KIND_COUNT
 } CTST_Kind;
 
@@ -5631,6 +5633,8 @@ static kindDefinition CTST_Kinds[KIND_COUNT] = {
 	 .referenceOnly = true, ATTACH_ROLES (CTST_RolesDisabledKindRoles)},
 	{true,  'f', "fieldMaker", "tag for testing field:" },
 	{true,  'n', "triggerNotice", "trigger notice output"},
+	{true,  'z', "emitNullTag", "emit a tag having an empty string"},
+	{true,  'Z', "dontEmitNullTag", "don't emit a tag having an empty string"},
 };
 
 typedef enum {
@@ -5820,6 +5824,16 @@ static void createCTSTTags (void)
 					case K_TRIGGER_NOTICE:
 						notice ("notice output for testing: %s", CTST_Kinds [i].name);
 						break;
+					case K_EMIT_NULL_TAG:
+						initTagEntry (&e, "", i);
+						e.allowNullTag = 1;
+						makeTagEntry (&e);
+						break;
+					case K_DONT_EMIT_NULL_TAG:
+						initTagEntry (&e, "", i);
+						e.allowNullTag = 0;
+						makeTagEntry (&e);
+						break;
 				}
 
 				if (quit)

diff --git a/main/writer-ctags.c b/main/writer-ctags.c
@@ -55,6 +55,7 @@ tagWriter uCtagsWriter = {
 	.rescanFailedEntry = NULL,
 	.treatFieldAsFixed = treatFieldAsFixed,
 	.checkOptions = checkCtagsOptions,
+	.canPrintNullTag = false,
 #ifdef _WIN32
 	.overrideFilenameSeparator = overrideFilenameSeparator,
 #endif
@@ -96,6 +97,7 @@ tagWriter eCtagsWriter = {
 	.rescanFailedEntry = NULL,
 	.treatFieldAsFixed = treatFieldAsFixed,
 	.checkOptions = checkCtagsOptions,
+	.canPrintNullTag = false,
 	.defaultFileName = CTAGS_FILE,
 };
 

diff --git a/main/writer-json.c b/main/writer-json.c
@@ -62,6 +62,7 @@ tagWriter jsonWriter = {
 	.postWriteEntry = NULL,
 	.rescanFailedEntry = NULL,
 	.treatFieldAsFixed = NULL,
+	.canPrintNullTag = true,
 	.defaultFileName = NULL,
 };
 
@@ -313,6 +314,7 @@ tagWriter jsonWriter = {
 	.preWriteEntry = NULL,
 	.postWriteEntry = NULL,
 	.defaultFileName = "-",
+	.canPrintNullTag = false,
 };
 
 extern bool ptagMakeJsonOutputVersion (ptagDesc *desc, langType language CTAGS_ATTR_UNUSED,

diff --git a/main/writer-xref.c b/main/writer-xref.c
@@ -37,6 +37,7 @@ tagWriter xrefWriter = {
 	.postWriteEntry = NULL,
 	.rescanFailedEntry = NULL,
 	.treatFieldAsFixed = NULL,
+	.canPrintNullTag = true,
 	.defaultFileName = NULL,
 };
 

diff --git a/main/writer.c b/main/writer.c
@@ -117,6 +117,10 @@ extern bool writerCanPrintPtag (void)
 	return (writer->writePtagEntry)? true: false;
 }
 
+extern bool writerCanPrintNullTag (void)
+{
+	return writer->canPrintNullTag;
+}
 extern bool writerDoesTreatFieldAsFixed (int fieldType)
 {
 	if (writer->treatFieldAsFixed)

diff --git a/main/writer_p.h b/main/writer_p.h
@@ -54,6 +54,8 @@ struct sTagWriter {
 
 	void (* checkOptions) (tagWriter *writer, bool fieldsWereReset);
 
+	bool canPrintNullTag;
+
 #ifdef _WIN32
 	enum filenameSepOp (* overrideFilenameSeparator) (enum filenameSepOp currentSetting);
 #endif	/* _WIN32 */
@@ -95,6 +97,7 @@ extern bool ptagMakeCtagsOutputFilesep (ptagDesc *desc, langType language CTAGS_
 extern bool ptagMakeCtagsOutputExcmd (ptagDesc *desc, langType language CTAGS_ATTR_UNUSED, const void *data);
 
 extern bool writerCanPrintPtag (void);
+extern bool writerCanPrintNullTag (void);
 extern bool writerDoesTreatFieldAsFixed (int fieldType);
 
 extern void writerCheckOptions (bool fieldsWereReset);

diff --git a/main/xtag.c b/main/xtag.c
@@ -77,6 +77,8 @@ static xtagDefinition xtagDefinitions [] = {
 	  "Include tags generated by subparsers"},
 	{ true, '\0', "anonymous",
 	  "Include tags for non-named objects like lambda"},
+	{ false, 'z', "nulltag",
+	  "Include tags with empty strings as their names"},
 };
 
 static unsigned int       xtagObjectUsed;

diff --git a/main/xtag.h b/main/xtag.h
@@ -34,6 +34,7 @@ typedef enum eXtagType { /* extra tag content control */
 	XTAG_TAGS_GENERATED_BY_GUEST_PARSERS = XTAG_GUEST, /* Geany uses the old name */
 	XTAG_SUBPARSER,
 	XTAG_ANONYMOUS,
+	XTAG_NULLTAG,
 
 	XTAG_COUNT
 } xtagType;
@@ -42,7 +43,7 @@ struct sXtagDefinition {
 	bool enabled;
 	/* letter, and ftype are initialized in the main part,
 	   not in a parser. */
-#define NUL_XTAG_LETTER '\0'
+#define NUL_XTAG_LETTER '\0'	/* Nothing todo with NULLTAG. */
 	unsigned char letter;
 	const char* name;	 /* used in extra: field */
 	const char* description;  /* displayed in --list-extra output */

diff --git a/man/ctags.1.rst.in b/man/ctags.1.rst.in
@@ -1835,6 +1835,8 @@ roles of tags to include in the output file for a particular language.
 Inquire the output of "``@CTAGS_NAME_EXECUTABLE@ --list-roles``" for the list of
 roles.
 
+.. _extras:
+
 Extras
 ~~~~~~
 
@@ -1927,6 +1929,24 @@ The meaning of major extras is as follows (long-name flag/one-letter flag):
 
 	The etags mode enables the ``Unknown`` parser implicitly.
 
+``nulltag``/``z``
+	Include tags (*null tags*) having empty strings as their names.
+	Generally speaking, trying to make a null tag is a sign of a parser bug
+	or broken input. @CTAGS_NAME_EXECUTABLE@ warns such trying or throws the
+	null tag away. To suppress the warnings, use ``--quiet`` option.
+
+	On the other hand, null tags are valid in some languages.
+	Consider ``{"":  val}`` in a JavaScript sourece code. The empty string is
+	valid as a key. If a parser intentionally makes a null tag (a valid null tag),
+	@CTAGS_NAME_EXECUTABLE@ doesn't warn but discard it by default.
+
+	The discards are due because some output formats may not consider null tags.
+
+	With ``nulltag``/``z`` extra, you can force ctags to emit the nulltags. This extra
+	is effective only if the output format supports null tags.
+
+	(since version 6.2.0)
+
 ``pseudo``/``p``
 	Include pseudo-tags. Enabled by default unless the tag file is
 	written to standard output. See ctags-client-tools(7) about