Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix parsing issues #149

Merged
merged 5 commits into from
Apr 9, 2021
Merged

Fix parsing issues #149

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ced05bb
Reference to issues #148, #147, #143
axunonb Apr 2, 2021
3d80b53
Reference to issues #148, #147, #143
axunonb Apr 2, 2021
c33c94a
Merge branch 'Fix_parsing_issues' of https://github.com/axuno/SmartFo…
axunonb Apr 2, 2021
1ed6107
Updated README.md
axunonb Apr 2, 2021
2c542c9
Fix for https://github.com/axuno/SmartFormat/pull/149#issuecomment-81…
axunonb Apr 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Reference to issues #148, #147, #143
  • Loading branch information
@axunonb
axunonb committed Apr 2, 2021
commit 3d80b53e6cb74f3162571e0a679a8285a96bf231
223 changes: 183 additions & 40 deletions src/SmartFormat.Tests/Core/ParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ public class ParserTests
[Test]
public void TestParser()
{
var parser = new SmartFormatter() {Settings = { ParseErrorAction = ErrorAction.ThrowError}}.Parser;
var parser = new SmartFormatter {Settings = { ParseErrorAction = ErrorAction.ThrowError}}.Parser;
parser.AddAlphanumericSelectors();
parser.AddAdditionalSelectorChars("_");
parser.AddOperators(".");
Expand All @@ -35,32 +35,26 @@ public void TestParser()
results.TryAll(r => Assert.AreEqual(r.format, r.parsed.ToString())).ThrowIfNotEmpty();
}

[Test]
public void Parser_Throws_Exceptions()
[TestCase("{")]
[TestCase("{0")]
[TestCase("}")]
[TestCase("0}")]
[TestCase("{{{")]
[TestCase("}}}")]
[TestCase("{.}")]
[TestCase("{.:}")]
[TestCase("{..}")]
[TestCase("{..:}")]
[TestCase("{0.}")]
[TestCase("{0.:}")]
public void Parser_Throws_Exceptions(string format)
{
// Let's set the "ErrorAction" to "Throw":
var formatter = Smart.CreateDefaultSmartFormat();
formatter.Settings.ParseErrorAction = ErrorAction.ThrowError;

var args = new object[] { TestFactory.GetPerson() };
var invalidFormats = new[] {
"{",
"{0",
"}",
"0}",
"{{{",
"}}}",
"{.}",
"{.:}",
"{..}",
"{..:}",
"{0.}",
"{0.:}",
};
foreach (var format in invalidFormats)
{
Assert.Throws<ParsingErrors>(() => formatter.Test(format, args, "Error"));
}
Assert.Throws<ParsingErrors>(() => formatter.Test(format, args, "Error"));
}

[Test]
Expand Down Expand Up @@ -119,42 +113,191 @@ public void Parser_Ignores_Exceptions()
[Test]
public void Parser_Error_Action_Ignore()
{
var invalidTemplate = "Hello, I'm {Name from {City}";
// | Literal | Erroneous | | Okay |
var invalidTemplate = "Hello, I'm {Name from {City} {Street}";

var smart = Smart.CreateDefaultSmartFormat();
smart.Settings.ParseErrorAction = ErrorAction.Ignore;

var result = smart.Format(invalidTemplate, new { Name = "John", City = "Oklahoma" });

Assert.AreEqual(string.Empty, result);

var parser = GetRegularParser();
parser.Settings.ParseErrorAction = ErrorAction.Ignore;
var parsed = parser.ParseFormat(invalidTemplate, new[] { Guid.NewGuid().ToString("N") });

Assert.That(parsed.Items.Count, Is.EqualTo(4), "Number of parsed items");
Assert.That(parsed.Items[0].RawText, Is.EqualTo("Hello, I'm "), "Literal text");
Assert.That(parsed.Items[1].RawText, Is.EqualTo(string.Empty), "Erroneous placeholder");
Assert.That(parsed.Items[2].RawText, Is.EqualTo(" "));
Assert.That(parsed.Items[3], Is.TypeOf(typeof(Placeholder)));
Assert.That(parsed.Items[3].RawText, Does.Contain("{Street}"), "Correct placeholder");
}

[Test]
public void Parser_Error_Action_MaintainTokens()
{
var invalidTemplate = "Hello, I'm {Name from {City}";

var smart = Smart.CreateDefaultSmartFormat();
smart.Settings.ParseErrorAction = ErrorAction.MaintainTokens;

var result = smart.Format(invalidTemplate, new { Name = "John", City = "Oklahoma" });

Assert.AreEqual("Hello, I'm {Name from {City}", result);
// | Literal | Erroneous | | Okay |
var invalidTemplate = "Hello, I'm {Name from {City} {Street}";

var parser = GetRegularParser();
parser.Settings.ParseErrorAction = ErrorAction.MaintainTokens;
var parsed = parser.ParseFormat(invalidTemplate, new[] { Guid.NewGuid().ToString("N") });

Assert.That(parsed.Items.Count, Is.EqualTo(4), "Number of parsed items");
Assert.That(parsed.Items[0].RawText, Is.EqualTo("Hello, I'm "));
Assert.That(parsed.Items[1].RawText, Is.EqualTo("{Name from {City}"));
Assert.That(parsed.Items[2].RawText, Is.EqualTo(" "));
Assert.That(parsed.Items[3], Is.TypeOf(typeof(Placeholder)));
Assert.That(parsed.Items[3].RawText, Does.Contain("{Street}"));
}

[Test]
public void Parser_Error_Action_OutputErrorInResult()
{
// | Literal | Erroneous |
var invalidTemplate = "Hello, I'm {Name from {City}";

var parser = GetRegularParser();
parser.Settings.ParseErrorAction = ErrorAction.OutputErrorInResult;
var parsed = parser.ParseFormat(invalidTemplate, new[] { Guid.NewGuid().ToString("N") });

var smart = Smart.CreateDefaultSmartFormat();
smart.Settings.ParseErrorAction = ErrorAction.OutputErrorInResult;
Assert.That(parsed.Items.Count, Is.EqualTo(1));
Assert.That(parsed.Items[0].RawText, Does.StartWith("The format string has 3 issues"));
}

var result = smart.Format(invalidTemplate, new { Name = "John", City = "Oklahoma" });
/// <summary>
/// SmartFormat is not designed for processing JavaScript because of interfering usage of {}[].
/// This example shows that even a comment can lead to parsing will work or not.
/// </summary>
[TestCase("/* The comment with this '}{' makes it fail */", "############### {TheVariable} ###############", false)]
[TestCase("", "############### {TheVariable} ###############", true)]
public void Parse_JavaScript_May_Succeed_Or_Fail(string var0, string var1, bool shouldSucceed)
{
var js = @"
(function(exports) {
'use strict';
/**
* Searches for specific element in a given array using
* the interpolation search algorithm.<br><br>
* Time complexity: O(log log N) when elements are uniformly
* distributed, and O(N) in the worst case
*
* @example
*
* var search = require('path-to-algorithms/src/searching/'+
* 'interpolation-search').interpolationSearch;
* console.log(search([1, 2, 3, 4, 5], 4)); // 3
*
* @public
* @module searching/interpolation-search
* @param {Array} sortedArray Input array.
* @param {Number} seekIndex of the element which index should be found.
* @returns {Number} Index of the element or -1 if not found.
*/
function interpolationSearch(sortedArray, seekIndex) {
let leftIndex = 0;
let rightIndex = sortedArray.length - 1;

while (leftIndex <= rightIndex) {
const rangeDiff = sortedArray[rightIndex] - sortedArray[leftIndex];
const indexDiff = rightIndex - leftIndex;
const valueDiff = seekIndex - sortedArray[leftIndex];

if (valueDiff < 0) {
return -1;
}

if (!rangeDiff) {
return sortedArray[leftIndex] === seekIndex ? leftIndex : -1;
}

const middleIndex =
leftIndex + Math.floor((valueDiff * indexDiff) / rangeDiff);

if (sortedArray[middleIndex] === seekIndex) {
return middleIndex;
}

if (sortedArray[middleIndex] < seekIndex) {
leftIndex = middleIndex + 1;
} else {
rightIndex = middleIndex - 1;
}
}
" + var0 + @"
/* " + var1 + @" */
return -1;
}
exports.interpolationSearch = interpolationSearch;
})(typeof window === 'undefined' ? module.exports : window);
";
var parser = GetRegularParser();
parser.Settings.ParseErrorAction = ErrorAction.MaintainTokens;
var parsed = parser.ParseFormat(js, new[] { Guid.NewGuid().ToString("N") });

Assert.IsTrue(
result.StartsWith("The format string has")
);
// No characters should get lost compared to the format string,
// no matter if a Placeholder can be identified or not
Assert.That(parsed.Items.Sum(i => i.RawText.Length), Is.EqualTo(js.Length), "No characters lost");

if (shouldSucceed)
{
Assert.That(parsed.Items.Count(i => i.GetType() == typeof(Placeholder)), Is.EqualTo(1),
"One placeholders");
Assert.That(parsed.Items.First(i => i.GetType() == typeof(Placeholder)).RawText,
Is.EqualTo("{TheVariable}"));
}
else
{
Assert.That(parsed.Items.Count(i => i.GetType() == typeof(Placeholder)), Is.EqualTo(0),
"NO placeholder");
}
}

/// <summary>
/// SmartFormat is not designed for processing CSS because of interfering usage of {}[].
/// This example shows that even a comment can lead to parsing will work or not.
/// </summary>
[TestCase("", "############### {TheVariable} ###############", false)]
[TestCase("/* This '}' in the comment makes it succeed */", "############### {TheVariable} ###############", true)]
public void Parse_Css_May_Succeed_Or_Fail(string var0, string var1, bool shouldSucceed)
{
var css = @"
.media {
display: grid;
grid-template-columns: 1fr 3fr;
}

.media .content {
font-size: .8rem;
}

.comment img {
border: 1px solid grey; " + var0 + @"
anything: '" + var1 + @"'
}

.list-item {
border-bottom: 1px solid grey;
}
";
var parser = GetRegularParser();
parser.Settings.ParseErrorAction = ErrorAction.MaintainTokens;
var parsed = parser.ParseFormat(css, new[] { Guid.NewGuid().ToString("N") });

// No characters should get lost compared to the format string,
// no matter if a Placeholder can be identified or not
Assert.That(parsed.Items.Sum(i => i.RawText.Length), Is.EqualTo(css.Length), "No characters lost");

if (shouldSucceed)
{
Assert.That(parsed.Items.Count(i => i.GetType() == typeof(Placeholder)), Is.EqualTo(1),
"One placeholders");
Assert.That(parsed.Items.First(i => i.GetType() == typeof(Placeholder)).RawText,
Is.EqualTo("{TheVariable}"));
}
else
{
Assert.That(parsed.Items.Count(i => i.GetType() == typeof(Placeholder)), Is.EqualTo(0),
"NO placeholder");
}
}

[Test]
Expand Down
4 changes: 2 additions & 2 deletions src/SmartFormat.Tests/SmartFormat.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<Version>2.6.2.0</Version>
<FileVersion>2.6.2.0</FileVersion>
<AssemblyVersion>2.6.2.0</AssemblyVersion>
<TargetFrameworks>net462;netcoreapp3.1;net5.0</TargetFrameworks>
<TargetFrameworks>net462;net5.0</TargetFrameworks>
<DefineConstants>$(DefineConstants)</DefineConstants>
<GenerateDocumentationFile>false</GenerateDocumentationFile>
<AssemblyName>SmartFormat.Tests</AssemblyName>
Expand Down Expand Up @@ -37,7 +37,7 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\SmartFormat\SmartFormat.csproj" />
<ProjectReference Include="..\SmartFormat\SmartFormat.csproj" />
</ItemGroup>

</Project>
1 change: 1 addition & 0 deletions src/SmartFormat/Core/Parsing/LiteralText.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public override string ToString()
private string ConvertCharacterLiteralsToUnicode()
{
var source = baseString.Substring(startIndex, endIndex - startIndex);
if (source.Length == 0) return source;

// No character literal escaping - nothing to do
if (source[0] != Parser.CharLiteralEscapeChar)
Expand Down
43 changes: 28 additions & 15 deletions src/SmartFormat/Core/Parsing/Parser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,8 @@ public Format ParseFormat(string format, string[] formatterExtensionNames)
// Make sure that this is a nested placeholder before we un-nest it:
if (current.parent == null)
{
// Don't swallow-up redundant closing braces, but treat them as literals
current.Items.Add(new LiteralText(Settings, current, i) {endIndex = i + 1});
parsingErrors.AddIssue(current, parsingErrorText[ParsingError.TooManyClosingBraces], i,
i + 1);
continue;
Expand Down Expand Up @@ -399,7 +401,7 @@ public Format ParseFormat(string format, string[] formatterExtensionNames)
currentPlaceholder.Selectors.Add(new Selector(Settings, format, lastI, i, operatorIndex,
selectorIndex));
else if (operatorIndex != i)
parsingErrors.AddIssue(current, parsingErrorText[ParsingError.TrailingOperatorsInSelector],
parsingErrors.AddIssue(current, $"'0x{Convert.ToByte(c):X}': " + parsingErrorText[ParsingError.TrailingOperatorsInSelector],
operatorIndex, i);
lastI = i + 1;

Expand All @@ -418,7 +420,7 @@ public Format ParseFormat(string format, string[] formatterExtensionNames)
currentPlaceholder.Selectors.Add(new Selector(Settings, format, lastI, i, operatorIndex,
selectorIndex));
else if (operatorIndex != i)
parsingErrors.AddIssue(current, parsingErrorText[ParsingError.TrailingOperatorsInSelector],
parsingErrors.AddIssue(current, $"'0x{Convert.ToByte(c):X}': " + parsingErrorText[ParsingError.TrailingOperatorsInSelector],
operatorIndex, i);
lastI = i + 1;

Expand All @@ -437,7 +439,7 @@ public Format ParseFormat(string format, string[] formatterExtensionNames)
|| _alphanumericSelectors && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z')
|| _allowedSelectorChars.IndexOf(c) != -1;
if (!isValidSelectorChar)
parsingErrors.AddIssue(current, parsingErrorText[ParsingError.InvalidCharactersInSelector],
parsingErrors.AddIssue(current, $"'0x{Convert.ToByte(c):X}': " + parsingErrorText[ParsingError.InvalidCharactersInSelector],
i, i + 1);
}
}
Expand Down Expand Up @@ -533,36 +535,47 @@ internal ParsingErrorText()
}

/// <summary>
/// Handles <see cref="ParsingError"/>s as defined in <see cref="SmartSettings.ParseErrorAction"/>,
/// which leads to results similar to <see cref="SmartSettings.FormatErrorAction"/>s
/// Handles <see cref="ParsingError"/>s as defined in <see cref="SmartSettings.ParseErrorAction"/>.
/// </summary>
/// <param name="parsingErrors"></param>
/// <param name="currentResult"></param>
/// <returns>The <see cref="Format"/> which will be further processed with formatting.</returns>
/// <returns>The <see cref="Format"/> which will be further processed by the formatter.</returns>
private Format HandleParsingErrors(ParsingErrors parsingErrors, Format currentResult)
{
switch (Settings.ParseErrorAction)
{
case ErrorAction.ThrowError:
throw parsingErrors;
case ErrorAction.MaintainTokens:
var fmt = new Format(Settings, currentResult.baseString) {
startIndex = 0,
endIndex = currentResult.baseString.Length
};
fmt.Items.Add(new LiteralText(Settings, fmt));
return fmt;
// Replace erroneous Placeholders with tokens as LiteralText
// Placeholder without issues are left unmodified
for (var i = 0; i < currentResult.Items.Count; i++)
{
if (currentResult.Items[i] is Placeholder ph && parsingErrors.Issues.Any(errItem => errItem.Index >= currentResult.Items[i].startIndex && errItem.Index <= currentResult.Items[i].endIndex))
{
currentResult.Items[i] = new LiteralText(Settings, ph.Format ?? new Format(Settings, ph.baseString), ph.startIndex){endIndex = ph.endIndex};
}
}
return currentResult;
case ErrorAction.Ignore:
return new Format(Settings, string.Empty);
// Replace erroneous Placeholders with an empty LiteralText
for (var i = 0; i < currentResult.Items.Count; i++)
{
if (currentResult.Items[i] is Placeholder ph && parsingErrors.Issues.Any(errItem => errItem.Index >= currentResult.Items[i].startIndex && errItem.Index <= currentResult.Items[i].endIndex))
{
currentResult.Items[i] = new LiteralText(Settings, ph.Format ?? new Format(Settings, ph.baseString), ph.startIndex){endIndex = ph.startIndex};
}
}
return currentResult;
case ErrorAction.OutputErrorInResult:
fmt = new Format(Settings, parsingErrors.Message) {
var fmt = new Format(Settings, parsingErrors.Message) {
startIndex = 0,
endIndex = parsingErrors.Message.Length
};
fmt.Items.Add(new LiteralText(Settings, fmt));
return fmt;
default:
return currentResult;
throw new ArgumentException("Illegal type for ParsingErrors", parsingErrors);
}
}

Expand Down
Loading