Skip to content

Commit

Permalink
Lots of Bugfixes (#3308)
Browse files Browse the repository at this point in the history
  • Loading branch information
@majora2007
majora2007 committed Oct 25, 2024
1 parent edbd44b commit 1a88dd4
Show file tree
Hide file tree
Showing 36 changed files with 331 additions and 588 deletions.
138 changes: 95 additions & 43 deletions API.Tests/Services/ScannerServiceTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Serialization;
using API.Data;
using API.Data.Metadata;
using API.Data.Repositories;
using API.Entities;
using API.Entities.Enums;
Expand Down Expand Up @@ -35,6 +38,7 @@ public class ScannerServiceTests : AbstractDbTest
private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests");
private readonly string _testcasesDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/TestCases");
private readonly string _imagePath = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/1x1.png");
private static readonly string[] ComicInfoExtensions = new[] { ".cbz", ".cbr", ".zip", ".rar" };

public ScannerServiceTests(ITestOutputHelper testOutputHelper)
{
Expand Down Expand Up @@ -125,9 +129,61 @@ public async Task ScanLibrary_FlatSeriesWithSpecial()
Assert.NotNull(postLib.Series.First().Volumes.FirstOrDefault(v => v.Chapters.FirstOrDefault(c => c.IsSpecial) != null));
}

private async Task<Library> GenerateScannerData(string testcase)
/// <summary>
/// This is testing that if the first file is named A and has a localized name of B if all other files are named B, it should still group and name the series A
/// </summary>
[Fact]
public async Task ScanLibrary_LocalizedSeries()
{
const string testcase = "Series with Localized - Manga.json";

// Get the first file and generate a ComicInfo
var infos = new Dictionary<string, ComicInfo>();
infos.Add("My Dress-Up Darling v01.cbz", new ComicInfo()
{
Series = "My Dress-Up Darling",
LocalizedSeries = "Sono Bisque Doll wa Koi wo Suru"
});

var library = await GenerateScannerData(testcase, infos);


var scanner = CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);

Assert.NotNull(postLib);
Assert.Single(postLib.Series);
Assert.Equal(3, postLib.Series.First().Volumes.Count);
}


/// <summary>
/// Files under a folder with a SP marker should group into one issue
/// </summary>
/// <remarks>https://github.com/Kareadita/Kavita/issues/3299</remarks>
[Fact]
public async Task ScanLibrary_ImageSeries_SpecialGrouping()
{
const string testcase = "Image Series with SP Folder - Manga.json";

var library = await GenerateScannerData(testcase);


var scanner = CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);

Assert.NotNull(postLib);
Assert.Single(postLib.Series);
Assert.Equal(3, postLib.Series.First().Volumes.Count);
}


#region Setup
private async Task<Library> GenerateScannerData(string testcase, Dictionary<string, ComicInfo> comicInfos = null)
{
var testDirectoryPath = await GenerateTestDirectory(Path.Join(_testcasesDirectory, testcase));
var testDirectoryPath = await GenerateTestDirectory(Path.Join(_testcasesDirectory, testcase), comicInfos);

var (publisher, type) = SplitPublisherAndLibraryType(Path.GetFileNameWithoutExtension(testcase));

Expand All @@ -148,11 +204,17 @@ private async Task<Library> GenerateScannerData(string testcase)

private ScannerService CreateServices()
{
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), new FileSystem());
var mockReadingService = new MockReadingItemService(ds, Substitute.For<IBookService>());
var fs = new FileSystem();
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
var archiveService = new ArchiveService(Substitute.For<ILogger<ArchiveService>>(), ds,
Substitute.For<IImageService>(), Substitute.For<IMediaErrorService>());
var readingItemService = new ReadingItemService(archiveService, Substitute.For<IBookService>(),
Substitute.For<IImageService>(), ds, Substitute.For<ILogger<ReadingItemService>>());


var processSeries = new ProcessSeries(_unitOfWork, Substitute.For<ILogger<ProcessSeries>>(),
Substitute.For<IEventHub>(),
ds, Substitute.For<ICacheHelper>(), mockReadingService, Substitute.For<IFileService>(),
ds, Substitute.For<ICacheHelper>(), readingItemService, new FileService(fs),
Substitute.For<IMetadataService>(),
Substitute.For<IWordCountAnalyzerService>(),
Substitute.For<IReadingListService>(),
Expand All @@ -161,7 +223,7 @@ private ScannerService CreateServices()
var scanner = new ScannerService(_unitOfWork, Substitute.For<ILogger<ScannerService>>(),
Substitute.For<IMetadataService>(),
Substitute.For<ICacheService>(), Substitute.For<IEventHub>(), ds,
mockReadingService, processSeries, Substitute.For<IWordCountAnalyzerService>());
readingItemService, processSeries, Substitute.For<IWordCountAnalyzerService>());
return scanner;
}

Expand Down Expand Up @@ -189,7 +251,7 @@ private static (string Publisher, LibraryType Type) SplitPublisherAndLibraryType



private async Task<string> GenerateTestDirectory(string mapPath)
private async Task<string> GenerateTestDirectory(string mapPath, Dictionary<string, ComicInfo> comicInfos = null)
{
// Read the map file
var mapContent = await File.ReadAllTextAsync(mapPath);
Expand All @@ -206,15 +268,15 @@ private async Task<string> GenerateTestDirectory(string mapPath)
Directory.CreateDirectory(testDirectory);

// Generate the files and folders
await Scaffold(testDirectory, filePaths);
await Scaffold(testDirectory, filePaths, comicInfos);

_testOutputHelper.WriteLine($"Test Directory Path: {testDirectory}");

return testDirectory;
}


private async Task Scaffold(string testDirectory, List<string> filePaths)
private async Task Scaffold(string testDirectory, List<string> filePaths, Dictionary<string, ComicInfo> comicInfos = null)
{
foreach (var relativePath in filePaths)
{
Expand All @@ -229,9 +291,9 @@ private async Task Scaffold(string testDirectory, List<string> filePaths)
}

var ext = Path.GetExtension(fullPath).ToLower();
if (new[] { ".cbz", ".cbr", ".zip", ".rar" }.Contains(ext))
if (ComicInfoExtensions.Contains(ext) && comicInfos != null && comicInfos.TryGetValue(Path.GetFileName(relativePath), out var info))
{
CreateMinimalCbz(fullPath, includeMetadata: true);
CreateMinimalCbz(fullPath, info);
}
else
{
Expand All @@ -242,54 +304,44 @@ private async Task Scaffold(string testDirectory, List<string> filePaths)
}
}

private void CreateMinimalCbz(string filePath, bool includeMetadata)
private void CreateMinimalCbz(string filePath, ComicInfo? comicInfo = null)
{
var tempImagePath = _imagePath; // Assuming _imagePath is a valid path to the 1x1 image

using (var archive = ZipFile.Open(filePath, ZipArchiveMode.Create))
{
// Add the 1x1 image to the archive
archive.CreateEntryFromFile(tempImagePath, "1x1.png");
archive.CreateEntryFromFile(_imagePath, "1x1.png");

if (includeMetadata)
if (comicInfo != null)
{
var comicInfo = GenerateComicInfo();
// Serialize ComicInfo object to XML
var comicInfoXml = SerializeComicInfoToXml(comicInfo);

// Create an entry for ComicInfo.xml in the archive
var entry = archive.CreateEntry("ComicInfo.xml");
using var entryStream = entry.Open();
using var writer = new StreamWriter(entryStream, Encoding.UTF8);
writer.Write(comicInfo);

// Write the XML to the archive
writer.Write(comicInfoXml);
}

}
Console.WriteLine($"Created minimal CBZ archive: {filePath} with{(includeMetadata ? "" : "out")} metadata.");
Console.WriteLine($"Created minimal CBZ archive: {filePath} with{(comicInfo != null ? "" : "out")} metadata.");
}

private string GenerateComicInfo()
{
var comicInfo = new StringBuilder();
comicInfo.AppendLine("<?xml version='1.0' encoding='utf-8'?>");
comicInfo.AppendLine("<ComicInfo>");

// People Tags
string[] people = { "Joe Shmo", "Tommy Two Hands"};
string[] genres = { /* Your list of genres here */ };

void AddRandomTag(string tagName, string[] choices)
{
if (new Random().Next(0, 2) == 1) // 50% chance to include the tag
{
var selected = choices.OrderBy(x => Guid.NewGuid()).Take(new Random().Next(1, 5)).ToArray();
comicInfo.AppendLine($" <{tagName}>{string.Join(", ", selected)}</{tagName}>");
}
}

foreach (var tag in new[] { "Writer", "Penciller", "Inker", "CoverArtist", "Publisher", "Character", "Imprint", "Colorist", "Letterer", "Editor", "Translator", "Team", "Location" })
private static string SerializeComicInfoToXml(ComicInfo comicInfo)
{
var xmlSerializer = new XmlSerializer(typeof(ComicInfo));
using var stringWriter = new StringWriter();
using (var xmlWriter = XmlWriter.Create(stringWriter, new XmlWriterSettings { Indent = true, Encoding = new UTF8Encoding(false), OmitXmlDeclaration = false}))
{
AddRandomTag(tag, people);
xmlSerializer.Serialize(xmlWriter, comicInfo);
}

AddRandomTag("Genre", genres);
comicInfo.AppendLine("</ComicInfo>");

return comicInfo.ToString();
// For the love of god, I spent 2 hours trying to get utf-8 with no BOM
return stringWriter.ToString().Replace("""<?xml version="1.0" encoding="utf-16"?>""",
@"<?xml version='1.0' encoding='utf-8'?>");
}
#endregion
}
6 changes: 6 additions & 0 deletions ...ests/Services/Test Data/ScannerService/TestCases/Image Series with SP Folder - Manga.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[
"My Dress-Up Darling/My Dress-Up Darling vol 1/0001.png",
"My Dress-Up Darling/My Dress-Up Darling vol 1/0002.png",
"My Dress-Up Darling/My Dress-Up Darling vol 2/0001.png",
"My Dress-Up Darling/Specials/My Dress-Up Darling SP01/0001.png"
]
5 changes: 5 additions & 0 deletions API.Tests/Services/Test Data/ScannerService/TestCases/Series with Localized - Manga.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[
"My Dress-Up Darling/My Dress-Up Darling v01.cbz",
"My Dress-Up Darling/Sono Bisque Doll wa Koi wo Suru v02.cbz",
"My Dress-Up Darling/Sono Bisque Doll wa Koi wo Suru ch 10.cbz"
]
5 changes: 3 additions & 2 deletions API/Controllers/PersonController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using Nager.ArticleNumber;

namespace API.Controllers;
#nullable enable

public class PersonController : BaseApiController
{
Expand Down Expand Up @@ -39,11 +40,11 @@ public async Task<ActionResult<IEnumerable<PersonRole>>> GetRolesForPersonByName
}

/// <summary>
/// Returns a list of authors for browsing
/// Returns a list of authors & artists for browsing

Check warning on line 43 in API/Controllers/PersonController.cs

View workflow job for this annotation

GitHub Actions / Build Canary Docker 

XML comment has badly formed XML -- 'Whitespace is not allowed at this location.'
/// </summary>
/// <param name="userParams"></param>
/// <returns></returns>
[HttpPost("authors")]
[HttpPost("all")]
public async Task<ActionResult<PagedList<BrowsePersonDto>>> GetAuthorsForBrowse([FromQuery] UserParams? userParams)
{
userParams ??= UserParams.Default;
Expand Down
1 change: 1 addition & 0 deletions API/Data/Repositories/PersonRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ public async Task<PagedList<BrowsePersonDto>> GetAllWritersAndSeriesCount(int us
Id = p.Id,
Name = p.Name,
Description = p.Description,
CoverImage = p.CoverImage,
SeriesCount = p.SeriesMetadataPeople
.Where(smp => roles.Contains(smp.Role))
.Select(smp => smp.SeriesMetadata.SeriesId)
Expand Down
6 changes: 3 additions & 3 deletions API/Extensions/QueryExtensions/Filtering/SeriesFilter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ public static IQueryable<Series> HasReadingDate(this IQueryable<Series> queryabl
public static IQueryable<Series> HasTags(this IQueryable<Series> queryable, bool condition,
FilterComparison comparison, IList<int> tags)
{
if (!condition || tags.Count == 0) return queryable;
if (!condition || (comparison != FilterComparison.IsEmpty && tags.Count == 0)) return queryable;

switch (comparison)
{
Expand Down Expand Up @@ -547,7 +547,7 @@ public static IQueryable<Series> HasPeopleLegacy(this IQueryable<Series> queryab
public static IQueryable<Series> HasGenre(this IQueryable<Series> queryable, bool condition,
FilterComparison comparison, IList<int> genres)
{
if (!condition || genres.Count == 0) return queryable;
if (!condition || (comparison != FilterComparison.IsEmpty && genres.Count == 0)) return queryable;

switch (comparison)
{
Expand Down Expand Up @@ -620,7 +620,7 @@ public static IQueryable<Series> HasFormat(this IQueryable<Series> queryable, bo
public static IQueryable<Series> HasCollectionTags(this IQueryable<Series> queryable, bool condition,
FilterComparison comparison, IList<int> collectionTags, IList<int> collectionSeries)
{
if (!condition || collectionTags.Count == 0) return queryable;
if (!condition || (comparison != FilterComparison.IsEmpty && collectionTags.Count == 0)) return queryable;


switch (comparison)
Expand Down
3 changes: 1 addition & 2 deletions API/Services/ReadingItemService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ namespace API.Services;

public interface IReadingItemService
{
ComicInfo? GetComicInfo(string filePath);
int GetNumberOfPages(string filePath, MangaFormat format);
string GetCoverImage(string filePath, string fileName, MangaFormat format, EncodeFormat encodeFormat, CoverImageSize size = CoverImageSize.Default);
void Extract(string fileFilePath, string targetDirectory, MangaFormat format, int imageCount = 1);
Expand Down Expand Up @@ -51,7 +50,7 @@ public ReadingItemService(IArchiveService archiveService, IBookService bookServi
/// </summary>
/// <param name="filePath">Fully qualified path of file</param>
/// <returns></returns>
public ComicInfo? GetComicInfo(string filePath)
private ComicInfo? GetComicInfo(string filePath)
{
if (Parser.IsEpub(filePath))
{
Expand Down
16 changes: 7 additions & 9 deletions API/Services/Tasks/Scanner/ProcessSeries.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ await _unitOfWork.SeriesRepository.GetFullSeriesByAnyName(firstInfo.Series, firs

try
{
_logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName);
_logger.LogInformation("[ScannerService] Processing series {SeriesName} with {Count} files", series.OriginalName, parsedInfos.Count);

// parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort)
var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, firstInfo);
Expand Down Expand Up @@ -423,7 +423,7 @@ private async Task UpdateCollectionTags(Series series, Chapter firstChapter)
var defaultAdmin = await _unitOfWork.UserRepository.GetDefaultAdminUser(AppUserIncludes.Collections);
if (defaultAdmin == null) return;

_logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
_logger.LogInformation("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
var sw = Stopwatch.StartNew();

foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
Expand Down Expand Up @@ -593,7 +593,6 @@ private async Task UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, b
{
// Add new volumes and update chapters per volume
var distinctVolumes = parsedInfos.DistinctVolumes();
_logger.LogTrace("[ScannerService] Updating {DistinctVolumes} volumes on {SeriesName}", distinctVolumes.Count, series.Name);
foreach (var volumeNumber in distinctVolumes)
{
Volume? volume;
Expand Down Expand Up @@ -621,7 +620,6 @@ private async Task UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, b
volume.LookupName = volumeNumber;
volume.Name = volume.GetNumberTitle();

_logger.LogTrace("[ScannerService] Parsing {SeriesName} - Volume {VolumeNumber}", series.Name, volume.Name);
var infos = parsedInfos.Where(p => p.Volumes == volumeNumber).ToArray();

await UpdateChapters(series, volume, infos, forceUpdate);
Expand All @@ -641,7 +639,7 @@ private void RemoveVolumes(Series series, IList<ParserInfo> parsedInfos)
if (series.Volumes.Count == nonDeletedVolumes.Count) return;


_logger.LogTrace("[ScannerService] Removed {Count} volumes from {SeriesName} where parsed infos were not mapping with volume name",
_logger.LogDebug("[ScannerService] Removed {Count} volumes from {SeriesName} where parsed infos were not mapping with volume name",
(series.Volumes.Count - nonDeletedVolumes.Count), series.Name);
var deletedVolumes = series.Volumes.Except(nonDeletedVolumes);
foreach (var volume in deletedVolumes)
Expand All @@ -655,7 +653,7 @@ private void RemoveVolumes(Series series, IList<ParserInfo> parsedInfos)
file);
}

_logger.LogTrace("[ScannerService] Removed {SeriesName} - Volume {Volume}: {File}", series.Name, volume.Name, file);
_logger.LogDebug("[ScannerService] Removed {SeriesName} - Volume {Volume}: {File}", series.Name, volume.Name, file);
}

series.Volumes = nonDeletedVolumes;
Expand All @@ -681,7 +679,7 @@ private async Task UpdateChapters(Series series, Volume volume, IList<ParserInfo

if (chapter == null)
{
_logger.LogTrace(
_logger.LogDebug(
"[ScannerService] Adding new chapter, {Series} - Vol {Volume} Ch {Chapter}", info.Series, info.Volumes, info.Chapters);
chapter = ChapterBuilder.FromParserInfo(info).Build();
volume.Chapters.Add(chapter);
Expand Down Expand Up @@ -778,7 +776,7 @@ private void RemoveChapters(Volume volume, IList<ParserInfo> parsedInfos)
// If no files remain after filtering, remove the chapter
if (existingChapter.Files.Count != 0) continue;

_logger.LogTrace("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName}",
_logger.LogDebug("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName}",
existingChapter.Range, volume.Name, parsedInfos[0].Series);
volume.Chapters.Remove(existingChapter);
}
Expand All @@ -789,7 +787,7 @@ private void RemoveChapters(Volume volume, IList<ParserInfo> parsedInfos)

// If no files exist, remove the chapter
if (filesExist) continue;
_logger.LogTrace("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName} as no files exist",
_logger.LogDebug("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName} as no files exist",
existingChapter.Range, volume.Name, parsedInfos[0].Series);
volume.Chapters.Remove(existingChapter);
}
Expand Down
Loading

0 comments on commit 1a88dd4

Please sign in to comment.