Fixing Anki card generation

ccos89 · Mar 27, 2023 · 1bb16e0 · 1bb16e0
1 parent e524cc4
commit 1bb16e0
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 19 deletions.
diff --git a/anki-importer.py b/anki-importer.py
@@ -196,11 +196,11 @@ def send_to_anki_connect(csv_path, note_template, field_mappings):
     successes = [x for x in notes_response if x is not None]
     failures = len(notes) - len(successes)
 
-    print("[+] Emptying any fields left empty intentionally")
-    empty_fields = replace_empty_fields(empty_fields_note_template)["fields"]
-    empty = []
-    for i in tqdm(range(0, len(successes), chunk_size)):
-        empty += set_empty(empty_fields, successes[i : i + chunk_size])
+    # print("[+] Emptying any fields left empty intentionally")
+    # empty_fields = replace_empty_fields(empty_fields_note_template)["fields"]
+    # empty = []
+    # for i in tqdm(range(0, len(successes), chunk_size)):
+    #     empty += set_empty(empty_fields, successes[i : i + chunk_size])
 
     print("[+] Created {} new notes".format(len(successes)))
     if failures:
@@ -249,15 +249,16 @@ def move_individual_media(file_data):
     shutil.move(file_data["src"], file_data["dest"])
 
 
+
 def move_media(srs_export_dir, anki_media_dir):
     print(f"Exporting media from {srs_export_dir}")
     files = grab_files(srs_export_dir, ["*.mp3"], False)
     file_data = []
     for file in files:
         dest = os.path.join(anki_media_dir, os.path.basename(file))
         file_data.append({"src": file, "dest": dest})
-
-    process_map(move_individual_media, files, max_workers=multiprocessing.cpu_count())
+    cpu_count = multiprocessing.cpu_count()
+    process_map(move_individual_media, file_data, max_workers=cpu_count, chunksize=len(file_data) // cpu_count)
 
 
 def get_srs_export_dir(csv_path):

diff --git a/merge.sh b/merge.sh
@@ -5,8 +5,8 @@ FOLDER="${1}"
 echo $FOLDER
 NAME="$(basename "$FOLDER")"
 echo $NAME
-SPLIT_DIR="$FOLDER/$(echo $NAME)_splitted/"
-REL_SPLIT_DIR="./$(echo $NAME)_splitted/"
+SPLIT_DIR="$FOLDER/$(echo $NAME)_merge/"
+REL_SPLIT_DIR="./$(echo $NAME)_merge/"
 ls -1v "$SPLIT_DIR"
 
 
@@ -19,4 +19,5 @@ ls -1v "$SPLIT_DIR"
 # ls -1 "$SPLIT_DIR"*.{mp4,mp3,m4b} > "$SPLIT_DIR/files.txt"
 # $FILES="$(cat $SPLIT_DIR/files.txt)"
 # echo $FILES
-docker run -it --rm -u $(id -u):$(id -g) -v "$FOLDER":/mnt sandreas/m4b-tool:latest merge "$REL_SPLIT_DIR" --output-file="./$NAME.m4b" --jobs $(nproc --all) --album $NAME
+docker run -it --rm -u $(id -u):$(id -g) -v "$FOLDER":/mnt sandreas/m4b-tool:latest merge "$REL_SPLIT_DIR"\
+ --output-file="./$NAME.m4b" --jobs $(nproc --all) --album "$NAME" --name "$NAME" --use-filenames-as-chapters
diff --git a/readme.md b/readme.md
@@ -33,14 +33,14 @@ Currently supports unix based OS's like Ubuntu 20.04 on WSL2.
 Primarily I'm using this for syncing audiobooks to their book script. So while you could use this for video files, I'm not doing that just yet. Note: I'm using the term "splitted" because that's what m4b refers to as the split files.
 
 1. `git clone https://github.com/kanjieater/AudiobookTextSync.git`
-2. Make sure you run any commands that start with `./` from the project root, eg after you clone you can run `cd ./AudiobookTextSync`
+1. Make sure you run any commands that start with `./` from the project root, eg after you clone you can run `cd ./AudiobookTextSync`
 1. Setup the folder. Create a folder to hold a single media file (like an audiobook). Name it whatever you name your media file, eg `Arslan Senki 7`, this is what should go anywhere you see me write `<name>`
-2. Get the book script as text from a digital copy. Put the script at: `./<name>/script.txt`. Everything in this file will show up in your subtitles. So it's important you trim out excess (table of contents, character bios that aren't in the audiobook etc)
-3. If you have less than ~8GB of free RAM you may need _both_ the audiobook as a full m4b (technically other formats would work), AND the split parts. As long as you have one, you can easily get the other. You could technically only use the full single file, but you may run out of ram for longer works. See [Whisper ~13GB Memory Usage Issue for 19hr Audiobook](https://github.com/jianfch/stable-ts/issues/79). By using small splits, we can have more confidence the Speech To Text analysis won't get killed by an Out Of Memory error. As of this tool using stable-ts 2.0.0, this has been improved significantly. I no longer recommend using split files, as it's more likely to introduce a ~50ms delay in the subtitles. It's shouldn't be an issue if you use MPV and can correct the subtitle timings with a hotkey press though.
-4. If you are not using split files you can skip this step. Split files should be `./<name>/<name>_splitted/`.If you have the full audiobook as a m4b, you can split it into chapters using `./split.sh "<full folder path>"`. eg `./split.sh "/mnt/d/Editing/Audiobooks/かがみの孤城/"`. If you don't want to use split files, make sure there isn't a folder with that name existing, or the program will automatically look for the split files.
-5. Single media file should be in `./<name>/<name>.m4b`. If you have the split audiobook as m4b,mp3, or mp4's you can run `./merge.sh "<full folder path>"`,
- eg `./merge.sh "/mnt/d/Editing/Audiobooks/ｍｅｄｉｕｍ霊媒探偵城塚翡翠"`
-6. If you have the `script.txt` and either `./<name>/<name>.m4b` or `./<name>/<name>_splitted/`, you can now run the GPU intense, time intense, and occasionally CPU intense script part. `./run.sh "<full folder path>"` eg `./run.sh "/mnt/d/Editing/Audiobooks/かがみの孤城/"`. This runs each split file individually to get a word level transcript. It then creates a sub format that can be matched to the `script.txt`. Each word level subtitle is merged into a phrase level, and your result should be a `<name>.srt` file that can be watched with `MPV`, showing audio in time with the full book as a subtitle. 
+1. Get the book script as text from a digital copy. Put the script at: `./<name>/script.txt`. Everything in this file will show up in your subtitles. So it's important you trim out excess (table of contents, character bios that aren't in the audiobook etc)
+1. Single media file should be in `./<name>/<name>.m4b`. If you have the split audiobook as m4b,mp3, or mp4's you can run `./merge.sh "<full folder path>"`,
+ eg `./merge.sh "/mnt/d/Editing/Audiobooks/ｍｅｄｉｕｍ霊媒探偵城塚翡翠"`. The split files must be in `./<name>/<name>_merge/`.
+1. If you are not using split files you can skip this step. If you have less than ~4GB of free RAM you may need _both_ the audiobook as a full m4b (technically other formats would work), AND the split parts. However, it is recommended to use only the single file as split  causes a stacking delay per file currently. That is a known bug and can be avoided by using the single combined file. By using small splits, we can have more confidence the Speech To Text analysis won't get killed by an Out Of Memory error. But this introduces a ~50ms delay in the subtitles. It's shouldn't be an issue if you use MPV and can correct the subtitle timings with a hotkey press though.
+Split files should be `./<name>/<name>_splitted/`.If you have the full audiobook as a m4b, you can split it into chapters using `./split.sh "<full folder path>"`. eg `./split.sh "/mnt/d/Editing/Audiobooks/かがみの孤城/"`. If you don't want to use split files, make sure there isn't a folder with `./<name>/<name>_splitted/` existing, or the program will automatically look for the split files.
+6. If you have the `script.txt` and either `./<name>/<name>.m4b` or `./<name>/<name>_splitted/`, you can now run the GPU intense, time intense, and occasionally CPU intense script part. `./run.sh "<full folder path>"` eg `./run.sh "/mnt/d/Editing/Audiobooks/かがみの孤城/"`. This runs each split file individually to get a word level transcript. It then creates a sub format that can be matched to the `script.txt`. Each word level subtitle is merged into a phrase level, and your result should be a `<name>.srt` file that can be watched with `MPV`, showing audio in time with the full book as a subtitle.
 7. From there, use a [texthooker](https://github.com/Renji-XD/texthooker-ui) with something like [mpv_websocket](https://github.com/kuroahna/mpv_websocket) and enjoy Immersion Reading.
 
 # Split m4b by chapter
@@ -124,9 +124,9 @@ You might see various issues while trying this out in the early state. Here are
 5. Split the script
 6. match the script to the generated transcription to get good timestamps
 
-# Getting Book Scripts 
+# Getting Book Scripts
 
-This program supports `txt` files. You may need to use an external program like Calibre to convert your `epub` or kindle formats like `azw3` to a `txt` file. 
+This program supports `txt` files. You may need to use an external program like Calibre to convert your `epub` or kindle formats like `azw3` to a `txt` file.
 
 To convert in Calibre:
 1. Right click on the book and convert the individual book (or use the batch option beneath it)

diff --git a/split_run.py b/split_run.py
@@ -148,6 +148,7 @@ def run():
 
     if path.exists(split_folder) and path.isdir(split_folder):
         working_folder = split_folder
+        print(f"Warning: Using split files causes a fixed delay for every split file. This is a known bug. Use the single file method instead")
     else:
         working_folder = full_folder
     print(f"Working on {working_folder}")