Skip to content

Commit

Permalink
Unify progress data (qarmin#972)
Browse files Browse the repository at this point in the history 
* Unify progress data code

* Do not increment in threads values every time when finding file/folder(works quite good, may works bad with folders with ~several thousands of files)

* Partial changes

* Simplify creating tree_view

* Allow setting thread number in CLI

* Simplified code and add tests with help of copilot
  • Loading branch information
@qarmin
qarmin authored May 10, 2023
1 parent 78d00ee commit 582e541
Show file tree
Hide file tree
Showing 21 changed files with 691 additions and 1,179 deletions.
28 changes: 28 additions & 0 deletions czkawka_cli/src/commands.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ pub enum Commands {

#[derive(Debug, clap::Args)]
pub struct DuplicatesArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand Down Expand Up @@ -163,6 +165,8 @@ pub struct DuplicatesArgs {

#[derive(Debug, clap::Args)]
pub struct EmptyFoldersArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand All @@ -180,6 +184,8 @@ pub struct EmptyFoldersArgs {

#[derive(Debug, clap::Args)]
pub struct BiggestFilesArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand All @@ -205,6 +211,8 @@ pub struct BiggestFilesArgs {

#[derive(Debug, clap::Args)]
pub struct EmptyFilesArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand All @@ -226,6 +234,8 @@ pub struct EmptyFilesArgs {

#[derive(Debug, clap::Args)]
pub struct TemporaryArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand All @@ -245,6 +255,8 @@ pub struct TemporaryArgs {

#[derive(Debug, clap::Args)]
pub struct SimilarImagesArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand Down Expand Up @@ -313,6 +325,8 @@ pub struct SimilarImagesArgs {

#[derive(Debug, clap::Args)]
pub struct SameMusicArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand Down Expand Up @@ -359,6 +373,8 @@ pub struct SameMusicArgs {

#[derive(Debug, clap::Args)]
pub struct InvalidSymlinksArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand All @@ -380,6 +396,8 @@ pub struct InvalidSymlinksArgs {

#[derive(Debug, clap::Args)]
pub struct BrokenFilesArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand All @@ -401,6 +419,8 @@ pub struct BrokenFilesArgs {

#[derive(Debug, clap::Args)]
pub struct SimilarVideosArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand Down Expand Up @@ -449,6 +469,8 @@ pub struct SimilarVideosArgs {

#[derive(Debug, clap::Args)]
pub struct BadExtensionsArgs {
#[clap(flatten)]
pub thread_number: ThreadNumber,
#[clap(flatten)]
pub directories: Directories,
#[clap(flatten)]
Expand Down Expand Up @@ -517,6 +539,12 @@ pub struct NotRecursive {
pub not_recursive: bool,
}

#[derive(Debug, clap::Args)]
pub struct ThreadNumber {
#[clap(short = 'T', long, default_value = "0", help = "Limits thread number, 0(default) will use all available threads")]
pub thread_number: usize,
}

#[cfg(target_family = "unix")]
#[derive(Debug, clap::Args)]
pub struct ExcludeOtherFilesystems {
Expand Down
37 changes: 34 additions & 3 deletions czkawka_cli/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use clap::Parser;

use commands::Commands;
use czkawka_core::big_file::SearchMode;
use czkawka_core::common::{get_number_of_threads, set_default_number_of_threads};
use czkawka_core::common::set_number_of_threads;
#[allow(unused_imports)] // It is used in release for print_results().
use czkawka_core::common_traits::*;
use czkawka_core::similar_images::test_image_conversion_speed;
Expand Down Expand Up @@ -34,8 +34,6 @@ mod commands;
fn main() {
let command = Args::parse().command;

set_default_number_of_threads();
println!("Set thread number to {}", get_number_of_threads());
#[cfg(debug_assertions)]
println!("{command:?}");

Expand All @@ -59,6 +57,7 @@ fn main() {

fn duplicates(duplicates: DuplicatesArgs) {
let DuplicatesArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -78,6 +77,8 @@ fn duplicates(duplicates: DuplicatesArgs) {
case_sensitive_name_comparison,
} = duplicates;

set_number_of_threads(thread_number.thread_number);

let mut df = DuplicateFinder::new();

df.set_included_directory(directories.directories);
Expand Down Expand Up @@ -113,6 +114,7 @@ fn duplicates(duplicates: DuplicatesArgs) {

fn empty_folders(empty_folders: EmptyFoldersArgs) {
let EmptyFoldersArgs {
thread_number,
directories,
delete_folders,
file_to_save,
Expand All @@ -122,6 +124,8 @@ fn empty_folders(empty_folders: EmptyFoldersArgs) {
exclude_other_filesystems,
} = empty_folders;

set_number_of_threads(thread_number.thread_number);

let mut ef = EmptyFolder::new();

ef.set_included_directory(directories.directories);
Expand All @@ -147,6 +151,7 @@ fn empty_folders(empty_folders: EmptyFoldersArgs) {

fn biggest_files(biggest_files: BiggestFilesArgs) {
let BiggestFilesArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -160,6 +165,8 @@ fn biggest_files(biggest_files: BiggestFilesArgs) {
smallest_mode,
} = biggest_files;

set_number_of_threads(thread_number.thread_number);

let mut bf = BigFile::new();

bf.set_included_directory(directories.directories);
Expand Down Expand Up @@ -193,6 +200,7 @@ fn biggest_files(biggest_files: BiggestFilesArgs) {

fn empty_files(empty_files: EmptyFilesArgs) {
let EmptyFilesArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -204,6 +212,8 @@ fn empty_files(empty_files: EmptyFilesArgs) {
exclude_other_filesystems,
} = empty_files;

set_number_of_threads(thread_number.thread_number);

let mut ef = EmptyFiles::new();

ef.set_included_directory(directories.directories);
Expand Down Expand Up @@ -234,6 +244,7 @@ fn empty_files(empty_files: EmptyFilesArgs) {

fn temporary(temporary: TemporaryArgs) {
let TemporaryArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -244,6 +255,8 @@ fn temporary(temporary: TemporaryArgs) {
not_recursive,
} = temporary;

set_number_of_threads(thread_number.thread_number);

let mut tf = Temporary::new();

tf.set_included_directory(directories.directories);
Expand Down Expand Up @@ -273,6 +286,7 @@ fn temporary(temporary: TemporaryArgs) {

fn similar_images(similar_images: SimilarImagesArgs) {
let SimilarImagesArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -288,6 +302,8 @@ fn similar_images(similar_images: SimilarImagesArgs) {
hash_size,
} = similar_images;

set_number_of_threads(thread_number.thread_number);

let mut sf = SimilarImages::new();

sf.set_included_directory(directories.directories);
Expand Down Expand Up @@ -320,6 +336,7 @@ fn similar_images(similar_images: SimilarImagesArgs) {

fn same_music(same_music: SameMusicArgs) {
let SameMusicArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -333,6 +350,8 @@ fn same_music(same_music: SameMusicArgs) {
music_similarity,
} = same_music;

set_number_of_threads(thread_number.thread_number);

let mut mf = SameMusic::new();

mf.set_included_directory(directories.directories);
Expand Down Expand Up @@ -365,6 +384,7 @@ fn same_music(same_music: SameMusicArgs) {

fn invalid_symlinks(invalid_symlinks: InvalidSymlinksArgs) {
let InvalidSymlinksArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -376,6 +396,8 @@ fn invalid_symlinks(invalid_symlinks: InvalidSymlinksArgs) {
delete_files,
} = invalid_symlinks;

set_number_of_threads(thread_number.thread_number);

let mut ifs = InvalidSymlinks::new();

ifs.set_included_directory(directories.directories);
Expand Down Expand Up @@ -405,6 +427,7 @@ fn invalid_symlinks(invalid_symlinks: InvalidSymlinksArgs) {

fn broken_files(broken_files: BrokenFilesArgs) {
let BrokenFilesArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -416,6 +439,8 @@ fn broken_files(broken_files: BrokenFilesArgs) {
exclude_other_filesystems,
} = broken_files;

set_number_of_threads(thread_number.thread_number);

let mut br = BrokenFiles::new();

br.set_included_directory(directories.directories);
Expand Down Expand Up @@ -446,6 +471,7 @@ fn broken_files(broken_files: BrokenFilesArgs) {

fn similar_videos(similar_videos: SimilarVideosArgs) {
let SimilarVideosArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -459,6 +485,8 @@ fn similar_videos(similar_videos: SimilarVideosArgs) {
allowed_extensions,
} = similar_videos;

set_number_of_threads(thread_number.thread_number);

let mut vr = SimilarVideos::new();

vr.set_included_directory(directories.directories);
Expand Down Expand Up @@ -488,6 +516,7 @@ fn similar_videos(similar_videos: SimilarVideosArgs) {

fn bad_extensions(bad_extensions: BadExtensionsArgs) {
let BadExtensionsArgs {
thread_number,
directories,
excluded_directories,
excluded_items,
Expand All @@ -498,6 +527,8 @@ fn bad_extensions(bad_extensions: BadExtensionsArgs) {
allowed_extensions,
} = bad_extensions;

set_number_of_threads(thread_number.thread_number);

let mut be = BadExtensions::new();

be.set_included_directory(directories.directories);
Expand Down
7 changes: 5 additions & 2 deletions czkawka_core/src/bad_extensions.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use mime_guess::get_mime_extensions;
use rayon::prelude::*;

use crate::common::{prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData};
use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
Expand Down Expand Up @@ -124,6 +124,7 @@ const WORKAROUNDS: &[(&str, &str)] = &[
("xml", "vbox"), // VirtualBox
("xml", "vbox-prev"), // VirtualBox
("xml", "vcproj"), // VisualStudio
("xml", "vcxproj"), // VisualStudio
("xml", "xba"), // Libreoffice
("xml", "xcd"), // Libreoffice files
("zip", "apk"), // Android apk
Expand Down Expand Up @@ -172,6 +173,7 @@ impl Info {
}

pub struct BadExtensions {
tool_type: ToolType,
text_messages: Messages,
information: Info,
files_to_check: Vec<FileEntry>,
Expand All @@ -191,6 +193,7 @@ impl BadExtensions {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::BadExtensions,
text_messages: Messages::new(),
information: Info::new(),
recursive_search: true,
Expand Down Expand Up @@ -314,7 +317,7 @@ impl BadExtensions {

fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&UnboundedSender<ProgressData>>) -> bool {
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None);
prepare_thread_handler_common(progress_sender, 1, 1, self.files_to_check.len(), CheckingMethod::None, self.tool_type);

let files_to_check = mem::take(&mut self.files_to_check);

Expand Down
7 changes: 5 additions & 2 deletions czkawka_core/src/big_file.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use humansize::BINARY;
use rayon::prelude::*;

use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData};
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
use crate::common_directory::Directories;
use crate::common_extensions::Extensions;
use crate::common_items::ExcludedItems;
Expand Down Expand Up @@ -55,6 +55,7 @@ impl Info {

/// Struct with required information's to work
pub struct BigFile {
tool_type: ToolType,
text_messages: Messages,
information: Info,
big_files: Vec<(u64, FileEntry)>,
Expand All @@ -72,6 +73,7 @@ impl BigFile {
#[must_use]
pub fn new() -> Self {
Self {
tool_type: ToolType::BigFile,
text_messages: Default::default(),
information: Info::new(),
big_files: Default::default(),
Expand Down Expand Up @@ -148,7 +150,8 @@ impl BigFile {
folders_to_check.push(id.clone());
}

let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) = prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None);
let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
prepare_thread_handler_common(progress_sender, 0, 0, 0, CheckingMethod::None, self.tool_type);

while !folders_to_check.is_empty() {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
Expand Down
Loading

0 comments on commit 582e541

Please sign in to comment.