Added reap function and memory control hooks. Added max-score/max-mem…

… limits
smarco · Jun 21, 2021 · 65eb506 · 65eb506
1 parent 2a0d373
commit 65eb506
Show file tree

Hide file tree

Showing 22 changed files with 328 additions and 156 deletions.
diff --git a/Makefile b/Makefile
@@ -42,6 +42,12 @@ debug: setup
 debug: MODE=all
 debug: $(SUBDIRS) lib_wfa tools
 
+# ASAN: ASAN_OPTIONS=detect_leaks=1:symbolize=1 LSAN_OPTIONS=verbosity=2:log_threads=1
+asan: CC_FLAGS+=-fsanitize=address -fno-omit-frame-pointer -fno-common
+asan: MODE=all
+asan: setup
+asan: $(SUBDIRS) lib_wfa tools
+
 setup:
 	@mkdir -p $(FOLDER_BIN) $(FOLDER_BUILD)
 

diff --git a/alignment/cigar.c b/alignment/cigar.c
@@ -53,6 +53,20 @@ void cigar_clear(
   cigar->end_offset = cigar->max_operations;
   cigar->score = INT32_MIN;
 }
+void cigar_resize(
+    cigar_t* const cigar,
+    const int max_operations) {
+  // Check maximum operations
+  if (max_operations > cigar->max_operations) {
+    cigar->max_operations = max_operations;
+    mm_allocator_free(cigar->mm_allocator,cigar->operations); // Free
+    cigar->operations = mm_allocator_malloc(
+        cigar->mm_allocator,max_operations); // Allocate
+  }
+  cigar->begin_offset = cigar->max_operations - 1;
+  cigar->end_offset = cigar->max_operations;
+  cigar->score = INT32_MIN;
+}
 void cigar_free(
     cigar_t* const cigar) {
   mm_allocator_free(cigar->mm_allocator,cigar->operations);

diff --git a/alignment/cigar.h b/alignment/cigar.h
@@ -72,6 +72,9 @@ void cigar_allocate(
     mm_allocator_t* const mm_allocator);
 void cigar_clear(
     cigar_t* const cigar);
+void cigar_resize(
+    cigar_t* const cigar,
+    const int max_operations);
 void cigar_free(
     cigar_t* const cigar);
 

diff --git a/system/mm_allocator.c b/system/mm_allocator.c
@@ -110,7 +110,7 @@ typedef struct {
 mm_allocator_segment_t* mm_allocator_segment_new(
     mm_allocator_t* const mm_allocator) {
   // Allocate handler
-  mm_allocator_segment_t* const segment = malloc(sizeof(mm_allocator_segment_t));
+  mm_allocator_segment_t* const segment = (mm_allocator_segment_t*) malloc(sizeof(mm_allocator_segment_t));
   // Index
   const uint64_t segment_idx = vector_get_used(mm_allocator->segments);
   segment->idx = segment_idx;
@@ -151,7 +151,7 @@ uint64_t mm_allocator_segment_get_num_requests(
 mm_allocator_t* mm_allocator_new(
     const uint64_t segment_size) {
   // Allocate handler
-  mm_allocator_t* const mm_allocator = malloc(sizeof(mm_allocator_t));
+  mm_allocator_t* const mm_allocator = (mm_allocator_t*) malloc(sizeof(mm_allocator_t));
   mm_allocator->request_ticker = 0;
   // Segments
   mm_allocator->segment_size = segment_size;
@@ -296,7 +296,7 @@ void* mm_allocator_allocate(
       memory_aligned = memory_aligned - ((uintptr_t)memory_aligned % align_bytes);
     }
     // Set mm_reference
-    mm_allocator_reference_t* const mm_reference = memory_aligned - sizeof(mm_allocator_reference_t);
+    mm_allocator_reference_t* const mm_reference = (mm_allocator_reference_t*)(memory_aligned - sizeof(mm_allocator_reference_t));
     mm_reference->segment_idx = segment->idx;
     mm_reference->request_idx = mm_allocator_segment_get_num_requests(segment);
     // Add request
@@ -323,7 +323,7 @@ void* mm_allocator_allocate(
       memory_aligned = memory_aligned - ((uintptr_t)memory_aligned % align_bytes);
     }
     // Set reference
-    mm_allocator_reference_t* const mm_reference = memory_aligned - sizeof(mm_allocator_reference_t);
+    mm_allocator_reference_t* const mm_reference = (mm_allocator_reference_t*)(memory_aligned - sizeof(mm_allocator_reference_t));
     mm_reference->segment_idx = UINT32_MAX;
     mm_reference->request_idx = vector_get_used(mm_allocator->malloc_requests);
     // Add malloc-request
@@ -424,7 +424,7 @@ void mm_allocator_free(
 #else
   // Get reference
   void* const effective_memory = memory - sizeof(mm_allocator_reference_t);
-  mm_allocator_reference_t* const mm_reference = effective_memory;
+  mm_allocator_reference_t* const mm_reference = (mm_allocator_reference_t*) effective_memory;
   if (mm_reference->segment_idx == UINT32_MAX) {
     // Free malloc memory
     mm_allocator_free_malloc_request(mm_allocator,mm_reference);

diff --git a/system/mm_stack.c b/system/mm_stack.c
@@ -70,7 +70,7 @@ typedef struct {
 mm_stack_segment_t* mm_stack_segment_new(
     mm_stack_t* const mm_stack) {
   // Allocate handler
-  mm_stack_segment_t* const segment = malloc(sizeof(mm_stack_segment_t));
+  mm_stack_segment_t* const segment = (mm_stack_segment_t*) malloc(sizeof(mm_stack_segment_t));
   // Memory
   segment->size = mm_stack->segment_size;
   segment->memory = malloc(mm_stack->segment_size);
@@ -95,7 +95,7 @@ void mm_stack_segment_delete(
 mm_stack_t* mm_stack_new(
     const uint64_t segment_size) {
   // Allocate handler
-  mm_stack_t* const mm_stack = malloc(sizeof(mm_stack_t));
+  mm_stack_t* const mm_stack = (mm_stack_t*) malloc(sizeof(mm_stack_t));
   // Memory segments
   mm_stack->segments = vector_new(MM_STACK_INITIAL_SEGMENTS,mm_stack_segment_t*);
   mm_stack->segment_size = segment_size;

diff --git a/system/profiler_counter.c b/system/profiler_counter.c
@@ -103,13 +103,13 @@ void counter_print_stats(
   // Print Samples
   const uint64_t num_samples = counter_get_num_samples(counter);
   if (num_samples >= METRIC_FACTOR_1G) {
-    fprintf(stream," (samples=%"PRIu64"G",num_samples/METRIC_FACTOR_1G);
+    fprintf(stream," (samples=%" PRIu64 "G",num_samples/METRIC_FACTOR_1G);
   } else if (num_samples >= METRIC_FACTOR_1M) {
-    fprintf(stream," (samples=%"PRIu64"M",num_samples/METRIC_FACTOR_1M);
+    fprintf(stream," (samples=%" PRIu64 "M",num_samples/METRIC_FACTOR_1M);
   } else if (num_samples >= METRIC_FACTOR_1K) {
-    fprintf(stream," (samples=%"PRIu64"K",num_samples/METRIC_FACTOR_1K);
+    fprintf(stream," (samples=%" PRIu64 "K",num_samples/METRIC_FACTOR_1K);
   } else {
-    fprintf(stream," (samples=%"PRIu64"",num_samples);
+    fprintf(stream," (samples=%" PRIu64 "",num_samples);
     if (num_samples==0) {
       fprintf(stream,",--n/a--)}\n");
       return;
@@ -222,13 +222,13 @@ void percentage_print(
   // Print Samples
   const uint64_t num_samples = counter_get_num_samples(counter);
   if (num_samples >= METRIC_FACTOR_1G) {
-    fprintf(stream," (samples=%"PRIu64"G",num_samples/METRIC_FACTOR_1G);
+    fprintf(stream," (samples=%" PRIu64 "G",num_samples/METRIC_FACTOR_1G);
   } else if (num_samples >= METRIC_FACTOR_1M) {
-    fprintf(stream," (samples=%"PRIu64"M",num_samples/METRIC_FACTOR_1M);
+    fprintf(stream," (samples=%" PRIu64 "M",num_samples/METRIC_FACTOR_1M);
   } else if (num_samples >= METRIC_FACTOR_1K) {
-    fprintf(stream," (samples=%"PRIu64"K",num_samples/METRIC_FACTOR_1K);
+    fprintf(stream," (samples=%" PRIu64 "K",num_samples/METRIC_FACTOR_1K);
   } else {
-    fprintf(stream," (samples=%"PRIu64"",num_samples);
+    fprintf(stream," (samples=%" PRIu64 "",num_samples);
   }
   if (num_samples == 0) {
     fprintf(stream,")\n");

diff --git a/system/profiler_timer.c b/system/profiler_timer.c
@@ -122,7 +122,7 @@ void timer_print(
   } else if (total_time_ns >= 1000) {
     fprintf(stream,"%7.2f us",TIMER_CONVERT_NS_TO_US(total_time_ns));
   } else {
-    fprintf(stream,"%7"PRIu64" ns",total_time_ns);
+    fprintf(stream,"%7" PRIu64 " ns",total_time_ns);
   }
   // Print percentage wrt reference
   if (ref_timer!=NULL) {
@@ -141,15 +141,15 @@ void timer_print(
   // Print Calls
   const uint64_t num_calls = timer_get_num_samples(timer);
   if (num_calls >= 1000000000) {
-    fprintf(stream," (%5"PRIu64" Gcalls",num_calls/1000000000);
+    fprintf(stream," (%5" PRIu64 " Gcalls",num_calls/1000000000);
   } else if (num_calls >= 1000000) {
-    fprintf(stream," (%5"PRIu64" Mcalls",num_calls/1000000);
+    fprintf(stream," (%5" PRIu64 " Mcalls",num_calls/1000000);
   } else if (num_calls >= 1000) {
-    fprintf(stream," (%5"PRIu64" Kcalls",num_calls/1000);
+    fprintf(stream," (%5" PRIu64 " Kcalls",num_calls/1000);
   } else if (num_calls > 1 || num_calls == 0) {
-    fprintf(stream," (%5"PRIu64"  calls",num_calls);
+    fprintf(stream," (%5" PRIu64 "  calls",num_calls);
   } else {
-    fprintf(stream," (%5"PRIu64"   call",num_calls);
+    fprintf(stream," (%5" PRIu64 "   call",num_calls);
   }
   // Print time/call
   if (num_calls==0) {
@@ -164,7 +164,7 @@ void timer_print(
     } else if (ns_per_call > 1000) {
       fprintf(stream,",%7.2f us/call",TIMER_CONVERT_NS_TO_US(ns_per_call));
     } else {
-      fprintf(stream,",%7"PRIu64" ns/call",ns_per_call);
+      fprintf(stream,",%7" PRIu64 " ns/call",ns_per_call);
     }
   }
   // Print Max
@@ -176,7 +176,7 @@ void timer_print(
   } else if (min_ns > 1000) {
     fprintf(stream," {min%.2fus",TIMER_CONVERT_NS_TO_US(min_ns));
   } else {
-    fprintf(stream," {min%"PRIu64"ns",min_ns);
+    fprintf(stream," {min%" PRIu64 "ns",min_ns);
   }
   // Print Min
   const uint64_t max_ns = timer_get_max_ns(timer);
@@ -187,6 +187,6 @@ void timer_print(
   } else if (max_ns > 1000) {
     fprintf(stream,",Max%.2fus})\n",TIMER_CONVERT_NS_TO_US(max_ns));
   } else {
-    fprintf(stream,",Max%"PRIu64"ns})\n",max_ns);
+    fprintf(stream,",Max%" PRIu64 "ns})\n",max_ns);
   }
 }
diff --git a/tools/align_benchmark.c b/tools/align_benchmark.c
@@ -152,29 +152,8 @@ benchmark_args parameters = {
  */
 void align_pairwise_test() {
   // Patters & Texts
-//  char * pattern = "GATTACA";
-//  char * text = "GATCACTA";
-  char * pattern = "GGTTACTTGCGTATGTGCTACCTCATTTGCTAATAAAAATCGGCGCCCGACAGACTTAGGGCGGATTCTTAAGGGATTCGTTGGATCAATGGCCCGACTTCTATGGACCGACCGTGGAGTAACCCTTGAGGGCGCTAGCGCGTTATCTTTGTTAACAATAGCGAAAAATCTATGTACAACATCATGCTGCAATTCAACAA";
-  char * text =    "GGTTACTTGCGTTGTCTACCTCATTTGCTAATAAAAATCGGCGCCCGACACGACTTGGGCGGATTCTTAAAGGATTCGTCGATCCAATGGCCCACTTGCTATGTGACGATCCGTGGAGTACCCTATGAGTTGCGTAGCGATTATTCTTTGATAACAAAGCGAAAAATATATGTACAACAACATGCTGAATTCAACAAGAT";
-
-//  char *pattern =  "GGTTACTTGCGTATGTGCTACCTCATTTGCTAATAAAAATCGGCGCCCGACAGACTTAGGGCGGATTCTTAAGGGATTCGTTGGATCAATGGCCCGACTTCT"
-//      "ATGGACCGACCGTGGAGTAACCCTTGAGGGCGCTAGCGCGTTATCTTTGTTAACAATAGCGAAAAATCTATGTACAACATCATGCTGCAATTCAACAAGACGTCTCGAGGTAGTG"
-//      "ATACACCAGCCTGCGGGCCTCGCCCACCGCAGCAACGGGAATAGATCTCTTGGCATCGCATCGCGTGAGCCGAGACCGCGCACTCCTTACTTTGACCTGTAACCTGGCAAAGACG"
-//      "GTCCTTCTGTAACTTCATTGTCCAGCAACGGACAGCATAGTATAAGCCGCTCTACCCGGTACAGGGTGGTCCAAGCGAGCCGGAGAGATAATGAGATCTCCATGCGTAGCCATAA"
-//      "GATTAATAGTGTGTGACTTTAGGAAACCACGTGCGGCCTTCCGTCCAGATGACCAGGGTTACGGTCCAGAATGGTTATGATCCCGCTCATAACTGAAAAGCCGTACTACGAATGC"
-//      "GTCTTCTTGGGTGTACACTACCTCGCGGAAATAGGATGAGATGTGGCTAATCGTTCAGTCGAATGTTGCACGTTCCCGACTATGTGTAATCTTGTGCTCTTGCGCGATCCCCCAT"
-//      "ATGCACGACCTTTGAGCTTAAGAATCCCTGAACTCCAAGCCCCATGTCCGCGAGGAATCTCCTTCACCATTGACTATGGCAATCGGCGCGCTACGCGATTTTCAAACATTCTTCA"
-//      "GTGACCGAGTTAGTTCTGGTTTCTGAAGACTGCCTGGCGACGATGACCTTCGTGCGTCATTTGAAGTTAGACAACGCGAAGCGGTTAGCCGAACCCTAATTACTCTGGGGTCTCT"
-//      "TAAGCCCGGGTTGAAGTCATTTCGCACGAGGTTGGAATGGCGGATGGGATGTGACAAGTACCAAAGCCCATTGCAGATAAGTAATCCTAAAGG";
-//  char *text =     "GGTTACTTGCGTTGTCTACCTCATTTGCTAATAAAAATCGGCGCCCGACACGACTTGGGCGGATTCTTAAAGGATTCGTCGATCCAATGGCCCACTTGCTAT"
-//      "GTGACGATCCGTGGAGTACCCTATGAGTTGCGTAGCGATTATTCTTTGATAACAAAGCGAAAAATATATGTACAACAACATGCTGAATTCAACAAGATGTCTGGTAGGTGGTGAT"
-//      "CACACCAGCCTTCGGGCTCGCCACTGCAGCAACGGAAATAGTCTCTTTGGCTACCATCGACGTGAGCCGAGCAGCACATTCCTTACTTTGACGTGTAACTGGCAAGACGCTCCTT"
-//      "CTTACTTCTTGTCAGCACCGGATCAGCATAGTATAAGCCGCTCTACCACGGTACAGGGTGGTCCAAGCGACCCGGAGAGCAAATGAGCTCTCCAGGCGTAGCGATAAGATTCAAT"
-//      "AGTGTGTAAACTTTAGGTAAACCATCTGTGCGGCCTTCCATGCCCAGATGACCCAGGTTACGTCCAGAATGTATATCCGCAGCCATAACGAAAAGCCGTACTAGATGCGTCTTCT"
-//      "TGGCGTGTACACTAACGCGCGAAATAGGCATGAGATGTGGTAATCGTTCAGGTCGGTGTGGCGACGTTCCCACATATGTGTTAATCTTGTGCTCTTGCGCGATCCCCTTCTGACA"
-//      "CACCTTTGAGCTAAAACTCCCTGAACTCCAAGGCCCATCGGTCCGCGAGGAGATCCTTTCACCTTGACTATGGCAATGGCGCGCCGCGATTATCAAACTTTCTTCAGTGACCCGA"
-//      "GTTCGTTCTGGTTTCTTGAAGACTGCCTGGCGTAGATGACTTCGGTGCTCATCTGAATAGACAACGCGAAGCGGTGTGGCCGAACCCTAATACTCTTAGACTCTCTTAAGCCCGG"
-//      "GTTAAGTATCATTTCGCACGAGGTTGGCAGGCGGAGGGGTGACATGTACGAAAGCACATGCGATAAATAATCCTAAAGG";
+  char * pattern = "GATTACA";
+  char * text = "GATCACTA";
 
   // MMAllocator
   mm_allocator_t* const mm_allocator = mm_allocator_new(BUFFER_SIZE_8M);
@@ -221,11 +200,12 @@ void align_pairwise_test() {
   wavefront_aligner_attr_t attributes = wavefront_aligner_attr_default;
   attributes.distance_metric = gap_affine;
   attributes.affine_penalties = affine_penalties;
-  attributes.affine2p_penalties = affine2p_penalties;
-  attributes.reduction.reduction_strategy = wavefront_reduction_none; // wavefront_reduction_dynamic
+  // attributes.affine2p_penalties = affine2p_penalties;
+  attributes.reduction.reduction_strategy = wavefront_reduction_none; // wavefront_reduction_adaptive
   attributes.reduction.min_wavefront_length = 10;
   attributes.reduction.max_distance_threshold = 50;
-  attributes.low_memory = false;
+  attributes.alignment_scope = alignment_scope_alignment; // alignment_scope_score
+  attributes.low_memory = true;
   attributes.mm_allocator = mm_allocator;
   wavefront_aligner_t* const wf_aligner =
       wavefront_aligner_new(strlen(pattern),strlen(text),&attributes);
@@ -255,8 +235,8 @@ wavefront_aligner_t* align_benchmark_configure_wf(
     attributes.alignment_scope = alignment_scope_score;
   }
   // WF-Reduction
-  if (parameters.reduction_type == wavefront_reduction_dynamic) {
-    attributes.reduction.reduction_strategy = wavefront_reduction_dynamic;
+  if (parameters.reduction_type == wavefront_reduction_adaptive) {
+    attributes.reduction.reduction_strategy = wavefront_reduction_adaptive;
     attributes.reduction.min_wavefront_length = parameters.min_wavefront_length;
     attributes.reduction.max_distance_threshold = parameters.max_distance_threshold;
   } else {
@@ -442,6 +422,7 @@ void align_benchmark() {
   // Free
   fclose(input_file);
   if (align_input.output_file != NULL) fclose(align_input.output_file);
+  if (align_input.wf_aligner) wavefront_aligner_delete(align_input.wf_aligner);
   mm_allocator_delete(align_input.mm_allocator);
   free(line1);
   free(line2);
@@ -542,7 +523,7 @@ void parse_arguments(int argc,char** argv) {
         parameters.reduction_type = wavefront_reduction_none;
         parameters.algorithm = alignment_edit_wavefront;
       } else if (strcmp(optarg,"edit-wfa-adaptive")==0) {
-        parameters.reduction_type = wavefront_reduction_dynamic;
+        parameters.reduction_type = wavefront_reduction_adaptive;
         parameters.algorithm = alignment_edit_wavefront;
       /* Gap-Lineal */
       } else if (strcmp(optarg,"gap-lineal-nw")==0 ||
@@ -552,7 +533,7 @@ void parse_arguments(int argc,char** argv) {
         parameters.reduction_type = wavefront_reduction_none;
         parameters.algorithm = alignment_gap_lineal_wavefront;
       } else if (strcmp(optarg,"gap-lineal-wfa-adaptive")==0) {
-        parameters.reduction_type = wavefront_reduction_dynamic;
+        parameters.reduction_type = wavefront_reduction_adaptive;
         parameters.algorithm = alignment_gap_lineal_wavefront;
       /* Gap-Affine */
       } else if (strcmp(optarg,"gap-affine-swg")==0 ||
@@ -565,7 +546,7 @@ void parse_arguments(int argc,char** argv) {
         parameters.reduction_type = wavefront_reduction_none;
         parameters.algorithm = alignment_gap_affine_wavefront;
       } else if (strcmp(optarg,"gap-affine-wfa-adaptive")==0) {
-        parameters.reduction_type = wavefront_reduction_dynamic;
+        parameters.reduction_type = wavefront_reduction_adaptive;
         parameters.algorithm = alignment_gap_affine_wavefront;
       /* Gap-Affine 2-Pieces */
       } else if (strcmp(optarg,"gap-affine2p-dp")==0) {
@@ -574,7 +555,7 @@ void parse_arguments(int argc,char** argv) {
         parameters.reduction_type = wavefront_reduction_none;
         parameters.algorithm = alignment_gap_affine2p_wavefront;
       } else if (strcmp(optarg,"gap-affine2p-wfa-adaptive")==0) {
-        parameters.reduction_type = wavefront_reduction_dynamic;
+        parameters.reduction_type = wavefront_reduction_adaptive;
         parameters.algorithm = alignment_gap_affine2p_wavefront;
       } else {
         fprintf(stderr,"Algorithm '%s' not recognized\n",optarg);

diff --git a/utils/vector.c b/utils/vector.c
@@ -43,12 +43,12 @@
 vector_t* vector_new_(
     const uint64_t num_initial_elements,
     const uint64_t element_size) {
-  vector_t* const vector_buffer = malloc(sizeof(vector_t));
+  vector_t* const vector_buffer = (vector_t*) malloc(sizeof(vector_t));
   vector_buffer->element_size = element_size;
   vector_buffer->elements_allocated = num_initial_elements;
   vector_buffer->memory = malloc(num_initial_elements*element_size);
   if (!vector_buffer->memory) {
-    fprintf(stderr,"Could not create new vector (%"PRIu64" bytes requested)",
+    fprintf(stderr,"Could not create new vector (%" PRIu64 " bytes requested)",
         num_initial_elements*element_size);
     exit(1);
   }
@@ -76,7 +76,7 @@ void vector_reserve(
     vector->elements_allocated = num_elements>proposed?num_elements:proposed;
     vector->memory = realloc(vector->memory,vector->elements_allocated*vector->element_size);
     if (!vector->memory) {
-      fprintf(stderr,"Could not reserve vector (%"PRIu64" bytes requested)",
+      fprintf(stderr,"Could not reserve vector (%" PRIu64 " bytes requested)",
           vector->elements_allocated*vector->element_size);
       exit(1);
     }

diff --git a/wavefront/wavefront.c b/wavefront/wavefront.c
@@ -144,4 +144,18 @@ void wavefront_init_victim(
     wavefront->bt_prev = wavefront->bt_prev_mem - lo; // Center at k=0
   }
 }
+/*
+ * Utils
+ */
+uint64_t wavefront_get_size(
+    wavefront_t* const wavefront) {
+  uint64_t total_size = wavefront->max_wavefront_elements*sizeof(wf_offset_t);
+  if (wavefront->bt_pcigar_mem) {
+    total_size += wavefront->max_wavefront_elements*(sizeof(pcigar_t)+sizeof(block_idx_t));
+  }
+  return total_size;
+}
+
+
+
 
diff --git a/wavefront/wavefront.h b/wavefront/wavefront.h
@@ -136,4 +136,10 @@ void wavefront_init_victim(
     const int lo,
     const int hi);
 
+/*
+ * Utils
+ */
+uint64_t wavefront_get_size(
+    wavefront_t* const wavefront);
+
 #endif /* WAVEFRONT_H_ */