Merge pull request cisco#418 from dybvig/compress-level

Add a compress-level parameter
gongchengra · Apr 18, 2019 · 3ea6f8e · 3ea6f8e
2 parents 7df2fb2 + 722ffab
commit 3ea6f8e
Show file tree

Hide file tree

Showing 63 changed files with 1,397 additions and 772 deletions.
diff --git a/LOG b/LOG
@@ -1268,3 +1268,72 @@
     externs.h, system.h, expeditor.c, configure, Mf-*, Makefile.*nt,
     workarea, mat.ss, io.ms, io.stex, objects.stex, release_notes.stex,
     root-experr*, patch*
+- added compress-level parameter to select a compression level for
+  file writing and changed the default for lz4 compression to do a
+  better job compressing.  finished splitting glz input routines
+  apart from glz output routines and did a bit of other restructuring.
+  removed gzxfile struct-as-bytevector wrapper and moved its fd
+  into glzFile.  moved DEACTIVATE to before glzdopen_input calls
+  in S_new_open_input_fd and S_compress_input_fd, since glzdopen_input
+  reads from the file and could block.  the compress format and now
+  level are now recorded directly the thread context.  replaced
+  as-gz? flag bit in compressed bytevector header word with a small
+  number of bits recording the compression format at the bottom of
+  the header word.  flushed a couple of bytevector compression mats
+  that depended on the old representation.  (these last few changes
+  should make adding new compression formats easier.)  added
+  s-directory build options to choose whether to compress and, if
+  so, the format and level.
+    compress-io.h, compress-io.c, new-io.c, equates.h, system.h,
+    scheme.c, gc.c,
+    io.ss, cmacros.ss, back.ss, bytevector.ss, primdata.ss, s/Mf-base,
+    io.ms, mat.ss, bytevector.ms, root-experr*,
+    release_notes.stex, io.stex, system.stex, objects.stex
+- improved the effectiveness of LZ4 boot-file compression to within
+  15% of gzip by increasing the lz4 output-port in_buffer size to
+  1<<18.  With the previous size (1<<14) LZ4-compressed boot files
+  were about 50% larger.  set the lz4 input-port in_buffer and
+  out_buffer sizes to 1<<12 and 1<<14.  there's no clear win at
+  present for larger input-port buffer sizes.
+    compress-io.c
+- To reduce the memory hit for the increased output-port in_buffer
+  size and the corresponding increase in computed out_buffer size,
+  one output-side out_buffer is now allocated (lazily) per thread
+  and stored in the thread context.  The other buffers are now
+  directly a part of the lz4File_out and lz4File_in structures
+  rather than allocated separately.
+    compress-io.c, scheme.c, gc.c,
+    cmacros.ss
+- split out the buffer emit code from glzwrite_lz4 into a
+  separate glzemit_lz4 helper that is now also used by gzclose
+  so we can avoid dealing with a NULL buffer in glzwrite_lz4.
+  glzwrite_lz4 also uses it to writing large buffers directly and
+  avoid the memcpy.
+    compress-io.c
+- replaced lz4File_out and lz4File_in mode enumeration with the
+  compress format and inputp boolean.  using switch to check and
+  raising exceptions for unexpected values to further simplify
+  adding new compression formats in the future.
+    compress-io.c
+- replaced the never-defined struct lz4File pointer in glzFile
+  union with the more specific struct lz4File_in_r and Lz4File_out_r
+  pointers.
+    compress-io.h, compress-io.c
+- added free of lz4 structures to gzclose.  also changed file-close
+  logic generally so that (1) port is marked closed before anything is
+  freed to avoid dangling pointers in the case of an interrupt or
+  error, and (2) structures are freed even in the case of a write
+  or close error, before the error is reported.  also now mallocing
+  glz and lz4 structures after possibility of errors have passed where
+  possible and freeing them when not.
+    compress-io.c,
+    io.ss
+- added return-value checks to malloc calls and to a couple of other
+  C-library calls.
+    compress-io.c
+- corrected EINTR checks to look at errno rather than return codes.
+    compress-io.c
+- added S_ prefixes to the glz* exports
+    externs.h, compress-io.c, new-io.c, scheme.c, fasl.c
+- added entries for mutex-name and mutex-thread
+    threads.stex
diff --git a/boot/a6le/equates.h b/boot/a6le/equates.h
@@ -22,6 +22,13 @@ typedef unsigned long U64;
 #define $c_func_closure_record_index 0x3
 #define $c_func_code_object_index 0x2
 #define $c_func_code_record_index 0x1
+#define COMPRESS_FORMAT_BITS 0x3
+#define COMPRESS_GZIP 0x0
+#define COMPRESS_HIGH 0x2
+#define COMPRESS_LOW 0x0
+#define COMPRESS_LZ4 0x1
+#define COMPRESS_MAX 0x3
+#define COMPRESS_MEDIUM 0x1
 #define ERROR_CALL_ARGUMENT_COUNT 0x4
 #define ERROR_CALL_NONPROCEDURE 0x3
 #define ERROR_CALL_NONPROCEDURE_SYMBOL 0x2
@@ -500,7 +507,7 @@ typedef unsigned long U64;
 #define size_rp_header 0x20
 #define size_rtd_counts 0x810
 #define size_symbol 0x30
-#define size_tc 0x290
+#define size_tc 0x2A0
 #define size_thread 0x10
 #define size_tlc 0x20
 #define size_typed_object 0x10
@@ -551,13 +558,15 @@ typedef unsigned long U64;
 #define tc_ac0_disp 0x28
 #define tc_ac1_disp 0x30
 #define tc_active_disp 0x134
-#define tc_alloc_counter_disp 0x278
+#define tc_alloc_counter_disp 0x290
 #define tc_ap_disp 0x50
 #define tc_arg_regs_disp 0x0
 #define tc_block_counter_disp 0x1D0
 #define tc_cchain_disp 0x120
 #define tc_code_ranges_to_flush_disp 0x128
 #define tc_compile_profile_disp 0x228
+#define tc_compress_format_disp 0x270
+#define tc_compress_level_disp 0x278
 #define tc_cp_disp 0x40
 #define tc_current_error_disp 0x1C8
 #define tc_current_input_disp 0x1B8
@@ -574,15 +583,16 @@ typedef unsigned long U64;
 #define tc_generate_procedure_source_information_disp 0x238
 #define tc_generate_profile_forms_disp 0x240
 #define tc_guardian_entries_disp 0x118
-#define tc_instr_counter_disp 0x270
+#define tc_instr_counter_disp 0x288
 #define tc_keyboard_interrupt_pending_disp 0x1A8
+#define tc_lz4_out_buffer_disp 0x280
 #define tc_meta_level_disp 0x220
 #define tc_null_immutable_bytevector_disp 0x210
 #define tc_null_immutable_fxvector_disp 0x208
 #define tc_null_immutable_string_disp 0x218
 #define tc_null_immutable_vector_disp 0x200
 #define tc_optimize_level_disp 0x248
-#define tc_parameters_disp 0x280
+#define tc_parameters_disp 0x298
 #define tc_random_seed_disp 0x130
 #define tc_real_eap_disp 0x90
 #define tc_ret_disp 0x60
@@ -906,13 +916,15 @@ typedef unsigned long U64;
 #define AC0(x) (*((void* *)((uptr)(x)+40)))
 #define AC1(x) (*((void* *)((uptr)(x)+48)))
 #define ACTIVE(x) (*((I32 *)((uptr)(x)+308)))
-#define ALLOCCOUNTER(x) (*((U64 *)((uptr)(x)+632)))
+#define ALLOCCOUNTER(x) (*((U64 *)((uptr)(x)+656)))
 #define AP(x) (*((void* *)((uptr)(x)+80)))
 #define ARGREGS(x,i) (((void* *)((uptr)(x)+0))[i])
 #define BLOCKCOUNTER(x) (*((ptr *)((uptr)(x)+464)))
 #define CCHAIN(x) (*((ptr *)((uptr)(x)+288)))
 #define CODERANGESTOFLUSH(x) (*((ptr *)((uptr)(x)+296)))
 #define COMPILEPROFILE(x) (*((ptr *)((uptr)(x)+552)))
+#define COMPRESSFORMAT(x) (*((ptr *)((uptr)(x)+624)))
+#define COMPRESSLEVEL(x) (*((ptr *)((uptr)(x)+632)))
 #define CP(x) (*((void* *)((uptr)(x)+64)))
 #define CURRENTERROR(x) (*((ptr *)((uptr)(x)+456)))
 #define CURRENTINPUT(x) (*((ptr *)((uptr)(x)+440)))
@@ -929,15 +941,16 @@ typedef unsigned long U64;
 #define GENERATEPROCEDURESOURCEINFORMATION(x) (*((ptr *)((uptr)(x)+568)))
 #define GENERATEPROFILEFORMS(x) (*((ptr *)((uptr)(x)+576)))
 #define GUARDIANENTRIES(x) (*((ptr *)((uptr)(x)+280)))
-#define INSTRCOUNTER(x) (*((U64 *)((uptr)(x)+624)))
+#define INSTRCOUNTER(x) (*((U64 *)((uptr)(x)+648)))
 #define KEYBOARDINTERRUPTPENDING(x) (*((ptr *)((uptr)(x)+424)))
+#define LZ4OUTBUFFER(x) (*((void* *)((uptr)(x)+640)))
 #define METALEVEL(x) (*((ptr *)((uptr)(x)+544)))
 #define NULLIMMUTABLEBYTEVECTOR(x) (*((ptr *)((uptr)(x)+528)))
 #define NULLIMMUTABLEFXVECTOR(x) (*((ptr *)((uptr)(x)+520)))
 #define NULLIMMUTABLESTRING(x) (*((ptr *)((uptr)(x)+536)))
 #define NULLIMMUTABLEVECTOR(x) (*((ptr *)((uptr)(x)+512)))
 #define OPTIMIZELEVEL(x) (*((ptr *)((uptr)(x)+584)))
-#define PARAMETERS(x) (*((ptr *)((uptr)(x)+640)))
+#define PARAMETERS(x) (*((ptr *)((uptr)(x)+664)))
 #define RANDOMSEED(x) (*((U32 *)((uptr)(x)+304)))
 #define REAL_EAP(x) (*((void* *)((uptr)(x)+144)))
 #define RET(x) (*((void* *)((uptr)(x)+96)))

diff --git a/boot/a6le/petite.boot b/boot/a6le/petite.boot
diff --git a/boot/a6le/scheme.boot b/boot/a6le/scheme.boot
diff --git a/boot/a6nt/equates.h b/boot/a6nt/equates.h
@@ -22,6 +22,13 @@ typedef unsigned long long U64;
 #define $c_func_closure_record_index 0x3
 #define $c_func_code_object_index 0x2
 #define $c_func_code_record_index 0x1
+#define COMPRESS_FORMAT_BITS 0x3
+#define COMPRESS_GZIP 0x0
+#define COMPRESS_HIGH 0x2
+#define COMPRESS_LOW 0x0
+#define COMPRESS_LZ4 0x1
+#define COMPRESS_MAX 0x3
+#define COMPRESS_MEDIUM 0x1
 #define ERROR_CALL_ARGUMENT_COUNT 0x4
 #define ERROR_CALL_NONPROCEDURE 0x3
 #define ERROR_CALL_NONPROCEDURE_SYMBOL 0x2
@@ -500,7 +507,7 @@ typedef unsigned long long U64;
 #define size_rp_header 0x20
 #define size_rtd_counts 0x810
 #define size_symbol 0x30
-#define size_tc 0x290
+#define size_tc 0x2A0
 #define size_thread 0x10
 #define size_tlc 0x20
 #define size_typed_object 0x10
@@ -551,13 +558,15 @@ typedef unsigned long long U64;
 #define tc_ac0_disp 0x28
 #define tc_ac1_disp 0x30
 #define tc_active_disp 0x134
-#define tc_alloc_counter_disp 0x278
+#define tc_alloc_counter_disp 0x290
 #define tc_ap_disp 0x50
 #define tc_arg_regs_disp 0x0
 #define tc_block_counter_disp 0x1D0
 #define tc_cchain_disp 0x120
 #define tc_code_ranges_to_flush_disp 0x128
 #define tc_compile_profile_disp 0x228
+#define tc_compress_format_disp 0x270
+#define tc_compress_level_disp 0x278
 #define tc_cp_disp 0x40
 #define tc_current_error_disp 0x1C8
 #define tc_current_input_disp 0x1B8
@@ -574,15 +583,16 @@ typedef unsigned long long U64;
 #define tc_generate_procedure_source_information_disp 0x238
 #define tc_generate_profile_forms_disp 0x240
 #define tc_guardian_entries_disp 0x118
-#define tc_instr_counter_disp 0x270
+#define tc_instr_counter_disp 0x288
 #define tc_keyboard_interrupt_pending_disp 0x1A8
+#define tc_lz4_out_buffer_disp 0x280
 #define tc_meta_level_disp 0x220
 #define tc_null_immutable_bytevector_disp 0x210
 #define tc_null_immutable_fxvector_disp 0x208
 #define tc_null_immutable_string_disp 0x218
 #define tc_null_immutable_vector_disp 0x200
 #define tc_optimize_level_disp 0x248
-#define tc_parameters_disp 0x280
+#define tc_parameters_disp 0x298
 #define tc_random_seed_disp 0x130
 #define tc_real_eap_disp 0x90
 #define tc_ret_disp 0x60
@@ -906,13 +916,15 @@ typedef unsigned long long U64;
 #define AC0(x) (*((void* *)((uptr)(x)+40)))
 #define AC1(x) (*((void* *)((uptr)(x)+48)))
 #define ACTIVE(x) (*((I32 *)((uptr)(x)+308)))
-#define ALLOCCOUNTER(x) (*((U64 *)((uptr)(x)+632)))
+#define ALLOCCOUNTER(x) (*((U64 *)((uptr)(x)+656)))
 #define AP(x) (*((void* *)((uptr)(x)+80)))
 #define ARGREGS(x,i) (((void* *)((uptr)(x)+0))[i])
 #define BLOCKCOUNTER(x) (*((ptr *)((uptr)(x)+464)))
 #define CCHAIN(x) (*((ptr *)((uptr)(x)+288)))
 #define CODERANGESTOFLUSH(x) (*((ptr *)((uptr)(x)+296)))
 #define COMPILEPROFILE(x) (*((ptr *)((uptr)(x)+552)))
+#define COMPRESSFORMAT(x) (*((ptr *)((uptr)(x)+624)))
+#define COMPRESSLEVEL(x) (*((ptr *)((uptr)(x)+632)))
 #define CP(x) (*((void* *)((uptr)(x)+64)))
 #define CURRENTERROR(x) (*((ptr *)((uptr)(x)+456)))
 #define CURRENTINPUT(x) (*((ptr *)((uptr)(x)+440)))
@@ -929,15 +941,16 @@ typedef unsigned long long U64;
 #define GENERATEPROCEDURESOURCEINFORMATION(x) (*((ptr *)((uptr)(x)+568)))
 #define GENERATEPROFILEFORMS(x) (*((ptr *)((uptr)(x)+576)))
 #define GUARDIANENTRIES(x) (*((ptr *)((uptr)(x)+280)))
-#define INSTRCOUNTER(x) (*((U64 *)((uptr)(x)+624)))
+#define INSTRCOUNTER(x) (*((U64 *)((uptr)(x)+648)))
 #define KEYBOARDINTERRUPTPENDING(x) (*((ptr *)((uptr)(x)+424)))
+#define LZ4OUTBUFFER(x) (*((void* *)((uptr)(x)+640)))
 #define METALEVEL(x) (*((ptr *)((uptr)(x)+544)))
 #define NULLIMMUTABLEBYTEVECTOR(x) (*((ptr *)((uptr)(x)+528)))
 #define NULLIMMUTABLEFXVECTOR(x) (*((ptr *)((uptr)(x)+520)))
 #define NULLIMMUTABLESTRING(x) (*((ptr *)((uptr)(x)+536)))
 #define NULLIMMUTABLEVECTOR(x) (*((ptr *)((uptr)(x)+512)))
 #define OPTIMIZELEVEL(x) (*((ptr *)((uptr)(x)+584)))
-#define PARAMETERS(x) (*((ptr *)((uptr)(x)+640)))
+#define PARAMETERS(x) (*((ptr *)((uptr)(x)+664)))
 #define RANDOMSEED(x) (*((U32 *)((uptr)(x)+304)))
 #define REAL_EAP(x) (*((void* *)((uptr)(x)+144)))
 #define RET(x) (*((void* *)((uptr)(x)+96)))

diff --git a/boot/a6nt/petite.boot b/boot/a6nt/petite.boot
diff --git a/boot/a6nt/scheme.boot b/boot/a6nt/scheme.boot
diff --git a/boot/a6osx/equates.h b/boot/a6osx/equates.h
@@ -22,6 +22,13 @@ typedef unsigned long U64;
 #define $c_func_closure_record_index 0x3
 #define $c_func_code_object_index 0x2
 #define $c_func_code_record_index 0x1
+#define COMPRESS_FORMAT_BITS 0x3
+#define COMPRESS_GZIP 0x0
+#define COMPRESS_HIGH 0x2
+#define COMPRESS_LOW 0x0
+#define COMPRESS_LZ4 0x1
+#define COMPRESS_MAX 0x3
+#define COMPRESS_MEDIUM 0x1
 #define ERROR_CALL_ARGUMENT_COUNT 0x4
 #define ERROR_CALL_NONPROCEDURE 0x3
 #define ERROR_CALL_NONPROCEDURE_SYMBOL 0x2
@@ -500,7 +507,7 @@ typedef unsigned long U64;
 #define size_rp_header 0x20
 #define size_rtd_counts 0x810
 #define size_symbol 0x30
-#define size_tc 0x290
+#define size_tc 0x2A0
 #define size_thread 0x10
 #define size_tlc 0x20
 #define size_typed_object 0x10
@@ -551,13 +558,15 @@ typedef unsigned long U64;
 #define tc_ac0_disp 0x28
 #define tc_ac1_disp 0x30
 #define tc_active_disp 0x134
-#define tc_alloc_counter_disp 0x278
+#define tc_alloc_counter_disp 0x290
 #define tc_ap_disp 0x50
 #define tc_arg_regs_disp 0x0
 #define tc_block_counter_disp 0x1D0
 #define tc_cchain_disp 0x120
 #define tc_code_ranges_to_flush_disp 0x128
 #define tc_compile_profile_disp 0x228
+#define tc_compress_format_disp 0x270
+#define tc_compress_level_disp 0x278
 #define tc_cp_disp 0x40
 #define tc_current_error_disp 0x1C8
 #define tc_current_input_disp 0x1B8
@@ -574,15 +583,16 @@ typedef unsigned long U64;
 #define tc_generate_procedure_source_information_disp 0x238
 #define tc_generate_profile_forms_disp 0x240
 #define tc_guardian_entries_disp 0x118
-#define tc_instr_counter_disp 0x270
+#define tc_instr_counter_disp 0x288
 #define tc_keyboard_interrupt_pending_disp 0x1A8
+#define tc_lz4_out_buffer_disp 0x280
 #define tc_meta_level_disp 0x220
 #define tc_null_immutable_bytevector_disp 0x210
 #define tc_null_immutable_fxvector_disp 0x208
 #define tc_null_immutable_string_disp 0x218
 #define tc_null_immutable_vector_disp 0x200
 #define tc_optimize_level_disp 0x248
-#define tc_parameters_disp 0x280
+#define tc_parameters_disp 0x298
 #define tc_random_seed_disp 0x130
 #define tc_real_eap_disp 0x90
 #define tc_ret_disp 0x60
@@ -906,13 +916,15 @@ typedef unsigned long U64;
 #define AC0(x) (*((void* *)((uptr)(x)+40)))
 #define AC1(x) (*((void* *)((uptr)(x)+48)))
 #define ACTIVE(x) (*((I32 *)((uptr)(x)+308)))
-#define ALLOCCOUNTER(x) (*((U64 *)((uptr)(x)+632)))
+#define ALLOCCOUNTER(x) (*((U64 *)((uptr)(x)+656)))
 #define AP(x) (*((void* *)((uptr)(x)+80)))
 #define ARGREGS(x,i) (((void* *)((uptr)(x)+0))[i])
 #define BLOCKCOUNTER(x) (*((ptr *)((uptr)(x)+464)))
 #define CCHAIN(x) (*((ptr *)((uptr)(x)+288)))
 #define CODERANGESTOFLUSH(x) (*((ptr *)((uptr)(x)+296)))
 #define COMPILEPROFILE(x) (*((ptr *)((uptr)(x)+552)))
+#define COMPRESSFORMAT(x) (*((ptr *)((uptr)(x)+624)))
+#define COMPRESSLEVEL(x) (*((ptr *)((uptr)(x)+632)))
 #define CP(x) (*((void* *)((uptr)(x)+64)))
 #define CURRENTERROR(x) (*((ptr *)((uptr)(x)+456)))
 #define CURRENTINPUT(x) (*((ptr *)((uptr)(x)+440)))
@@ -929,15 +941,16 @@ typedef unsigned long U64;
 #define GENERATEPROCEDURESOURCEINFORMATION(x) (*((ptr *)((uptr)(x)+568)))
 #define GENERATEPROFILEFORMS(x) (*((ptr *)((uptr)(x)+576)))
 #define GUARDIANENTRIES(x) (*((ptr *)((uptr)(x)+280)))
-#define INSTRCOUNTER(x) (*((U64 *)((uptr)(x)+624)))
+#define INSTRCOUNTER(x) (*((U64 *)((uptr)(x)+648)))
 #define KEYBOARDINTERRUPTPENDING(x) (*((ptr *)((uptr)(x)+424)))
+#define LZ4OUTBUFFER(x) (*((void* *)((uptr)(x)+640)))
 #define METALEVEL(x) (*((ptr *)((uptr)(x)+544)))
 #define NULLIMMUTABLEBYTEVECTOR(x) (*((ptr *)((uptr)(x)+528)))
 #define NULLIMMUTABLEFXVECTOR(x) (*((ptr *)((uptr)(x)+520)))
 #define NULLIMMUTABLESTRING(x) (*((ptr *)((uptr)(x)+536)))
 #define NULLIMMUTABLEVECTOR(x) (*((ptr *)((uptr)(x)+512)))
 #define OPTIMIZELEVEL(x) (*((ptr *)((uptr)(x)+584)))
-#define PARAMETERS(x) (*((ptr *)((uptr)(x)+640)))
+#define PARAMETERS(x) (*((ptr *)((uptr)(x)+664)))
 #define RANDOMSEED(x) (*((U32 *)((uptr)(x)+304)))
 #define REAL_EAP(x) (*((void* *)((uptr)(x)+144)))
 #define RET(x) (*((void* *)((uptr)(x)+96)))

diff --git a/boot/a6osx/petite.boot b/boot/a6osx/petite.boot
diff --git a/boot/a6osx/scheme.boot b/boot/a6osx/scheme.boot