继续调整内存使用情况。

lihansunbai · May 30, 2024 · 5d1304c · 5d1304c
1 parent e064418
commit 5d1304c
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/EDGAR/EDGAR_eof.py b/EDGAR/EDGAR_eof.py
@@ -971,7 +971,7 @@ def multivariates_EOF_correlative_exporter(self, eof_results_dict, output_path=N
 
             # 由于dask 计算的特性，这里需要先将数据主动计算出结果
             item[1].compute_chunk_sizes()
-            temp_write_data = item[1].rechunk({0: -1, 1: 'auto'}, block_size_limit=1e8)
+            temp_write_data = item[1].rechunk({0: -1, 1: 'auto'}, block_size_limit=5e8)
             temp_write_data_chunked = temp_write_data.compute()
             temp_mode_state_group = temp_mode_group.create_group(str(item[0]))
             temp_mode_state_date = temp_mode_state_group.create_dataset(name='modes',
@@ -1114,8 +1114,8 @@ def multivariates_EOF_exporter(self, eof_results_dict, output_path=None):
             self.EE_logger.debug('Export eof modes to hdf: {}'.format(str(item[0])))
 
             # 由于dask 计算的特性，这里需要先将数据主动计算出结果
-            item[1].compute_chunk_sizes()
-            temp_write_data = item[1].rechunk({0: -1, 1: 'auto'}, block_size_limit=1e8)
+            # item[1].compute_chunk_sizes()
+            temp_write_data = item[1].rechunk({0: -1, 1: 'auto'}, block_size_limit=1e9)
             temp_write_data_chunked = temp_write_data.compute()
             temp_mode_state_group = temp_mode_group.create_group(str(item[0]))
             temp_mode_state_date = temp_mode_state_group.create_dataset(name='modes',