From 02a88b9b19624ca10b722ec4ef14aff877495f5d Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 23 Aug 2020 10:59:30 +0900 Subject: [PATCH 01/13] Fix lint errors --- pytablewriter/writer/_table_writer.py | 208 +++++++++++++------------- pytablewriter/writer/binary/_excel.py | 44 +++--- 2 files changed, 126 insertions(+), 126 deletions(-) diff --git a/pytablewriter/writer/_table_writer.py b/pytablewriter/writer/_table_writer.py index 898d2d35..2262cea2 100644 --- a/pytablewriter/writer/_table_writer.py +++ b/pytablewriter/writer/_table_writer.py @@ -109,18 +109,6 @@ def callback_example(iter_count, iter_length): - second argument: a total number of iteration """ - @property - def is_formatting_float(self) -> bool: - return self._dp_extractor.is_formatting_float - - @is_formatting_float.setter - def is_formatting_float(self, value: bool) -> None: - if self._dp_extractor.is_formatting_float == value: - return - - self._dp_extractor.is_formatting_float = value - self.__clear_preprocess() - @property def margin(self) -> int: raise NotImplementedError() @@ -129,17 +117,6 @@ def margin(self) -> int: def margin(self, value: int) -> None: raise NotImplementedError() - @property - def headers(self) -> Sequence[str]: - """Headers of a table to be outputted. - """ - - return self._dp_extractor.headers - - @headers.setter - def headers(self, value: Sequence[str]) -> None: - self._dp_extractor.headers = value - @property def header_list(self): warnings.warn("'header_list' has moved to 'headers'", DeprecationWarning) @@ -180,63 +157,13 @@ def table_format(self): """Get the format of the writer. Returns: - TableFormat: + TableFormat: """ from .._table_format import TableFormat return TableFormat.from_name(self.format_name) - @property - def type_hints(self) -> Sequence: - """ - Type hints for each column of the tabular data. - Writers convert data for each column using the type hints information - before writing tables when you call ``write_xxx`` methods. - - Acceptable values are as follows: - - - |None| (automatically detect column type from values in the column) - - :py:class:`pytablewriter.typehint.Bool` - - :py:class:`pytablewriter.typehint.DateTime` - - :py:class:`pytablewriter.typehint.Dictionary` - - :py:class:`pytablewriter.typehint.Infinity` - - :py:class:`pytablewriter.typehint.Integer` - - :py:class:`pytablewriter.typehint.IpAddress` - - :py:class:`pytablewriter.typehint.List` - - :py:class:`pytablewriter.typehint.Nan` - - :py:class:`pytablewriter.typehint.NoneType` - - :py:class:`pytablewriter.typehint.NullString` - - :py:class:`pytablewriter.typehint.RealNumber` - - :py:class:`pytablewriter.typehint.String` - - If a type-hint value is not |None|, the writer tries to - convert data for each data in a column to type-hint class. - If the type-hint value is |None| or failed to convert data, - the writer automatically detect column data type from - the column data. - - If ``type_hints`` is |None|, the writer detects data types for all - of the columns automatically and writes a table by using detected column types. - - Defaults to |None|. - - :Examples: - - :ref:`example-type-hint-js` - - :ref:`example-type-hint-python` - """ - - return self._dp_extractor.column_type_hints - - @type_hints.setter - def type_hints(self, value: Sequence) -> None: - hints = list(value) - if self.type_hints == hints: - return - - self.__set_type_hints(hints) - self.__clear_preprocess() - @property def type_hint_list(self): warnings.warn("'type_hint_list' has moved to 'type_hints'", DeprecationWarning) @@ -271,10 +198,6 @@ def style_list(self, value): self.column_styles = value - def register_trans_func(self, trans_func: TransFunc) -> None: - self._dp_extractor.register_trans_func(trans_func) - self.__clear_preprocess() - @property def value_preprocessor(self): return self._dp_extractor.preprocessor @@ -292,15 +215,6 @@ def value_preprocessor(self, value): self._dp_extractor.preprocessor = value self.__clear_preprocess() - def update_preprocessor(self, **kwargs) -> None: - # TODO: documentation - # is_escape_formula_injection: for CSV/Excel - - if not self._dp_extractor.update_preprocessor(**kwargs): - return - - self.__clear_preprocess() - @property def escape_formula_injection(self): # Deprecated @@ -323,23 +237,6 @@ def stream(self): def stream(self, value) -> None: self._stream = value - @property - def _quoting_flags(self) -> Dict[Typecode, bool]: - return self._dp_extractor.quoting_flags - - @_quoting_flags.setter - def _quoting_flags(self, value: Mapping[Typecode, bool]) -> None: - self._dp_extractor.quoting_flags = value - self.__clear_preprocess() - - @property - def max_workers(self) -> int: - return self._dp_extractor.max_workers - - @max_workers.setter - def max_workers(self, value: Optional[int]) -> None: - self._dp_extractor.max_workers = value # type: ignore - @abc.abstractmethod def _write_table(self, **kwargs) -> None: pass @@ -430,6 +327,37 @@ def __clear_preprocess_data(self) -> None: self._table_value_matrix = [] # type: List[Union[List[str], Dict]] self._table_value_dp_matrix = [] # type: Sequence[Sequence[DataProperty]] + @property + def headers(self) -> Sequence[str]: + """Headers of a table to be outputted. + """ + + return self._dp_extractor.headers + + @headers.setter + def headers(self, value: Sequence[str]) -> None: + self._dp_extractor.headers = value + + @property + def is_formatting_float(self) -> bool: + return self._dp_extractor.is_formatting_float + + @is_formatting_float.setter + def is_formatting_float(self, value: bool) -> None: + if self._dp_extractor.is_formatting_float == value: + return + + self._dp_extractor.is_formatting_float = value + self.__clear_preprocess() + + @property + def max_workers(self) -> int: + return self._dp_extractor.max_workers + + @max_workers.setter + def max_workers(self, value: Optional[int]) -> None: + self._dp_extractor.max_workers = value # type: ignore + @property def table_name(self) -> str: """Name of a table. @@ -441,6 +369,56 @@ def table_name(self) -> str: def table_name(self, value: str) -> None: self._table_name = value + @property + def type_hints(self) -> Sequence: + """ + Type hints for each column of the tabular data. + Writers convert data for each column using the type hints information + before writing tables when you call ``write_xxx`` methods. + + Acceptable values are as follows: + + - |None| (automatically detect column type from values in the column) + - :py:class:`pytablewriter.typehint.Bool` + - :py:class:`pytablewriter.typehint.DateTime` + - :py:class:`pytablewriter.typehint.Dictionary` + - :py:class:`pytablewriter.typehint.Infinity` + - :py:class:`pytablewriter.typehint.Integer` + - :py:class:`pytablewriter.typehint.IpAddress` + - :py:class:`pytablewriter.typehint.List` + - :py:class:`pytablewriter.typehint.Nan` + - :py:class:`pytablewriter.typehint.NoneType` + - :py:class:`pytablewriter.typehint.NullString` + - :py:class:`pytablewriter.typehint.RealNumber` + - :py:class:`pytablewriter.typehint.String` + + If a type-hint value is not |None|, the writer tries to + convert data for each data in a column to type-hint class. + If the type-hint value is |None| or failed to convert data, + the writer automatically detect column data type from + the column data. + + If ``type_hints`` is |None|, the writer detects data types for all + of the columns automatically and writes a table by using detected column types. + + Defaults to |None|. + + :Examples: + - :ref:`example-type-hint-js` + - :ref:`example-type-hint-python` + """ + + return self._dp_extractor.column_type_hints + + @type_hints.setter + def type_hints(self, value: Sequence) -> None: + hints = list(value) + if self.type_hints == hints: + return + + self.__set_type_hints(hints) + self.__clear_preprocess() + @property def default_style(self) -> Style: """Default |Style| of table cells. @@ -513,6 +491,15 @@ def enable_ansi_escape(self, value: bool) -> None: self.__enable_ansi_escape = value self.__clear_preprocess() + @property + def _quoting_flags(self) -> Dict[Typecode, bool]: + return self._dp_extractor.quoting_flags + + @_quoting_flags.setter + def _quoting_flags(self, value: Mapping[Typecode, bool]) -> None: + self._dp_extractor.quoting_flags = value + self.__clear_preprocess() + def add_style_filter(self, style_filter: StyleFilterFunc) -> None: """Add a style filter function. @@ -849,6 +836,19 @@ def from_writer( self._is_complete_header_preprocess = writer._is_complete_header_preprocess self._is_complete_value_matrix_preprocess = writer._is_complete_value_matrix_preprocess + def register_trans_func(self, trans_func: TransFunc) -> None: + self._dp_extractor.register_trans_func(trans_func) + self.__clear_preprocess() + + def update_preprocessor(self, **kwargs) -> None: + # TODO: documentation + # is_escape_formula_injection: for CSV/Excel + + if not self._dp_extractor.update_preprocessor(**kwargs): + return + + self.__clear_preprocess() + def write_table(self, **kwargs) -> None: """ |write_table|. diff --git a/pytablewriter/writer/binary/_excel.py b/pytablewriter/writer/binary/_excel.py index aa2d1f03..00061542 100644 --- a/pytablewriter/writer/binary/_excel.py +++ b/pytablewriter/writer/binary/_excel.py @@ -33,6 +33,28 @@ def support_split_write(self) -> bool: def workbook(self) -> Optional[ExcelWorkbookInterface]: return self._workbook + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + + self._workbook = None # type: Optional[ExcelWorkbookInterface] + + self._dp_extractor.type_value_map = { + typepy.Typecode.INFINITY: "Inf", + typepy.Typecode.NAN: "NaN", + } + + self._first_header_row = 0 + self._last_header_row = self.first_header_row + self._first_data_row = self.last_header_row + 1 + self._first_data_col = 0 + self._last_data_row = None # type: Optional[int] + self._last_data_col = None # type: Optional[int] + + self._current_data_row = self._first_data_row + + self._quoting_flags = copy.deepcopy(dataproperty.NOT_QUOTING_FLAGS) + self._quoting_flags[typepy.Typecode.DATETIME] = True + @property def first_header_row(self) -> int: """ @@ -99,28 +121,6 @@ def last_data_col(self) -> Optional[int]: return self._last_data_col - def __init__(self, **kwargs) -> None: - super().__init__(**kwargs) - - self._workbook = None # type: Optional[ExcelWorkbookInterface] - - self._dp_extractor.type_value_map = { - typepy.Typecode.INFINITY: "Inf", - typepy.Typecode.NAN: "NaN", - } - - self._first_header_row = 0 - self._last_header_row = self.first_header_row - self._first_data_row = self.last_header_row + 1 - self._first_data_col = 0 - self._last_data_row = None # type: Optional[int] - self._last_data_col = None # type: Optional[int] - - self._current_data_row = self._first_data_row - - self._quoting_flags = copy.deepcopy(dataproperty.NOT_QUOTING_FLAGS) - self._quoting_flags[typepy.Typecode.DATETIME] = True - def is_opened(self) -> bool: return self.workbook is not None From 6d65fc2a3bce6b09ac3db745645557f63ffa439b Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 23 Aug 2020 11:02:31 +0900 Subject: [PATCH 02/13] Modify typehint module definition --- pytablewriter/{typehint.py => typehint/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pytablewriter/{typehint.py => typehint/__init__.py} (100%) diff --git a/pytablewriter/typehint.py b/pytablewriter/typehint/__init__.py similarity index 100% rename from pytablewriter/typehint.py rename to pytablewriter/typehint/__init__.py From 1f70528caf5d70ad3adbb6fdd2f2d960d9706b9f Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 23 Aug 2020 11:04:25 +0900 Subject: [PATCH 03/13] Remove deprecated properties --- pytablewriter/writer/_table_writer.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pytablewriter/writer/_table_writer.py b/pytablewriter/writer/_table_writer.py index 2262cea2..693d78be 100644 --- a/pytablewriter/writer/_table_writer.py +++ b/pytablewriter/writer/_table_writer.py @@ -215,20 +215,6 @@ def value_preprocessor(self, value): self._dp_extractor.preprocessor = value self.__clear_preprocess() - @property - def escape_formula_injection(self): - # Deprecated - return self._dp_extractor.preprocessor.is_escape_formula_injection - - @escape_formula_injection.setter - def escape_formula_injection(self, value) -> None: - # Deprecated - if self._dp_extractor.preprocessor.is_escape_formula_injection == value: - return - - self._dp_extractor.preprocessor.is_escape_formula_injection = value - self.__clear_preprocess() - @property def stream(self): return self._stream From 8e3dc51d102b57fa3596b69d758a44bd67863b4e Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 16:58:58 +0900 Subject: [PATCH 04/13] Apply formatter --- pytablewriter/style/_style.py | 3 +- pytablewriter/writer/_table_writer.py | 15 ++---- pytablewriter/writer/text/_text_writer.py | 26 ++++++---- pytablewriter/writer/text/_toml.py | 3 +- test/data.py | 56 ++++++++++++++++++--- test/test_style.py | 3 +- test/test_table_format.py | 6 ++- test/writer/text/test_json_writer.py | 7 ++- test/writer/text/test_latex_table_writer.py | 6 ++- test/writer/text/test_markdown_writer.py | 14 +++++- test/writer/text/test_numpy_writer.py | 8 ++- test/writer/text/test_pandas_writer.py | 8 ++- test/writer/text/test_unicode_writer.py | 6 ++- test/writer/text/test_yaml_writer.py | 9 +++- 14 files changed, 128 insertions(+), 42 deletions(-) diff --git a/pytablewriter/style/_style.py b/pytablewriter/style/_style.py index f0c824f0..ea4c402e 100644 --- a/pytablewriter/style/_style.py +++ b/pytablewriter/style/_style.py @@ -250,8 +250,7 @@ def __ne__(self, other): return NotImplemented if equal is NotImplemented else not equal def update(self, **kwargs) -> None: - """Update specified style attributes. - """ + """Update specified style attributes.""" self.__update(initialize=False, **kwargs) def __update(self, initialize: bool, **kwargs) -> None: diff --git a/pytablewriter/writer/_table_writer.py b/pytablewriter/writer/_table_writer.py index 693d78be..7046c3d5 100644 --- a/pytablewriter/writer/_table_writer.py +++ b/pytablewriter/writer/_table_writer.py @@ -130,8 +130,7 @@ def header_list(self, value): @property def value_matrix(self) -> List: - """Data of a table to be outputted. - """ + """Data of a table to be outputted.""" return self.__value_matrix_org @@ -315,8 +314,7 @@ def __clear_preprocess_data(self) -> None: @property def headers(self) -> Sequence[str]: - """Headers of a table to be outputted. - """ + """Headers of a table to be outputted.""" return self._dp_extractor.headers @@ -346,8 +344,7 @@ def max_workers(self, value: Optional[int]) -> None: @property def table_name(self) -> str: - """Name of a table. - """ + """Name of a table.""" return self._table_name @@ -407,8 +404,7 @@ def type_hints(self, value: Sequence) -> None: @property def default_style(self) -> Style: - """Default |Style| of table cells. - """ + """Default |Style| of table cells.""" return self.__default_style @@ -508,8 +504,7 @@ def add_col_separator_style_filter(self, style_filter: ColSeparatorStyleFilterFu raise NotImplementedError("this method only implemented in text format writer classes") def clear_theme(self) -> None: - """Remove all of the style filters. - """ + """Remove all of the style filters.""" if not self._style_filters: return diff --git a/pytablewriter/writer/text/_text_writer.py b/pytablewriter/writer/text/_text_writer.py index f06c1702..2c7c0a67 100644 --- a/pytablewriter/writer/text/_text_writer.py +++ b/pytablewriter/writer/text/_text_writer.py @@ -182,8 +182,7 @@ def add_col_separator_style_filter(self, style_filter: ColSeparatorStyleFilterFu self._clear_preprocess() def clear_theme(self) -> None: - """Remove all of the style filters. - """ + """Remove all of the style filters.""" super().clear_theme() @@ -359,7 +358,8 @@ def __get_row_separator_items(self, margin_format: str, separator_char: str) -> def _get_header_format_string(self, col_dp: ColumnDataProperty, value_dp: DataProperty) -> str: return "{{:{:s}{:s}}}".format( - get_align_char(Align.CENTER), str(self._get_padding_len(col_dp, value_dp)), + get_align_char(Align.CENTER), + str(self._get_padding_len(col_dp, value_dp)), ) def _to_header_item(self, col_dp: ColumnDataProperty, value_dp: DataProperty) -> str: @@ -420,7 +420,9 @@ def __to_column_delimiter( ) style = self._fetch_col_separator_style( - left_cell=left_cell, right_cell=right_cell, default_style=self.default_style, + left_cell=left_cell, + right_cell=right_cell, + default_style=self.default_style, ) return self._styler.apply_terminal_style(col_delimiter, style=style) @@ -432,7 +434,10 @@ def _write_row(self, row: int, values: Sequence[str]) -> None: col_delimiters = ( [ self.__to_column_delimiter( - row, None, self._column_dp_list[0], self.char_left_side_row, + row, + None, + self._column_dp_list[0], + self.char_left_side_row, ) ] + [ @@ -446,7 +451,10 @@ def _write_row(self, row: int, values: Sequence[str]) -> None: ] + [ self.__to_column_delimiter( - row, self._column_dp_list[-1], None, self.char_right_side_row, + row, + self._column_dp_list[-1], + None, + self.char_right_side_row, ) ] ) @@ -556,14 +564,12 @@ def set_indent_level(self, indent_level: int) -> None: self._indent_level = indent_level def inc_indent_level(self) -> None: - """Increment the indentation level. - """ + """Increment the indentation level.""" self._indent_level += 1 def dec_indent_level(self) -> None: - """Decrement the indentation level. - """ + """Decrement the indentation level.""" self._indent_level -= 1 diff --git a/pytablewriter/writer/text/_toml.py b/pytablewriter/writer/text/_toml.py index 86c4940c..06290d55 100644 --- a/pytablewriter/writer/text/_toml.py +++ b/pytablewriter/writer/text/_toml.py @@ -72,7 +72,8 @@ def __init__(self, _dict=dict, preserve=False): row = {} for header, value in zip( - self.headers, [serialize_dp(value_dp) for value_dp in value_dp_list], + self.headers, + [serialize_dp(value_dp) for value_dp in value_dp_list], ): if typepy.is_null_string(value): continue diff --git a/test/data.py b/test/data.py index b6512526..1677be06 100644 --- a/test/data.py +++ b/test/data.py @@ -42,7 +42,18 @@ "time", ] mix_value_matrix = [ - [1, 1.1, "aa", 1, 1, True, INF, NAN, 1.0, TIME,], + [ + 1, + 1.1, + "aa", + 1, + 1, + True, + INF, + NAN, + 1.0, + TIME, + ], [ 2, 2.2, @@ -55,7 +66,18 @@ INF, "2017-01-02 03:04:05+09:00", ], - [3, 3.33, "cccc", -3, "ccc", True, float("infinity"), float("NAN"), NAN, TIME,], + [ + 3, + 3.33, + "cccc", + -3, + "ccc", + True, + float("infinity"), + float("NAN"), + NAN, + TIME, + ], ] mix_tabledata = TableData(table_name="mix data", headers=mix_header_list, rows=mix_value_matrix) @@ -70,15 +92,33 @@ ) value_matrix_iter = [ - [[1, 2, 3], [11, 12, 13],], - [[1, 2, 3], [11, 12, 13],], - [[101, 102, 103], [1001, 1002, 1003],], + [ + [1, 2, 3], + [11, 12, 13], + ], + [ + [1, 2, 3], + [11, 12, 13], + ], + [ + [101, 102, 103], + [1001, 1002, 1003], + ], ] value_matrix_iter_1 = [ - [["a b c d e f g h i jklmn", 2.1, 3], ["aaaaa", 12.1, 13],], - [["bbb", 2, 3], ["cc", 12, 13],], - [["a", 102, 103], ["", 1002, 1003],], + [ + ["a b c d e f g h i jklmn", 2.1, 3], + ["aaaaa", 12.1, 13], + ], + [ + ["bbb", 2, 3], + ["cc", 12, 13], + ], + [ + ["a", 102, 103], + ["", 1002, 1003], + ], ] Data = collections.namedtuple("Data", "table indent header value expected") diff --git a/test/test_style.py b/test/test_style.py index b4b9f4b9..3d90fb08 100644 --- a/test/test_style.py +++ b/test/test_style.py @@ -19,7 +19,8 @@ class Test_Cell_is_header_row: @pytest.mark.parametrize( - ["row", "expected"], [[-1, True], [0, False], [sys.maxsize, False]], + ["row", "expected"], + [[-1, True], [0, False], [sys.maxsize, False]], ) def test_normal(self, row, expected): cell = Cell(row=row, col=0, value=None, default_style=None) diff --git a/test/test_table_format.py b/test/test_table_format.py index 70eed984..73c2cf73 100644 --- a/test/test_table_format.py +++ b/test/test_table_format.py @@ -52,7 +52,11 @@ def test_normal(self, value, expected): class Test_TableFormat_from_name: @pytest.mark.parametrize( ["value", "expected"], - [["csv", TableFormat.CSV], ["CSV", TableFormat.CSV], ["excel", TableFormat.EXCEL_XLSX],], + [ + ["csv", TableFormat.CSV], + ["CSV", TableFormat.CSV], + ["excel", TableFormat.EXCEL_XLSX], + ], ) def test_normal(self, value, expected): assert TableFormat.from_name(value) == expected diff --git a/test/writer/text/test_json_writer.py b/test/writer/text/test_json_writer.py index 09ae2bfb..92c5beeb 100644 --- a/test/writer/text/test_json_writer.py +++ b/test/writer/text/test_json_writer.py @@ -207,7 +207,12 @@ ] exception_test_data_list = [ - Data(table="", header=[], value=normal_test_data_list[0].value, expected=ValueError,), + Data( + table="", + header=[], + value=normal_test_data_list[0].value, + expected=ValueError, + ), ] table_writer_class = pytablewriter.JsonTableWriter diff --git a/test/writer/text/test_latex_table_writer.py b/test/writer/text/test_latex_table_writer.py index 00ea4e28..16f7cc37 100644 --- a/test/writer/text/test_latex_table_writer.py +++ b/test/writer/text/test_latex_table_writer.py @@ -58,7 +58,11 @@ \end{array} """, ), - Data(header=[], value=[], expected="",), + Data( + header=[], + value=[], + expected="", + ), ] diff --git a/test/writer/text/test_markdown_writer.py b/test/writer/text/test_markdown_writer.py index 9517a4ec..e759dd6b 100644 --- a/test/writer/text/test_markdown_writer.py +++ b/test/writer/text/test_markdown_writer.py @@ -392,7 +392,14 @@ """ ), ), - Data(table="", indent=0, header=[], value=[], is_formatting_float=True, expected="",), + Data( + table="", + indent=0, + header=[], + value=[], + is_formatting_float=True, + expected="", + ), ] table_writer_class = ptw.MarkdownTableWriter @@ -422,7 +429,10 @@ def test_normal_kwargs(self): writer = table_writer_class( headers=["w/ strike", "w/ line through"], value_matrix=[["strike", "line-through"]], - column_styles=[Style(decoration_line="strike"), Style(decoration_line="line-through"),], + column_styles=[ + Style(decoration_line="strike"), + Style(decoration_line="line-through"), + ], ) expected = dedent( diff --git a/test/writer/text/test_numpy_writer.py b/test/writer/text/test_numpy_writer.py index 3923fad4..cb893dc0 100644 --- a/test/writer/text/test_numpy_writer.py +++ b/test/writer/text/test_numpy_writer.py @@ -165,7 +165,13 @@ def test_exception(self, table, indent, header, value, expected): [ [data.table, data.indent, data.header, data.value, data.expected] for data in [ - Data(table=None, indent=0, header=headers, value=value_matrix, expected="",) + Data( + table=None, + indent=0, + header=headers, + value=value_matrix, + expected="", + ) ] ], ) diff --git a/test/writer/text/test_pandas_writer.py b/test/writer/text/test_pandas_writer.py index 99e31196..d8d916e3 100644 --- a/test/writer/text/test_pandas_writer.py +++ b/test/writer/text/test_pandas_writer.py @@ -124,7 +124,13 @@ """ ), ), - Data(table="empty", indent=0, header=[], value=[], expected="",), + Data( + table="empty", + indent=0, + header=[], + value=[], + expected="", + ), ] exception_test_data_list = [ diff --git a/test/writer/text/test_unicode_writer.py b/test/writer/text/test_unicode_writer.py index 0221a397..07c298e4 100644 --- a/test/writer/text/test_unicode_writer.py +++ b/test/writer/text/test_unicode_writer.py @@ -15,7 +15,11 @@ class Test_UnicodeTableWriter_write_new_line: @pytest.mark.parametrize( - ["table_writer_class"], [[UnicodeTableWriter], [BoldUnicodeTableWriter],] + ["table_writer_class"], + [ + [UnicodeTableWriter], + [BoldUnicodeTableWriter], + ], ) def test_normal(self, capsys, table_writer_class): writer = table_writer_class() diff --git a/test/writer/text/test_yaml_writer.py b/test/writer/text/test_yaml_writer.py index 3947be9c..2131e973 100644 --- a/test/writer/text/test_yaml_writer.py +++ b/test/writer/text/test_yaml_writer.py @@ -55,7 +55,11 @@ ), ), Data( - tabledata=TableData(table_name=None, headers=[], rows=value_matrix,), + tabledata=TableData( + table_name=None, + headers=[], + rows=value_matrix, + ), expected=dedent( """\ - - 1 @@ -118,7 +122,8 @@ def test_normal(self, capsys): class Test_YamlTableWriter_write_table: @pytest.mark.parametrize( - ["value", "expected"], [[data.tabledata, data.expected] for data in normal_test_data_list], + ["value", "expected"], + [[data.tabledata, data.expected] for data in normal_test_data_list], ) def test_normal(self, capsys, value, expected): writer = table_writer_class() From 6ee5ebc5627d9198c178d118d7904f5b3fba64c0 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 18:11:20 +0900 Subject: [PATCH 05/13] Fix initialization by keyword arguments of writer class constructor --- pytablewriter/writer/_null.py | 8 ++++++++ pytablewriter/writer/binary/_interface.py | 2 ++ pytablewriter/writer/text/_rst.py | 1 - pytablewriter/writer/text/_spacealigned.py | 2 +- pytablewriter/writer/text/_unicode.py | 2 -- pytablewriter/writer/text/sourcecode/_pandas.py | 1 - pytablewriter/writer/text/sourcecode/_python.py | 1 - 7 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pytablewriter/writer/_null.py b/pytablewriter/writer/_null.py index 26eddef6..5c29dc68 100644 --- a/pytablewriter/writer/_null.py +++ b/pytablewriter/writer/_null.py @@ -10,6 +10,14 @@ class NullTableWriter(IndentationInterface, TextWriterInterface, TableWriterInterface): FORMAT_NAME = "null" + def __init__(self, **kwargs) -> None: + self.table_name = kwargs.get("table_name", "") + self.value_matrix = kwargs.get("value_matrix", []) + self.is_formatting_float = kwargs.get("is_formatting_float", True) + self.headers = kwargs.get("headers", []) + self.type_hints = kwargs.get("type_hints", []) + self.max_workers = kwargs.get("max_workers", 1) + def __repr__(self) -> str: return self.dumps() diff --git a/pytablewriter/writer/binary/_interface.py b/pytablewriter/writer/binary/_interface.py index 4f33a74a..db3fac6c 100644 --- a/pytablewriter/writer/binary/_interface.py +++ b/pytablewriter/writer/binary/_interface.py @@ -32,6 +32,8 @@ def stream(self, value) -> None: def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + self.table_name = kwargs.get("table_name", "") + self._stream = None def dumps(self) -> str: diff --git a/pytablewriter/writer/text/_rst.py b/pytablewriter/writer/text/_rst.py index b08a45b4..c621836d 100644 --- a/pytablewriter/writer/text/_rst.py +++ b/pytablewriter/writer/text/_rst.py @@ -19,7 +19,6 @@ class RstTableWriter(IndentationTextTableWriter): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.table_name = "" self.char_header_row_separator = "=" self.char_cross_point = "+" self.char_left_cross_point = "+" diff --git a/pytablewriter/writer/text/_spacealigned.py b/pytablewriter/writer/text/_spacealigned.py index 09cddd00..d6a6608c 100644 --- a/pytablewriter/writer/text/_spacealigned.py +++ b/pytablewriter/writer/text/_spacealigned.py @@ -33,6 +33,6 @@ def __init__(self, **kwargs) -> None: self.char_cross_point = " " self.is_padding = True - self.is_formatting_float = True + self.is_formatting_float = kwargs.get("is_formatting_float", True) self._quoting_flags = copy.deepcopy(dataproperty.NOT_QUOTING_FLAGS) diff --git a/pytablewriter/writer/text/_unicode.py b/pytablewriter/writer/text/_unicode.py index 0eb1dfd8..3607ec57 100644 --- a/pytablewriter/writer/text/_unicode.py +++ b/pytablewriter/writer/text/_unicode.py @@ -26,8 +26,6 @@ def support_split_write(self) -> bool: def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.table_name = "" - self.column_delimiter = "│" self.char_left_side_row = "│" self.char_right_side_row = "│" diff --git a/pytablewriter/writer/text/sourcecode/_pandas.py b/pytablewriter/writer/text/sourcecode/_pandas.py index 8f7f2296..b9d52b16 100644 --- a/pytablewriter/writer/text/sourcecode/_pandas.py +++ b/pytablewriter/writer/text/sourcecode/_pandas.py @@ -61,7 +61,6 @@ def format_name(self) -> str: def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.table_name = "" self.import_pandas_as = "pd" self.is_write_header = False diff --git a/pytablewriter/writer/text/sourcecode/_python.py b/pytablewriter/writer/text/sourcecode/_python.py index 1cb9fcb9..3c1955d3 100644 --- a/pytablewriter/writer/text/sourcecode/_python.py +++ b/pytablewriter/writer/text/sourcecode/_python.py @@ -52,7 +52,6 @@ def support_split_write(self) -> bool: def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.table_name = "" self._dp_extractor.type_value_map = { typepy.Typecode.NONE: None, typepy.Typecode.INFINITY: 'float("inf")', From dbf1b6f77f6384c3f2ab3437d7d103e9ef59caf0 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 18:12:33 +0900 Subject: [PATCH 06/13] Add support for keyword arguments initialization to TableWriterFactory instantiation --- pytablewriter/_factory.py | 12 ++++++---- test/test_writer_factory.py | 44 +++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/pytablewriter/_factory.py b/pytablewriter/_factory.py index b9dbb238..0f3337c3 100644 --- a/pytablewriter/_factory.py +++ b/pytablewriter/_factory.py @@ -19,7 +19,7 @@ class TableWriterFactory: """ @classmethod - def create_from_file_extension(cls, file_extension: str) -> AbstractTableWriter: + def create_from_file_extension(cls, file_extension: str, **kwargs) -> AbstractTableWriter: """ Create a table writer class instance from a file extension. Supported file extensions are as follows: @@ -52,6 +52,8 @@ def create_from_file_extension(cls, file_extension: str) -> AbstractTableWriter: :param str file_extension: File extension string (case insensitive). + :param kwargs: + Keyword arguments that passing to writer class constructor. :return: Writer instance that coincides with the ``file_extension``. :rtype: @@ -77,7 +79,7 @@ def create_from_file_extension(cls, file_extension: str) -> AbstractTableWriter: logger.debug("create a {} instance".format(table_format.writer_class.__name__)) - return table_format.writer_class() + return table_format.writer_class(**kwargs) raise WriterNotFoundError( "\n".join( @@ -90,7 +92,7 @@ def create_from_file_extension(cls, file_extension: str) -> AbstractTableWriter: ) @classmethod - def create_from_format_name(cls, format_name: str) -> AbstractTableWriter: + def create_from_format_name(cls, format_name: str, **kwargs) -> AbstractTableWriter: """ Create a table writer class instance from a format name. Supported file format names are as follows: @@ -126,6 +128,8 @@ def create_from_format_name(cls, format_name: str) -> AbstractTableWriter: ============================================= =================================== :param str format_name: Format name string (case insensitive). + :param kwargs: + Keyword arguments that passing to writer class constructor. :return: Writer instance that coincides with the ``format_name``: :rtype: :py:class:`~pytablewriter.writer._table_writer.TableWriterInterface` @@ -141,7 +145,7 @@ def create_from_format_name(cls, format_name: str) -> AbstractTableWriter: ): logger.debug("create a {} instance".format(table_format.writer_class.__name__)) - return table_format.writer_class() + return table_format.writer_class(**kwargs) raise WriterNotFoundError( "\n".join( diff --git a/test/test_writer_factory.py b/test/test_writer_factory.py index 179ded5c..6d46530f 100644 --- a/test/test_writer_factory.py +++ b/test/test_writer_factory.py @@ -3,10 +3,12 @@ """ import itertools +import sys import pytest import pytablewriter as ptw +from pytablewriter.typehint import Integer class Test_WriterFactory_get_format_names: @@ -170,9 +172,28 @@ class Test_WriterFactory_create_from_file_extension: ), ) def test_normal(self, value, expected): - writer = ptw.TableWriterFactory.create_from_file_extension(value) + table_name = "dummy" + headers = ["a", "b"] + value_matrix = [[1, 2]] + type_hints = [Integer, Integer] + is_formatting_float = False + writer = ptw.TableWriterFactory.create_from_file_extension( + value, + table_name=table_name, + headers=headers, + value_matrix=value_matrix, + type_hints=type_hints, + is_formatting_float=is_formatting_float, + ) + + print(type(writer), file=sys.stderr) assert isinstance(writer, expected) + assert writer.table_name == table_name + assert writer.headers == headers + assert writer.value_matrix == value_matrix + assert writer.type_hints == type_hints + assert writer.is_formatting_float == is_formatting_float @pytest.mark.parametrize( ["value", "expected"], @@ -242,9 +263,28 @@ class Test_FileLoaderFactory_create_from_format_name: ], ) def test_normal(self, format_name, expected): - writer = ptw.TableWriterFactory.create_from_format_name(format_name) + table_name = "dummy" + headers = ["a", "b"] + value_matrix = [[1, 2]] + type_hints = [Integer, Integer] + is_formatting_float = False + + writer = ptw.TableWriterFactory.create_from_format_name( + format_name, + table_name=table_name, + headers=headers, + value_matrix=value_matrix, + type_hints=type_hints, + is_formatting_float=is_formatting_float, + ) + print(format_name, type(writer), file=sys.stderr) assert isinstance(writer, expected) + assert writer.table_name == table_name + assert writer.headers == headers + assert writer.value_matrix == value_matrix + assert writer.type_hints == type_hints + assert writer.is_formatting_float == is_formatting_float @pytest.mark.parametrize( ["format_name", "expected"], From f4438ebd4df13190b868bf19ffaa312e56cb74c7 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 22:30:48 +0900 Subject: [PATCH 07/13] Extract methods to the base class --- pytablewriter/writer/binary/_excel.py | 7 ------- pytablewriter/writer/binary/_interface.py | 13 +++++++++++++ pytablewriter/writer/binary/_sqlite.py | 13 ------------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/pytablewriter/writer/binary/_excel.py b/pytablewriter/writer/binary/_excel.py index 00061542..6052b21e 100644 --- a/pytablewriter/writer/binary/_excel.py +++ b/pytablewriter/writer/binary/_excel.py @@ -25,10 +25,6 @@ class ExcelTableWriter(AbstractBinaryTableWriter, metaclass=abc.ABCMeta): def format_name(self) -> str: return self.FORMAT_NAME - @property - def support_split_write(self) -> bool: - return True - @property def workbook(self) -> Optional[ExcelWorkbookInterface]: return self._workbook @@ -221,9 +217,6 @@ def _write_table(self, **kwargs) -> None: self._write_value_matrix() self._postprocess() - def _write_value_row_separator(self) -> None: - pass - def _write_value_matrix(self) -> None: for value_dp_list in self._table_value_dp_matrix: for col_idx, value_dp in enumerate(value_dp_list): diff --git a/pytablewriter/writer/binary/_interface.py b/pytablewriter/writer/binary/_interface.py index db3fac6c..33b47b6d 100644 --- a/pytablewriter/writer/binary/_interface.py +++ b/pytablewriter/writer/binary/_interface.py @@ -29,6 +29,10 @@ def stream(self, value) -> None: "cannot assign a stream to binary format writers. use open method instead." ) + @property + def support_split_write(self) -> bool: + return True + def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -36,9 +40,18 @@ def __init__(self, **kwargs) -> None: self._stream = None + def __del__(self) -> None: + self.close() + + def is_opened(self) -> bool: + return self.stream is not None + def dumps(self) -> str: raise NotImplementedError("binary format writers did not support dumps method") def _verify_stream(self) -> None: if self.stream is None: raise OSError("null output stream. required to open(file_path) first.") + + def _write_value_row_separator(self) -> None: + pass diff --git a/pytablewriter/writer/binary/_sqlite.py b/pytablewriter/writer/binary/_sqlite.py index b3940138..88d2e4c5 100644 --- a/pytablewriter/writer/binary/_sqlite.py +++ b/pytablewriter/writer/binary/_sqlite.py @@ -26,10 +26,6 @@ class SqliteTableWriter(AbstractBinaryTableWriter): def format_name(self) -> str: return self.FORMAT_NAME - @property - def support_split_write(self) -> bool: - return True - def __init__(self, **kwargs) -> None: import copy @@ -46,12 +42,6 @@ def __init__(self, **kwargs) -> None: self._quoting_flags = copy.deepcopy(dataproperty.NOT_QUOTING_FLAGS) - def __del__(self) -> None: - self.close() - - def is_opened(self) -> bool: - return self.stream is not None - def open(self, file_path: str) -> None: """ Open a SQLite database file. @@ -109,6 +99,3 @@ def _write_table(self, **kwargs) -> None: max_workers=self.max_workers, ) self.stream.create_table_from_tabledata(table_data) - - def _write_value_row_separator(self) -> None: - pass From 2cce8cb735b13089679b635f449a4e7621a4bea1 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 23:04:18 +0900 Subject: [PATCH 08/13] Fix a test --- test/writer/text/test_numpy_writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/writer/text/test_numpy_writer.py b/test/writer/text/test_numpy_writer.py index cb893dc0..654c99b0 100644 --- a/test/writer/text/test_numpy_writer.py +++ b/test/writer/text/test_numpy_writer.py @@ -151,14 +151,14 @@ def test_normal(self, capsys, table, indent, header, value, expected): for data in null_test_data_list ], ) - def test_exception(self, table, indent, header, value, expected): + def test_exception_null(self, table, indent, header, value, expected): writer = table_writer_class() writer.table_name = table writer.set_indent_level(indent) writer.headers = header writer.value_matrix = value - assert writer.write_table() == "" + assert writer.dumps() == "" @pytest.mark.parametrize( ["table", "indent", "header", "value", "expected"], From fab27cca81852887e87be9b8856b6cc5e19320e6 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 23:09:48 +0900 Subject: [PATCH 09/13] Refactor a test case --- test/writer/text/test_multibyte.py | 36 +++++++++++++----------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/test/writer/text/test_multibyte.py b/test/writer/text/test_multibyte.py index b9f5b119..068de80c 100644 --- a/test/writer/text/test_multibyte.py +++ b/test/writer/text/test_multibyte.py @@ -10,26 +10,20 @@ class Test_CsvTableWriter_write_table: @pytest.mark.parametrize( ["format_name"], - [ - [format_name] - for format_name in ptw.TableWriterFactory.get_format_names() - if format_name not in ["null", "elasticsearch", "excel", "sqlite"] - ], + [[tblfmt.names[0]] for tblfmt in ptw.TableFormat.find_all_attr(ptw.FormatAttr.TEXT)], ) - def test_smoke_multi_byte(self, capsys, format_name): - writer = ptw.TableWriterFactory.create_from_format_name(format_name) - writer.table_name = "生成に関するパターン" - writer.headers = ["パターン名", "概要", "GoF", "Code Complete[1]"] - writer.value_matrix = [ - ["Abstract Factory", "関連する一連のインスタンスを状況に応じて、適切に生成する方法を提供する。", "Yes", "Yes"], - ["Builder", "複合化されたインスタンスの生成過程を隠蔽する。", "Yes", "No"], - ["Factory Method", "実際に生成されるインスタンスに依存しない、インスタンスの生成方法を提供する。", "Yes", "Yes"], - ["Prototype", "同様のインスタンスを生成するために、原型のインスタンスを複製する。", "Yes", "No"], - ["Singleton", "あるクラスについて、インスタンスが単一であることを保証する。", "Yes", "Yes"], - ] + def test_smoke_multi_byte(self, format_name): + writer = ptw.TableWriterFactory.create_from_format_name( + format_name, + table_name="生成に関するパターン", + headers=["パターン名", "概要", "GoF", "Code Complete[1]"], + value_matrix=[ + ["Abstract Factory", "関連する一連のインスタンスを状況に応じて、適切に生成する方法を提供する。", "Yes", "Yes"], + ["Builder", "複合化されたインスタンスの生成過程を隠蔽する。", "Yes", "No"], + ["Factory Method", "実際に生成されるインスタンスに依存しない、インスタンスの生成方法を提供する。", "Yes", "Yes"], + ["Prototype", "同様のインスタンスを生成するために、原型のインスタンスを複製する。", "Yes", "No"], + ["Singleton", "あるクラスについて、インスタンスが単一であることを保証する。", "Yes", "Yes"], + ], + ) - writer.write_table() - - out, _err = capsys.readouterr() - - assert len(out) > 100 + assert len(writer.dumps()) > 100 From 6b232a9deea3ec6e04e7bbde698d071bf32712cb Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 23:11:45 +0900 Subject: [PATCH 10/13] Add PandasDataFramePickleWriter class --- pytablewriter/__init__.py | 1 + pytablewriter/_table_format.py | 7 ++ pytablewriter/writer/__init__.py | 7 +- pytablewriter/writer/binary/__init__.py | 1 + pytablewriter/writer/binary/_pandas.py | 96 ++++++++++++++ test/test_table_format.py | 10 +- test/test_writer_factory.py | 2 + test/writer/binary/test_pandas_writer.py | 151 +++++++++++++++++++++++ 8 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 pytablewriter/writer/binary/_pandas.py create mode 100644 test/writer/binary/test_pandas_writer.py diff --git a/pytablewriter/__init__.py b/pytablewriter/__init__.py index b07e0d60..267f2b07 100644 --- a/pytablewriter/__init__.py +++ b/pytablewriter/__init__.py @@ -51,6 +51,7 @@ MediaWikiTableWriter, NullTableWriter, NumpyTableWriter, + PandasDataFramePickleWriter, PandasDataFrameWriter, PythonCodeTableWriter, RstCsvTableWriter, diff --git a/pytablewriter/_table_format.py b/pytablewriter/_table_format.py index bcb5a1ff..b76c1a80 100644 --- a/pytablewriter/_table_format.py +++ b/pytablewriter/_table_format.py @@ -24,6 +24,7 @@ MediaWikiTableWriter, NullTableWriter, NumpyTableWriter, + PandasDataFramePickleWriter, PandasDataFrameWriter, PythonCodeTableWriter, RstCsvTableWriter, @@ -168,6 +169,12 @@ class TableFormat(enum.Enum): FormatAttr.FILE | FormatAttr.TEXT | FormatAttr.SOURCECODE | FormatAttr.SECONDARY_EXT, ["py"], ) + PANDAS_PICKLE = ( + [PandasDataFramePickleWriter.FORMAT_NAME], # type: ignore + PandasDataFramePickleWriter, + FormatAttr.FILE | FormatAttr.BIN, + [], + ) PYTHON = ( [PythonCodeTableWriter.FORMAT_NAME, "py"], PythonCodeTableWriter, diff --git a/pytablewriter/writer/__init__.py b/pytablewriter/writer/__init__.py index 4663fd41..2ad46138 100644 --- a/pytablewriter/writer/__init__.py +++ b/pytablewriter/writer/__init__.py @@ -1,7 +1,12 @@ from ._elasticsearch import ElasticsearchWriter from ._null import NullTableWriter from ._table_writer import AbstractTableWriter -from .binary import ExcelXlsTableWriter, ExcelXlsxTableWriter, SqliteTableWriter +from .binary import ( + ExcelXlsTableWriter, + ExcelXlsxTableWriter, + PandasDataFramePickleWriter, + SqliteTableWriter, +) from .text import ( BoldUnicodeTableWriter, BorderlessTableWriter, diff --git a/pytablewriter/writer/binary/__init__.py b/pytablewriter/writer/binary/__init__.py index 7e705f90..7c285afc 100644 --- a/pytablewriter/writer/binary/__init__.py +++ b/pytablewriter/writer/binary/__init__.py @@ -1,2 +1,3 @@ from ._excel import ExcelXlsTableWriter, ExcelXlsxTableWriter +from ._pandas import PandasDataFramePickleWriter from ._sqlite import SqliteTableWriter diff --git a/pytablewriter/writer/binary/_pandas.py b/pytablewriter/writer/binary/_pandas.py new file mode 100644 index 00000000..91ddb90b --- /dev/null +++ b/pytablewriter/writer/binary/_pandas.py @@ -0,0 +1,96 @@ +from typing import Optional # noqa + +import tabledata + +from ...error import EmptyValueError +from ._interface import AbstractBinaryTableWriter + + +class PandasDataFramePickleWriter(AbstractBinaryTableWriter): + """ + A table writer class for pandas DataFrame pickle. + + .. py:method:: write_table() + + Write a table to a pandas DataFrame pickle file. + """ + + FORMAT_NAME = "pandas_pickle" + + @property + def format_name(self) -> str: + return self.FORMAT_NAME + + @property + def support_split_write(self) -> bool: + return False + + def __init__(self, **kwargs) -> None: + import copy + + import dataproperty + + super().__init__(**kwargs) + + self.is_padding = False + self.is_formatting_float = False + self._use_default_header = True + + self._quoting_flags = copy.deepcopy(dataproperty.NOT_QUOTING_FLAGS) + + self.__filepath = None # type: Optional[str] + + def is_opened(self) -> bool: + return self.__filepath is not None + + def open(self, file_path: str) -> None: + self.__filepath = file_path + + def close(self) -> None: + super().close() + self.__filepath = None + + def dump(self, output: str, close_after_write: bool = True, **kwargs) -> None: + """Write data to a DataFrame pickle file. + + Args: + output (file descriptor or filepath): + """ + + self.open(output) + try: + self.write_table(**kwargs) + finally: + if close_after_write: + self.close() + + def _verify_stream(self) -> None: + pass + + def _write_table(self, **kwargs) -> None: + if not self.is_opened(): + self._logger.logger.error("required to open(file_path) first.") + return + + try: + self._verify_value_matrix() + except EmptyValueError: + self._logger.logger.debug("no tabular data found") + return + + self._preprocess() + + table_data = tabledata.TableData( + self.table_name, + self.headers, + [ + [value_dp.data for value_dp in value_dp_list] + for value_dp_list in self._table_value_dp_matrix + ], + type_hints=self.type_hints, + max_workers=self.max_workers, + ) + table_data.as_dataframe().to_pickle(self.__filepath) + + def _write_table_iter(self, **kwargs) -> None: + self._write_table(**kwargs) diff --git a/test/test_table_format.py b/test/test_table_format.py index 73c2cf73..05e68caa 100644 --- a/test/test_table_format.py +++ b/test/test_table_format.py @@ -40,7 +40,15 @@ class Test_TableFormat_search_table_format: TableFormat.YAML, ], ], - [FormatAttr.BIN, [TableFormat.EXCEL_XLS, TableFormat.EXCEL_XLSX, TableFormat.SQLITE]], + [ + FormatAttr.BIN, + [ + TableFormat.EXCEL_XLS, + TableFormat.EXCEL_XLSX, + TableFormat.SQLITE, + TableFormat.PANDAS_PICKLE, + ], + ], [FormatAttr.API, [TableFormat.ELASTICSEARCH]], [0, []], ], diff --git a/test/test_writer_factory.py b/test/test_writer_factory.py index 6d46530f..5dbe8143 100644 --- a/test/test_writer_factory.py +++ b/test/test_writer_factory.py @@ -38,6 +38,7 @@ def test_normal(self): "null", "numpy", "pandas", + "pandas_pickle", "py", "python", "rst", @@ -242,6 +243,7 @@ class Test_FileLoaderFactory_create_from_format_name: ["NULL", ptw.NullTableWriter], ["numpy", ptw.NumpyTableWriter], ["pandas", ptw.PandasDataFrameWriter], + ["pandas_pickle", ptw.PandasDataFramePickleWriter], ["py", ptw.PythonCodeTableWriter], ["Python", ptw.PythonCodeTableWriter], ["rst", ptw.RstGridTableWriter], diff --git a/test/writer/binary/test_pandas_writer.py b/test/writer/binary/test_pandas_writer.py new file mode 100644 index 00000000..e0e74dcc --- /dev/null +++ b/test/writer/binary/test_pandas_writer.py @@ -0,0 +1,151 @@ +""" +.. codeauthor:: Tsuyoshi Hombashi +""" + +import collections +import os +import sys +from decimal import Decimal + +import pytest +from tabledata import TableData + +import pytablewriter as ptw + +from ..._common import print_test_result +from ...data import headers, value_matrix + + +try: + import pandas as pd + + SKIP_DATAFRAME_TEST = False +except ImportError: + SKIP_DATAFRAME_TEST = True + + +inf = Decimal("Infinity") +nan = None + +Data = collections.namedtuple("Data", "table header value expected") + +normal_test_data_list = [ + Data( + table="tablename", + header=headers, + value=value_matrix, + expected=TableData( + "tablename", + ["a", "b", "c", "dd", "e"], + [[1, 123.1, "a", 1, 1], [2, 2.2, "bb", 2.2, 2.2], [3, 3.3, "ccc", 3, "cccc"]], + ), + ), + Data( + table="empty header", + header=[], + value=value_matrix, + expected=TableData( + "empty_header", + ["A", "B", "C", "D", "E"], + [[1, 123.1, "a", 1, 1], [2, 2.2, "bb", 2.2, 2.2], [3, 3.3, "ccc", 3, "cccc"]], + ), + ), +] + +empty_test_data_list = [ + Data(table="dummy", header=[], value=[], expected=None), + Data(table="dummy", header=headers, value=[], expected=None), +] + + +@pytest.mark.skipif(SKIP_DATAFRAME_TEST, reason="required package not found") +class Test_PandasDataFramePickleWriter_write_table: + @pytest.mark.parametrize( + ["table", "header", "value", "expected"], + [[data.table, data.header, data.value, data.expected] for data in normal_test_data_list], + ) + def test_normal(self, tmpdir, table, header, value, expected): + test_filepath = tmpdir.join("test1.pkl") + + writer = ptw.PandasDataFramePickleWriter( + table_name=table, + headers=header, + value_matrix=value, + ) + writer.open(str(test_filepath)) + writer.write_table() + + print(expected, file=sys.stderr) + + actual = ptw.PandasDataFramePickleWriter() + actual.from_dataframe(pd.read_pickle(test_filepath)) + actual.table_name = expected.table_name + + print_test_result( + expected=ptw.dumps_tabledata(expected), actual=ptw.dumps_tabledata(actual.tabledata) + ) + + assert ptw.dumps_tabledata(actual.tabledata) == ptw.dumps_tabledata(expected) + + @pytest.mark.parametrize( + ["table", "header", "value", "expected"], + [[data.table, data.header, data.value, data.expected] for data in empty_test_data_list], + ) + def test_smoke_empty(self, tmpdir, table, header, value, expected): + test_filepath = str(tmpdir.join("empty.pkl")) + writer = ptw.PandasDataFramePickleWriter( + table_name=table, headers=header, value_matrix=value + ) + writer.open(test_filepath) + writer.write_table() + assert not os.path.isfile(test_filepath) + + def test_exception(self): + writer = ptw.PandasDataFramePickleWriter( + table_name="tablename", + headers=["ha", "hb", "hc"], + value_matrix=[[1.0, 2.0, 3.0], [11.0, 12.0, 13.0], [1.0, 2.0, 3.0]], + ) + + writer.write_table() + + +@pytest.mark.skipif(SKIP_DATAFRAME_TEST, reason="required package not found") +class Test_PandasDataFramePickleWriter_dump: + def test_normal_single_table(self, tmpdir): + test_filepath = str(tmpdir.join("test.pkl")) + data = TableData( + "tablename", ["ha", "hb", "hc"], [[1.0, 2.0, 3.0], [11.0, 12.0, 13.0], [1.0, 2.0, 3.0]] + ) + + writer = ptw.PandasDataFramePickleWriter() + writer.from_tabledata(data) + writer.dump(test_filepath) + + actual = ptw.PandasDataFramePickleWriter() + actual.from_dataframe(pd.read_pickle(test_filepath)) + actual.table_name = data.table_name + + assert actual.tabledata == data + + def test_normal_multi_table(self, tmpdir): + test_filepath = str(tmpdir.join("test.pkl")) + data = TableData("first", ["ha1", "hb1", "hc1"], [[1.0, 2.0, 3.0], [11.0, 12.0, 13.0]]) + writer = ptw.PandasDataFramePickleWriter() + + writer.from_tabledata(data) + writer.dump(test_filepath, close_after_write=False) + + actual = ptw.PandasDataFramePickleWriter() + actual.from_dataframe(pd.read_pickle(test_filepath)) + actual.table_name = data.table_name + + assert actual.tabledata == data + + +class Test_PandasDataFramePickleWriter_dumps: + def test_exception(self, tmpdir): + writer = ptw.PandasDataFramePickleWriter() + + with pytest.raises(NotImplementedError): + writer.dumps() From dd7d451d707c965c7ee1f1f4f82526ed899592eb Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 23:18:25 +0900 Subject: [PATCH 11/13] Update docs --- docs/pages/introduction/feature.txt | 1 + docs/pages/reference/writers/binary/pandas_pickle.rst | 6 ++++++ docs/pages/reference/writers/index.rst | 1 + 3 files changed, 8 insertions(+) create mode 100644 docs/pages/reference/writers/binary/pandas_pickle.rst diff --git a/docs/pages/introduction/feature.txt b/docs/pages/introduction/feature.txt index 24b771ff..e23f830d 100644 --- a/docs/pages/introduction/feature.txt +++ b/docs/pages/introduction/feature.txt @@ -18,6 +18,7 @@ Features - Unicode - Binary file formats: - Microsoft Excel :superscript:`TM` (``.xlsx``/``.xls`` file format) + - `pandas.DataFrame `__ pickle file - SQLite database - Application specific formats: - `Elasticsearch `__ diff --git a/docs/pages/reference/writers/binary/pandas_pickle.rst b/docs/pages/reference/writers/binary/pandas_pickle.rst new file mode 100644 index 00000000..59b7f28b --- /dev/null +++ b/docs/pages/reference/writers/binary/pandas_pickle.rst @@ -0,0 +1,6 @@ +Pandas DataFrame writer class +------------------------------- +.. autoclass:: pytablewriter.PandasDataFramePickleWriter + :members: + :inherited-members: + :show-inheritance: diff --git a/docs/pages/reference/writers/index.rst b/docs/pages/reference/writers/index.rst index edc0218f..f496b9e3 100644 --- a/docs/pages/reference/writers/index.rst +++ b/docs/pages/reference/writers/index.rst @@ -22,6 +22,7 @@ Binary formats ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. toctree:: binary/excel + binary/pandas_pickle binary/sqlite From 6c1c2f585e9bd9c5c19a840f5066e4f19e2955f1 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 23:18:38 +0900 Subject: [PATCH 12/13] Update README --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 361362b1..567eb671 100644 --- a/README.rst +++ b/README.rst @@ -54,6 +54,7 @@ Features - Unicode - Binary file formats: - Microsoft Excel :superscript:`TM` (``.xlsx``/``.xls`` file format) + - `pandas.DataFrame `__ pickle file - SQLite database - Application specific formats: - `Elasticsearch `__ From 6103e1c91d41cea1f95c30487332dc1e82e694ec Mon Sep 17 00:00:00 2001 From: Tsuyoshi Hombashi Date: Sun, 30 Aug 2020 23:25:48 +0900 Subject: [PATCH 13/13] Bump version --- pytablewriter/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytablewriter/__version__.py b/pytablewriter/__version__.py index 3c659883..65f777e7 100644 --- a/pytablewriter/__version__.py +++ b/pytablewriter/__version__.py @@ -1,6 +1,6 @@ __author__ = "Tsuyoshi Hombashi" __copyright__ = "Copyright 2016, {}".format(__author__) __license__ = "MIT License" -__version__ = "0.57.0" +__version__ = "0.58.0" __maintainer__ = __author__ __email__ = "tsuyoshi.hombashi@gmail.com"