From c27d0c3cb18444278e30de00620ea0729e6d7c27 Mon Sep 17 00:00:00 2001 From: zR <2448370773@qq.com> Date: Wed, 3 Jul 2024 15:19:37 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E5=90=88?= =?UTF-8?q?=EF=BC=8C=E6=94=AF=E6=8C=81=E8=87=AA=E5=AE=9A=E4=B9=89=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 67 ++++- README_CN.md | 93 ++++--- gptpdf/parse.py | 276 ++++++++------------ test_output/output.md | 573 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 794 insertions(+), 215 deletions(-) create mode 100644 test_output/output.md diff --git a/README.md b/README.md index 61c90cf..7f803e1 100644 --- a/README.md +++ b/README.md @@ -53,26 +53,67 @@ print(content) See more in [test/test.py](test/test.py) - ## API -**parse_pdf**(pdf_path, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False) +### parse_pdf + +**Function**: `parse_pdf(pdf_path, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False, gpt_worker=1)` + +Parses a PDF file into a Markdown file and returns the Markdown content along with all image paths. + +**Parameters**: + +- **pdf_path**: *str* + Path to the PDF file + +- **output_dir**: *str*, default: './' + Output directory to store all images and the Markdown file -parse pdf file to markdown file, and return markdown content and all image paths. +- **api_key**: *Optional[str]*, optional + OpenAI API key. If not provided, the `OPENAI_API_KEY` environment variable will be used. -- **pdf_path**: pdf file path +- **base_url**: *Optional[str]*, optional + OpenAI base URL. If not provided, the `OPENAI_BASE_URL` environment variable will be used. This can be modified to call other large model services with OpenAI API interfaces, such as `GLM-4V`. -- **output_dir**: output directory. store all images and markdown file +- **model**: *str*, default: 'gpt-4o' + OpenAI API formatted multimodal large model. If you need to use other models, such as: + - [qwen-vl-max](https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start) (untested) + - [GLM-4V](https://open.bigmodel.cn/dev/api#glm-4v) (tested) + - Azure OpenAI, by setting the `base_url` to `https://xxxx.openai.azure.com/` to use Azure OpenAI, where `api_key` is the Azure API key, and the model is similar to `azure_xxxx`, where `xxxx` is the deployed model name (tested). -- **api_key**: OpenAI API Key (optional). If not provided, Use OPENAI_API_KEY environment variable. +- **verbose**: *bool*, default: False + Verbose mode. When enabled, the content parsed by the large model will be displayed in the command line. -- **base_url**: OpenAI Base URL. (optional). If not provided, Use OPENAI_BASE_URL environment variable. +- **gpt_worker**: *int*, default: 1 + Number of GPT parsing worker threads. If your machine has better performance, you can increase this value to speed up the parsing. -- **model**: OpenAI Vision Large Model, default is 'gpt-4o'. - You also can use [qwen-vl-max](https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start) (not tested yet) - [GLM-4V](https://open.bigmodel.cn/dev/api#glm-4v) by change the `OPENAI_BASE_URL` or specify `base_url`. - Also you can use Azure OpenAI by specify `base_url` to `https://xxxx.openai.azure.com/`, api_key is Azure API Key, model is like 'azure_xxxx' where xxxx is the deployed model name (not openai model name) +- **prompt**: *dict*, optional + If the model you are using does not match the default prompt provided in this repository and cannot achieve the best results, we support adding custom prompts. The prompts in the repository are divided into three parts: + - `prompt`: Mainly used to guide the model on how to process and convert text content in images. + - `rect_prompt`: Used to handle cases where specific areas (such as tables or images) are marked in the image. + - `role_prompt`: Defines the role of the model to ensure the model understands it is performing a PDF document parsing task. -- **verbose**: verbose mode + You can pass custom prompts in the form of a dictionary to replace any of the prompts. Here is an example: -- **gpt_worker**: gpt parse worker number. default is 1. If your machine performance is good, you can increase it appropriately to improve parsing speed. + ```python + prompt = { + "prompt": "Custom prompt text", + "rect_prompt": "Custom rect prompt", + "role_prompt": "Custom role prompt" + } + + content, image_paths = parse_pdf( + pdf_path=pdf_path, + output_dir='./output', + model="gpt-4o", + prompt=prompt, + verbose=False, + ) + +## Join Us 👏🏻 + +Scan the QR code below with WeChat to join our group chat or contribute. + +

+wechat +

diff --git a/README_CN.md b/README_CN.md index 25d14f3..23e7bc6 100644 --- a/README_CN.md +++ b/README_CN.md @@ -15,8 +15,6 @@ [pdfgpt-ui](https://github.com/daodao97/gptpdf-ui) 是一个基于 gptpdf 的可视化工具。 - - ## 处理流程 1. 使用 PyMuPDF 库，对 PDF 进行解析出所有非文本区域，并做好标记，比如: @@ -25,13 +23,10 @@ 2. 使用视觉大模型（如 GPT-4o）进行解析，得到 markdown 文件。 - - ## 样例 -有关 PDF，请参阅 [examples/attention_is_all_you_need/output.md](examples/attention_is_all_you_need/output.md) [examples/attention_is_all_you_need.pdf](examples/attention_is_all_you_need.pdf)。 - - +有关 +PDF，请参阅 [examples/attention_is_all_you_need/output.md](examples/attention_is_all_you_need/output.md) [examples/attention_is_all_you_need.pdf](examples/attention_is_all_you_need.pdf)。 ## 安装 @@ -39,12 +34,11 @@ pip install gptpdf ``` - - ## 使用 ```python from gptpdf import parse_pdf + api_key = 'Your OpenAI API Key' content, image_paths = parse_pdf(pdf_path, api_key=api_key) print(content) @@ -52,34 +46,63 @@ print(content) 更多内容请见 [test/test.py](test/test.py) - - ## API -**parse_pdf**(pdf_path, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False) - -将 pdf 文件解析为 markdown 文件，并返回 markdown 内容和所有图片路径列表。 - -- **pdf_path**：pdf 文件路径 - -- **output_dir**：输出目录。存储所有图片和 markdown 文件 - -- **api_key**：OpenAI API 密钥（可选）。如果未提供，则使用 OPENAI_API_KEY 环境变量。 - -- **base_url**：OpenAI 基本 URL。（可选）。如果未提供，则使用 OPENAI_BASE_URL 环境变量。 - -- **model**：OpenAI API格式的多模态大模型，默认为 “gpt-4o”。 - 如果您需要使用其他模型，例如 [qwen-vl-max](https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start) (尚未测试) - - [GLM-4V](https://open.bigmodel.cn/dev/api#glm-4v), 可以通过修改环境变量 `OPENAI_BASE_URL` 或 指定API参数 `base_url` 来使用。 (已经测试) - - 您也可以通过将 `base_url` 指定为 `https://xxxx.openai.azure.com/` 来使用 Azure OpenAI，api_key 是 Azure API 密钥，模型类似于 'azure_xxxx'，其中 xxxx 是部署的模型名称（不是 openai 模型名称）(已经测试) - -- **verbose**：详细模式 - -- **gpt_worker**: gpt解析工作线程数，默认为1. 如果您的机器性能较好，可以适当调高，以提高解析速度。 - - +### parse_pdf + +**函数 +**：`parse_pdf(pdf_path, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False, gpt_worker=1)` + +将 PDF 文件解析为 Markdown 文件，并返回 Markdown 内容和所有图片路径列表。 + +**参数**： + +- **pdf_path**：*str* + PDF 文件路径 + +- **output_dir**：*str*，默认值：'./' + 输出目录，存储所有图片和 Markdown 文件 + +- **api_key**：*Optional[str]*，可选 + OpenAI API 密钥。如果未提供，则使用 `OPENAI_API_KEY` 环境变量。 + +- **base_url**：*Optional[str]*，可选 + OpenAI 基本 URL。如果未提供，则使用 `OPENAI_BASE_URL` 环境变量。可以通过修改该环境变量调用 OpenAI API 类接口的其他大模型服务，例如`GLM-4V`。 + +- **model**：*str*，默认值：'gpt-4o'。OpenAI API 格式的多模态大模型。如果需要使用其他模型，例如 + - [qwen-vl-max](https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start)（尚未测试） + - [GLM-4V](https://open.bigmodel.cn/dev/api#glm-4v)（已测试） + - Azure OpenAI，通过将 `base_url` 指定为 `https://xxxx.openai.azure.com/` 来使用 Azure OpenAI，`api_key` 是 Azure API 密钥，模型类似于 `azure_xxxx`，其中 `xxxx` 是部署的模型名称（已测试）。 + +- **verbose**：*bool*，默认值：False，详细模式，开启后会在命令行显示大模型解析的内容。 + +- **gpt_worker**：*int*，默认值：1 + GPT 解析工作线程数。如果您的机器性能较好，可以适当调高，以提高解析速度。 + +- **prompt**: *dict*, 可选，如果您使用的模型与本仓库默认的提示词不匹配，无法发挥出最佳效果，我们支持自定义加入提示词。 + 仓库中，提示词分为三个部分，分别是： + + `prompt`：主要用于指导模型如何处理和转换图片中的文本内容。 + + `rect_prompt`：用于处理图片中标注了特定区域（例如表格或图片）的情况。 + + `role_prompt`：定义了模型的角色，确保模型理解它在执行PDF文档解析任务。 + 您可以用字典的形式传入自定义的提示词，实现对任意提示词的替换，这是一个例子： + + ```python + prompt = { + "prompt": "自定义提示词语", + "rect_prompt": "自定义提示词", + "role_prompt": "自定义提示词" + } + + content, image_paths = parse_pdf( + pdf_path=pdf_path, + output_dir='./output', + model="gpt-4o", + prompt="", + verbose=False, + ) + + ``` + 您不需要替换所有的提示词，如果您没有传入自定义提示词，仓库会自动使用默认的提示词。默认提示词使用的是中文，如果您的PDF文档是英文的，或者您的模型不支持中文，建议您自定义提示词。 ## 加入我们👏🏻 diff --git a/gptpdf/parse.py b/gptpdf/parse.py index 639672b..fa19cd2 100644 --- a/gptpdf/parse.py +++ b/gptpdf/parse.py @@ -1,46 +1,60 @@ +import os +from typing import List, Tuple, Optional +import logging + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') import fitz # PyMuPDF import shapely.geometry as sg +from shapely.geometry.base import BaseGeometry from shapely.validation import explain_validity import concurrent.futures +# This Default Prompt Using Chinese and could be changed to other languages. -def _is_near(rect1, rect2, distance=20): +DEFAULT_PROMPT = """使用markdown语法，将图片中识别到的文字转换为markdown格式输出。你必须做到： +1. 输出和使用识别到的图片的相同的语言，例如，识别到英语的字段，输出的内容必须是英语。 +2. 不要解释和输出无关的文字，直接输出图片中的内容。例如，严禁输出 “以下是我根据图片内容生成的markdown文本：”这样的例子，而是应该直接输出markdown。 +3. 内容不要包含在```markdown ```中、段落公式使用 $$ $$ 的形式、行内公式使用 $ $ 的形式、忽略掉长直线、忽略掉页码。 + +再次强调，不要解释和输出无关的文字，直接输出图片中的内容。 +""" +DEFAULT_RECT_PROMPT = """图片中用红色框和名称(%s)标注出了一些区域。 +如果区域是表格或者图片，使用 ![]() 的形式插入到输出内容中，否则直接输出文字内容。 +""" +DEFAULT_ROLE_PROMPT = """你是一个PDF文档解析器，使用markdown和latex语法输出图片的内容。 +""" + + +def _is_near(rect1: BaseGeometry, rect2: BaseGeometry, distance: float = 20) -> bool: """ - check if two rectangles are near each other if the distance between them is less than the target + Check if two rectangles are near each other if the distance between them is less than the target. """ - # 检测两个矩形的距离是否小于distance return rect1.buffer(1).distance(rect2.buffer(1)) < distance -def _is_horizontal_near(rect1, rect2, distance=100): +def _is_horizontal_near(rect1: BaseGeometry, rect2: BaseGeometry, distance: float = 100) -> bool: """ - check if two rectangles are near horizontally if one of them is a horizontal line + Check if two rectangles are near horizontally if one of them is a horizontal line. """ result = False - # rect1和rect2中有一个是水平线 if abs(rect1.bounds[3] - rect1.bounds[1]) < 0.1 or abs(rect2.bounds[3] - rect2.bounds[1]) < 0.1: if abs(rect1.bounds[0] - rect2.bounds[0]) < 0.1 and abs(rect1.bounds[2] - rect2.bounds[2]) < 0.1: - result = abs(rect1.bounds[3] - rect1.bounds[3]) < distance - # print(rect1.bounds, rect2.bounds, result) + result = abs(rect1.bounds[3] - rect2.bounds[3]) < distance return result -def _union_rects(rect1, rect2): +def _union_rects(rect1: BaseGeometry, rect2: BaseGeometry) -> BaseGeometry: """ - union two rectangles + Union two rectangles. """ - # 合并两个矩形 return sg.box(*(rect1.union(rect2).bounds)) -def _merge_rects(rect_list, distance=20, horizontal_distance=None): +def _merge_rects(rect_list: List[BaseGeometry], distance: float = 20, horizontal_distance: Optional[float] = None) -> \ + List[BaseGeometry]: """ - merge rectangles in the list if the distance between them is less than the target - :param rect_list: list of rectangles - :param distance: distance threshold - :param horizontal_distance: horizontal distance threshold when one of the rectangles is a horizontal line + Merge rectangles in the list if the distance between them is less than the target. """ - # 合并矩形列表 merged = True while merged: merged = False @@ -58,16 +72,10 @@ def _merge_rects(rect_list, distance=20, horizontal_distance=None): return rect_list -def _adsorb_rects_to_rects(source_rects, target_rects, distance=10): - """ - adsorb a set of rectangles to another set of rectangles - :param source_rects: source rectangle set - :param target_rects: target rectangle set +def _adsorb_rects_to_rects(source_rects: List[BaseGeometry], target_rects: List[BaseGeometry], distance: float = 10) -> \ + Tuple[List[BaseGeometry], List[BaseGeometry]]: """ - """ - 吸附一个集合到另外一个集合 - :param source_rects: 源矩形集合 - :param target_rects: 目标矩形集合 + Adsorb a set of rectangles to another set of rectangles. """ new_source_rects = [] for text_area_rect in source_rects: @@ -83,175 +91,115 @@ def _adsorb_rects_to_rects(source_rects, target_rects, distance=10): return target_rects, new_source_rects -def _parse_drawings(page): - """ - parse drawings in the page and merge adjacent rectangles - """ +def _parse_drawings(page: fitz.Page) -> List[Tuple[float, float, float, float]]: """ - 解析页面中的绘图元素，合并相邻的矩形 + Parse drawings in the page and merge adjacent rectangles. """ drawings = page.get_drawings() - - # for drawing in drawings: - # print(drawing) - rect_list = [drawing['rect'] for drawing in drawings] - # 转成 shapely 的 矩形对象 rect_list = [sg.box(rect[0], rect[1], rect[2], rect[3]) for rect in rect_list] merged_rects = _merge_rects(rect_list, distance=10, horizontal_distance=100) - - # 并删除无效的矩形 merged_rects = [rect for rect in merged_rects if explain_validity(rect) == 'Valid Geometry'] - - # 提取所有的字符串区域矩形框 - text_area_rects = [] - for text_area in page.get_text('dict')['blocks']: - rect = text_area['bbox'] - text_area_rects.append(sg.box(rect[0], rect[1], rect[2], rect[3])) - - # 吸附文字到矩形区域 + text_area_rects = [sg.box(text_area['bbox'][0], text_area['bbox'][1], text_area['bbox'][2], text_area['bbox'][3]) + for text_area in page.get_text('dict')['blocks']] merged_rects, text_area_rects = _adsorb_rects_to_rects(text_area_rects, merged_rects, distance=5) - - # 二次合并 merged_rects = _merge_rects(merged_rects, distance=10) - - # 过滤掉高度 或者 宽度 不足 10 的矩形 merged_rects = [rect for rect in merged_rects if rect.bounds[2] - rect.bounds[0] > 10 and rect.bounds[3] - rect.bounds[1] > 10] - - # 将Polygon对象抽取bounds属性 - merged_rects = [rect.bounds for rect in merged_rects] - - return merged_rects + return [rect.bounds for rect in merged_rects] -def _parse_images(page): +def _parse_images(page: fitz.Page) -> List[Tuple[float, float, float, float]]: """ - 解析页面中的图片元素 + Parse images in the page. """ images = page.get_image_info() return [image['bbox'] for image in images] -def _parse_tables(page): +def _parse_tables(page: fitz.Page) -> List[Tuple[float, float, float, float]]: """ - 解析页面中的表格元素 + Parse tables in the page. """ - tables = page.find_tables( - snap_tolerance=20, # 调整容差以捕捉更多的表格线条 - ) + tables = page.find_tables(snap_tolerance=20) return [table.bbox for table in tables] -def _parse_rects(page): - """ - parse rectangles in the page - :param page: page object +def _parse_rects(page: fitz.Page) -> List[Tuple[float, float, float, float]]: """ - """ - 解析页面中的矩形元素 + Parse rectangles in the page. """ return _parse_images(page) + _parse_drawings(page) -def _parse_pdf_to_images(pdf_path, output_dir='./'): +def _parse_pdf_to_images(pdf_path: str, output_dir: str = './') -> List[Tuple[str, List[str]]]: """ - parse pdf to images and save to output_dir - :param pdf_path: pdf file path - :param output_dir: output directory - :return: image_infos [(page_image, rect_images)] + Parse PDF to images and save to output_dir. """ - import os - # 打开PDF文件 pdf_document = fitz.open(pdf_path) - image_infos = [] - for page_index, page in enumerate(pdf_document): - print(f'parse page: {page_index}') - # 保存页面为图片 - page_image = page.get_pixmap(matrix=fitz.Matrix(3, 3)) + for page_index, page in enumerate(pdf_document): + logging.info(f'parse page: {page_index}') rect_images = [] - # 解析页面中的矩形 - # if page_index != 5: - # continue rects = _parse_rects(page) - # rects = _parse_tables(page) for index, rect in enumerate(rects): - # print(page_index, index, rect) fitz_rect = fitz.Rect(rect) pix = page.get_pixmap(clip=fitz_rect, matrix=fitz.Matrix(4, 4)) name = f'{page_index}_{index}.png' pix.save(os.path.join(output_dir, name)) - # 存储最简相对路径 rect_images.append(name) - - # 在页面上绘制红色矩形(膨胀一个像素) - # page.draw_rect(fitz_rect, color=(1, 0, 0), width=1) big_fitz_rect = fitz.Rect(fitz_rect.x0 - 1, fitz_rect.y0 - 1, fitz_rect.x1 + 1, fitz_rect.y1 + 1) page.draw_rect(big_fitz_rect, color=(1, 0, 0), width=1) - - # 在矩形内的左上角写上矩形的索引name，添加一些偏移量 - text_x = fitz_rect.x0 + 2 # 偏移量为2个单位 - text_y = fitz_rect.y0 + 10 # 偏移量为10个单位 - text_rect = fitz.Rect(text_x, text_y - 9, text_x + 80, text_y + 2) # 创建一个白色背景的矩形 - - # 绘制白色背景矩形 + text_x = fitz_rect.x0 + 2 + text_y = fitz_rect.y0 + 10 + text_rect = fitz.Rect(text_x, text_y - 9, text_x + 80, text_y + 2) page.draw_rect(text_rect, color=(1, 1, 1), fill=(1, 1, 1)) - - # 插入带有白色背景的文字 page.insert_text((text_x, text_y), name, fontsize=10, color=(1, 0, 0)) - - # 重新生成带有矩形的页面图像 page_image_with_rects = page.get_pixmap(matrix=fitz.Matrix(3, 3)) - # 整页使用绝对路径 page_image = os.path.join(output_dir, f'{page_index}.png') page_image_with_rects.save(page_image) - image_infos.append((page_image, rect_images)) - # 关闭PDF文件 pdf_document.close() return image_infos -def _gpt_parse_images(image_infos, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False, - gpt_worker=1): +def _gpt_parse_images( + image_infos: List[Tuple[str, List[str]]], + prompt: Optional[str], + output_dir: str = './', + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model: str = 'gpt-4o', + verbose: bool = False, + gpt_worker: int = 1 +) -> str: """ Parse images to markdown content. - - :param image_infos: List of tuples containing page images and rect images [(page_image, rect_images)]. - :param output_dir: Directory where the output will be saved. - :param api_key: OpenAI API Key used for authentication. - :param base_url: Base URL for the OpenAI API. - :param model: OpenAI Vision LLM Model to be used. - :param verbose: Boolean flag to indicate if verbose logging is enabled. - :param gpt_worker: Number of GPT worker threads to use. - :return: Parsed markdown content as a string. """ - import os from GeneralAgent import Agent - # TODO： 提示词优化，适配更多模型。 - prompt = """ -使用markdown语法，将图片中识别到的文字转换为markdown格式输出。你必须做到： -1. 输出和使用识别到的图片的相同的语言，例如，识别到英语的字段，输出的内容必须是英语。 -2. 不要解释和输出无关的文字，直接输出图片中的内容。例如，严禁输出 “以下是我根据图片内容生成的markdown文本：”这样的例子，而是应该直接输出markdown。 -3. 内容不要包含在```markdown ```中、段落公式使用 $$ $$ 的形式、行内公式使用 $ $ 的形式、忽略掉长直线、忽略掉页码。 - -再次强调，不要解释和输出无关的文字，直接输出图片中的内容。 -""" - # TODO： 提示词优化，适配更多模型。 - rect_prompt = """ -图片中用红色框和名称(%s)标注出了一些区域。 -如果区域是表格或者图片，使用 ![]() 的形式插入到输出内容中，否则直接输出文字内容。 -""" - - role = '你是一个PDF文档解析器，使用markdown和latex语法输出图片的内容。' - def _process_page(index, image_info): - print(f'gpt parse page: {index}') - agent = Agent(role=role, api_key=api_key, base_url=base_url, model=model, disable_python_run=True) + if prompt is None: + prompt = DEFAULT_PROMPT + logging.info("prompt is not provided, using default prompt.") + if isinstance(prompt, dict) and 'rect_prompt' in prompt: + rect_prompt = prompt['rect_prompt'] + logging.info("rect_prompt is provided, using user prompt.") + else: + rect_prompt = DEFAULT_RECT_PROMPT + logging.info("rect_prompt is not provided, using default prompt.") + if isinstance(prompt, dict) and 'role_prompt' in prompt: + role_prompt = prompt['role_prompt'] + logging.info("role_prompt is provided, using user prompt.") + else: + role_prompt = DEFAULT_ROLE_PROMPT + logging.info("role_prompt is not provided, using default prompt.") + + def _process_page(index: int, image_info: Tuple[str, List[str]]) -> Tuple[int, str]: + logging.info(f'gpt parse page: {index}') + agent = Agent(role=role_prompt, api_key=api_key, base_url=base_url, model=model, disable_python_run=True) page_image, rect_images = image_info local_prompt = prompt if rect_images: @@ -266,52 +214,46 @@ def _process_page(index, image_info): index, content = future.result() contents[index] = content - # 输出结果 output_path = os.path.join(output_dir, 'output.md') - with open(output_path, 'w',encoding='utf-8') as f: + with open(output_path, 'w', encoding='utf-8') as f: f.write('\n\n'.join(contents)) return '\n\n'.join(contents) -def parse_pdf(pdf_path, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False, gpt_worker=1): +def parse_pdf( + pdf_path: str, + output_dir: str = './', + prompt: Optional[str] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model: str = 'gpt-4o', + verbose: bool = False, + gpt_worker: int = 1 +) -> Tuple[str, List[str]]: """ Parse a PDF file to a markdown file. - - :param pdf_path: Path to the PDF file. - :param output_dir: Directory to store all images and the markdown file. - :param api_key: OpenAI API Key (optional). If not provided, the OPENAI_API_KEY environment variable will be used. - :param base_url: OpenAI Base URL (optional). If not provided, the OPENAI_BASE_URL environment variable will be used. - :param model: OpenAI Vision LLM Model, default is 'gpt-4o'. You can also use 'qwen-vl-max'. - :param verbose: Boolean flag to enable verbose mode. - :param gpt_worker: Number of GPT parse worker threads to use. - :return: Markdown content with image links (![](path/to/image.png)) and paths to all rect images (e.g., images, tables, charts). - """ """ - 解析PDF文件到markdown文件 - - :param pdf_path: pdf文件路径 - :param output_dir: 输出目录。存储所有的图片和markdown文件 - :param api_key: OpenAI API Key（可选）。如果未提供，则使用OPENAI_API_KEY环境变量。 - :param base_url: OpenAI Base URL（可选）。如果未提供，则使用OPENAI_BASE_URL环境变量。 - :param model: OpenAI Vision LLM Model，默认为'gpt-4o'。您还可以使用'qwen-vl-max'。 - :param verbose: 详细模式，默认为False。 - :param gpt_worker: gpt解析工作线程数，默认为1。 - :return: (content, all_rect_images)，markdown内容，带有![](path/to/image.png)和所有矩形图像（图像、表格、图表等）路径列表。 - """ - import os if not os.path.exists(output_dir): os.makedirs(output_dir) image_infos = _parse_pdf_to_images(pdf_path, output_dir=output_dir) - content = _gpt_parse_images(image_infos, output_dir=output_dir, api_key=api_key, base_url=base_url, model=model, - verbose=verbose, gpt_worker=gpt_worker) + content = _gpt_parse_images( + image_infos=image_infos, + output_dir=output_dir, + prompt=prompt, + api_key=api_key, + base_url=base_url, + model=model, + verbose=verbose, + gpt_worker=gpt_worker + ) - # 删除每页的图片 & 保留所有的矩形图片 all_rect_images = [] - for page_image, rect_images in image_infos: - if os.path.exists(page_image): - os.remove(page_image) - all_rect_images.extend(rect_images) - + # remove all rect images + if not verbose: + for page_image, rect_images in image_infos: + if os.path.exists(page_image): + os.remove(page_image) + all_rect_images.extend(rect_images) return content, all_rect_images diff --git a/test_output/output.md b/test_output/output.md new file mode 100644 index 0000000..6d04237 --- /dev/null +++ b/test_output/output.md @@ -0,0 +1,573 @@ +# Attention Is All You Need + +Ashish Vaswani +Google Brain +avaswani@google.com +& +Noam Shazeer +Google Brain +noam@google.com +& +Niki Parmar +Google Research +nikip@google.com +& +Jakob Uszkoreit +Google Research +uszko@google.com +& +Lion Jones +Google Research +lion@google.com +& +Aidan N. Gomez +University of Toronto +aidan@cs.toronto.edu +& +Lukasz Kaiser +Google Brain +lukaszkaiser@google.com +& +Ilia Polosukhin +illia.polosukhin@gmail.com + +Equal contribution. Listing order is random. Jakob proposed replacing RNNs with self-attention and started the effort to +evaluate this idea. Ashish, with Illia, designed and implemented the first Transformer models and has been crucially +involved in every aspect of this work. Noam proposed scaled dot-product attention, multi-head attention and the +parameter-free position representation and became the other person involved in nearly every detail. Niki designed, +implemented, tuned and evaluated countless model variants in our original codebase and tensor2tensor. Lion also +experimented with novel model variants, was responsible for our initial codebase, and efficient inference and +visualizations. Lukasz and Aidan spent countless long days designing various parts of and implementing tensor2tensor, +replacing our earlier codebase, greatly improving results and massively accelerating our research. + +Work performed while at Google Brain. +Work performed while at Google Research. + +[31st Conference on Neural Information Processing Systems (NIPS 2017), Long Beach, CA, USA.] + +###### Abstract + +Recurrent neural networks, long short-term memory [?] and gated recurrent [?] neural networks in particular, have been +firmly established as state of the art approaches in sequence modeling and transduction problems such as language +modeling and machine translation [?, ?, ?]. Numerous efforts have since continued to push the boundaries of recurrent +language models and encoder-decoder architectures [?, ?, ?]. + +Recurrent models typically factor computation along the symbol positions of the input and output sequences. Aligning the +positions to steps in computation time, they generate a sequence of hidden states\( h_{t}\), as a function of the +previous hidden state\( h_{t-1}\), and the input for position\( t\). This inherently sequential nature precludes +parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit +batching across examples. Recent work has achieved significant improvements in computational efficiency through +factorization tricks [?] and conditional computation [?], while also improving model performance in case of the latter. +The fundamental constraint of sequential computation, however, remains. + +Attention mechanisms have become an integral part of compelling sequence modeling and transduction models in various +tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences [?, ?]. In +all but a few cases [?], however, such attention mechanisms are used in conjunction with a recurrent network. + +In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an +attention mechanism to draw global dependencies between input and output. The Transformer allows for significantly more +parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve +hours on eight P100 GPUs. + +## 2 Background + +The goal of reducing sequential computation also forms the foundation of the Extended Neural GPU [?], ByteNet [?] and +ConvS2S [?], all of which use convolutional neural networks as basic building block, computing hidden representations in +parallel for all input and output positions. In these models, the number of operations required to relate signals from +two arbitrary input or output positions grows in the distance between positions, linearly for ConvS2S and +logarithmically for ByteNet. This makes it more difficult to learn dependencies between distant positions [?]. In the +Transformer this is reduced to a constant number of operations, albeit at the cost of reduced effective resolution due +to averaging attention-weighted operations, an effect we counteract with Multi-Head Attention as described in section +3.2. + +Self-attention, sometimes called intra-attention is an attention mechanism relating different positions of a single +sequence in order to compute a representation of the sequence. Self-attention has been used successfully in a variety of +tasks including reading comprehension, abstractive summarization, textual entailment and learning task-independent +sentence representations [?, ?, ?, ?]. + +End-to-end memory networks are based on a recurrent attention mechanism instead of sequence-aligned recurrence and have +been shown to perform well on simple-language question answering and language modeling tasks [?]. + +To the best of our knowledge, however, the Transformer is the first transduction model relying entirely on +self-attention to compute representations of its input and output without using sequence-aligned RNNs or convolution. In +the following sections, we will describe the Transformer, motivate self-attention and discuss its advantages over models +such as [?, ?] and [?]. + +## 3 Model Architecture + +Most competitive neural sequence transduction models have an encoder-decoder structure [?, ?, ?]. Here, the encoder maps +an input sequence of symbol representations\( x_{1},...,x_{n}\) to a sequence of continuous representations\( z = (z_ +{1},...,z_{n})\). Given\( z\), the decoder then generates an output sequence\( y_{1},...,y_{m}\) of symbols one element +at a time. At each step the model is auto-regressive [?], consuming the previously generated symbols as additional input +when generating the next + +### Figure 1: The Transformer - model architecture. + +The Transformer follows this overall architecture using stacked self-attention and point-wise, fully connected layers +for both the encoder and decoder, shown in the left and right halves of Figure 1, respectively. + +### Encoder and Decoder Stacks + +#### Encoder + +The encoder is composed of a stack of N = 6 identical layers. Each layer has two sub-layers. The first is a multi-head +self-attention mechanism, and the second is a simple, position-wise fully connected feed-forward network. We employ a +residual connection [1] around each of the two sub-layers, followed by layer normalization [1]. That is, the output of +each sub-layer is LayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer itself. To +facilitate these residual connections, all sub-layers in the model, as well as the embedding layers, produce outputs of +dimension d_model = 512. + +#### Decoder + +The decoder is also composed of a stack of N = 6 identical layers. In addition to the two sub-layers in each encoder +layer, the decoder inserts a third sub-layer, which performs multi-head attention over the output of the encoder stack. +Similar to the encoder, we employ residual connections around each of the sub-layers, followed by layer normalization. +We also modify the self-attention sub-layer in the decoder stack to prevent positions from attending to subsequent +positions. This masking, combined with fact that the output embeddings are offset by one position, ensures that the +predictions for position i can depend only on the known outputs at positions less than i. + +### Attention + +An attention function can be described as mapping a query and a set of key-value pairs to an output, where the query, +keys, values, and output are all vectors. The output is computed as a weighted sum + +### Scaled Dot-Product Attention + +We call our particular attention "Scaled Dot-Product Attention" (Figure 2). The input consists of queries and keys of +dimension\(d_{k\), and values of dimension\(d_{v\). We compute the dot products of the query with all keys, divide each +by\(\sqrt{d_{k\), and apply a softmax function to obtain the weights on the values. + +In practice, we compute the attention function on a set of queries simultaneously, packed together into a matrix\(Q\). +The keys and values are also packed together into matrices\(K\) and\(V\). We compute the matrix of outputs as: + +\[ Attention(Q, K, V) = softmax\left(\frac{QK^{T\sqrt{d_{k}}}\right)V\] (1) + +The two most commonly used attention functions are additive attention [2], and dot-product (multiplicative) attention. +Dot-product attention is identical to our algorithm, except for the scaling factor of\(\frac{1\sqrt{d_{k}}\). Additive +attention computes the compatibility function using a feed-forward network with a single hidden layer. While the two are +similar in theoretical complexity, dot-product attention is much faster and more space-efficient in practice, since it +can be implemented using highly optimized matrix multiplication code. + +While for small values of\(d_{k\) the two mechanisms perform similarly, additive attention outperforms dot product +attention without scaling for larger values of\(d_{k\)[3]. We suspect that for large values of\(d_{k\), the dot products +grow large in magnitude, pushing the softmax function into regions where it has extremely small gradients 4. To +counteract this effect, we scale the dot products by\(\frac{1\sqrt{d_{k}}\). + +Footnote 4: To illustrate why the dot products get large, assume that the components of\(q\) and\(k\) are independent +random variables with mean 0 and variance 1. Then their dot product,\(q\cdot k =\sum_{i=1}^{d_{k}} q_{i} k_{i\), has +mean 0 and variance\(d_{k\). + +### Multi-Head Attention + +Instead of performing a single attention function with\(d\text{model\)-dimensional keys, values and queries, we found it +beneficial to linearly project the queries, keys and values\(h\) times with different, learned linear projections to\(d_ +{k\),\(d_{k\), and\(d_{v\) dimensions, respectively. On each of these projected versions of queries, keys and values +then perform the attention function in parallel, yielding\(d_{v\)-dimensional + +图中展示的是一个关于神经网络模型的技术文档的一部分，其中包含了模型结构的设计细节和注意力机制的应用。以下是该部分内容的详细说明： + +### Multi-Head Attention + +多头注意力机制允许模型同时关注不同表示子空间上的信息。与单头注意力相比，平均抑制法被用来防止过拟合。其计算公式如下： + +\[ MultiHead(Q,K,V\text{Concat\text{head}_1,...\text{head}_h)W^O\] +\[\text{where head}_i\text{Attention}(QW_{i}^{Q},KW_{i}^{K},VW_{i}^{V})\] + +其中，投影矩阵\( W_i^{Q}\in\mathbb{R}^{d\text{model}}\times d_k\)，\( W_i^{K}\in\mathbb{R}^{d\text{model}}\times d_k\)，\( +W_i^{V}\in\mathbb{R}^{d\text{model}}\times d_v\)，以及\( W^O\in\mathbb{R}^{h_{dv}\times d\text{model}}}\)。 + +在本文中，我们使用了8个并行的注意力层，每个头的大小为\( d_k = d\text{model}} / h = 64\)。由于每个头的维数减少，总的计算成本与全尺寸的头相似。 + +### Applications of Attention in our Model + +Transformer 在三个不同的地方使用了多头注意力： + +1. **编码器-解码器注意力** + ：在编码器和解码器中，查询来自前一层，而记忆键和值来自输入序列。这使得解码器中的每个位置都可以关注输入序列中的所有位置。这种机制模仿了典型的序列到序列模型中的编码器-解码器注意力机制，如 [38, 2, 9]。 + +2. **自注意力层**：在自注意力层中，所有的键、值和查询都来自同一个地方，即上一层的输出。每个位置的编码器都可以关注到前一层的所有位置。 + +3. **解码器中的自注意力层**：解码器中的自注意力层使每个位置的解码器关注到解码器上直到该位置的所有位置。我们需要这样做来防止左向信息流，以保持自回归性质。我们通过掩蔽设置（设置为 + -∞）来实现可伸缩的点积注意力，以屏蔽非法连接。参见图2。 + +### Position-wise Feed-Forward Networks + +除了注意力子层外，我们模型中的每个编码器和解码器层都包含一个全连接的前馈网络。它被单独且相同地应用于每个位置。这个网络由两个线性变换组成，中间有一个 +ReLU 激活函数。其计算公式如下： + +\[\text{FFN}(x\max(0,xW_1+b_1)W_2+b_2\] + +虽然线性变换对不同的位置是相同的，但它们的使用参数从层到层是不同的。另一种描述方法是两个卷积，其核大小为1。输入和输出的维度都是\( +d\text{model}} = 512\)，而内层维度是\( d_{ff} = 2048\)。 + +### Embeddings and Softmax + +与其他序列转录模型类似，我们使用学习到的嵌入将输入标记和输出标记转换为维度为\( d\text{model}}\) 的向量。我们也使用通常学习的线性变换和 +softmax 函数将解码器的输出转换为预测下一个标记的概率。在我们的模型中，编码器和解码器共享相同的权重矩阵。在嵌入层，我们乘以\(\sqrt{d\text{model}}}\)。 + +### Positional Encoding + +Since our model contains no recurrence and no convolution, in order for the model to make use of the order of the +sequence, we must inject some information about the relative or absolute position of the tokens in the sequence. To this +end, we add "positional encodings" to the input embeddings at the bottoms of the encoder and decoder stacks. The +positional encodings have the same dimension as the embeddings, so that the two can be summed. There are many choices of +positional encodings, learned and fixed [9]. +In this work, we use sine and cosine functions of different frequencies: + +\[ PE_{(pos, 2i)} = sin(pos / 10000^{2i / d_{model}})\] +\[ PE_{(pos, 2i+1)} = cos(pos / 10000^{2i / d_{model}})\] + +where\( pos\) is the position and\( i\) is the dimension. That is, each dimension of the positional encoding corresponds +to a sinusoid. The wavelengths form a geometric progression from\( 2π\) to\( 10000 * 2π\). We chose this function +because we hypothesized it would allow the model to easily learn to attend to relative positions, since for any fixed +offset\( k\),\( PE_{(pos + k)}\) can be represented as a linear function of\( PE_{(pos)}\). + +We also experimented with using learned positional embeddings [9] instead, and found that the two versions produced +nearly identical results (see Table 3 row (E)). We chose the sinusoidal version because it may allow the model to +extrapolate to sequence lengths longer than the ones encountered during training. + +## 4 Why Self-Attention + +In this section we compare various aspects of self-attention layers to the recurrent and convolutional layers commonly +used for mapping one variable-length sequence of symbol representations\((x_1, ..., x_n\) to another sequence of equal +length\((z_1, ..., z_n\), with\( x_i, z_i\in\mathbb{R}^d\), such as a hidden layer in a typical sequence transduction +encoder or decoder. Motivating our use of self-attention we consider three desiderata. + +One is the total computational complexity per layer. Another is the amount of computation that can be parallelized, as +measured by the minimum number of sequential operations required. + +The third is the path length between long-range dependencies in the network. Learning long-range dependencies is a key +challenge in many sequence transduction tasks. One key factor affecting the ability to learn such dependencies is the +length of the paths forward and backward signals have to traverse in the network. The shorter these paths between any +combination of positions in the input and output sequences, the easier it is to learn long-range dependencies [12]. +Hence we also compare the maximum path length between any two input and output positions in networks composed of the +different layer types. + +As noted in Table 1, a self-attention layer connects all positions with a constant number of sequentially executed +operations, whereas a recurrent layer requires\( O(n)\) sequential operations. In terms of computational complexity, +self-attention layers are faster than recurrent layers when the sequence + +### Training Data and Batching + +We trained on the standard WMT 2014 English-German dataset consisting of about 4.5 million sentence pairs. Sentences +were encoded using byte-pair encoding [3], which has a shared source-target vocabulary of about 37000 tokens. For +English-French, we used the significantly larger WMT 2014 English-French dataset consisting of 36M sentences and split +tokens into a 32000 word-piece vocabulary [38]. Sentence pairs were batched together by approximate sequence length. +Each training batch contained a set of sentence pairs containing approximately 25000 source tokens and 25000 target +tokens. + +### Hardware and Schedule + +We trained our models on one machine with 8 NVIDIA P100 GPUs. For our base models using the parameters described +throughout the paper, each training step took about 0.4 seconds. We trained the base models for a total of 100,000 steps +or 12 hours. For our big models (described on the bottom line of table 3), step time was 1.0 seconds. The big models +were trained for 300,000 steps (3.5 days). + +### Optimizer + +We used the Adam optimizer [20] with\beta_1=0.9$,\beta_2=0.98$ and\epsilon = 10^{-9}$. We varied the learning rate over +the course of training, according to the formula: +\[ lnrate = d_{model}^{-0.5}\cdot min(step\_num^{-0.5}, step\_num\cdot warmup\_steps^{-1.5})\] (3) +This corresponds to increasing the learning rate linearly for the first\( warmup\_steps\) training steps, and decreasing +it thereafter proportionally to the inverse square root of the step number. We used\( warmup\_steps = 4000\). + +### Regularization + +We employ three types of regularization during training: + +### Residual Dropout + +We apply dropout [33] to the output of each sub-layer, before it is added to the sub-layer input and normalized. In +addition, we apply dropout to the sums of the embeddings and the positional encodings in both the encoder and decoder +stacks. For the base model, we use a rate of $ P_{drop} = 0.1 $. + +### Label Smoothing + +During training, we employed label smoothing of value\epsilon_{ls} = 0.1$. [36]. This hurts perplexity, as the model +learns to be more unsure; but improves accuracy and BLEU score. + +### Table 3: Variations on the Transformer architecture. Unlisted values are identical to those of the base model. All metrics are on the English-to-German translation development set, newstest2013. Listed perplexities are per-wordpiece, according to our byte-pair encoding, and should not be compared to per-word perplexities. + +| * | d_model | d_{ff} | h | d_{k} | d_{v} | P_{drop} | ε_{ls} | train steps | PPL (dev) | BLEU (dev) | params ×10^6 | +|--------------------|-----------------------------------------|-----------------------|------|-------|-------|----------|--------|-------------|-----------|------------|--------------| +| base | 6 | 512 | 2048 | 8 | 64 | 0.1 | 0.1 | 100K | 4.92 | 25.8 | 65 | +| (A) | 4 | 512 | 512 | 1 | 512 | 512 | 0.1 | 100K | 5.29 | 24.9 | 65 | +| | & & & 4 & 128 | 128 | 0.1 | 100K | 5.00 | 25.5 | 65 | +| & & & & 16 & 32 | 32 | 0.1 | 100K | 4.91 | 25.8 | 65 | +| (B) | 6 | 32 | 16 | 16 | 32 | 0.1 | 0.1 | 100K | 5.01 | 25.4 | 58 | +| | & & & & & & & & & | +| | 2 | & & & & & & 5.16 | 25.1 | 58 | +| | 4 | & & & & & & 6.11 | 25.7 | 60 | +| (C) | 8 | 256 | 32 | 32 | 32 | 0.1 | 0.1 | 100K | 4.88 | 25.5 | 56 | +| | 1024 | 1024 | 128 | 128 | 128 | 0.1 | 0.1 | 100K | 4.66 | 24.5 | 80 | +| | 4096 | 4096 | 16 | 16 | 16 | 0.1 | 0.1 | 100K | 4.52 | 26.0 | 168 | +| | & & & & & & & 4.75 | 25.4 | 53 | +| (D) | & & & & & 0.0 | 5.77 | 24.6 | | +| | & & & & & 0.2 | 4.95 | 25.5 | | +| | & & & & & 0.0 | 4.67 | 25.3 | | +| | & & & & & 0.2 | 5.47 | 25.7 | | +| (E) | positional embedding instead of sinoids | | | | | | | 4.92 | 25.7 | | +| big | 6 | 1024 | 4096 | 16 | 0.3 | 300K | 4.33 | 26.4 | 213 | | +| | & & & & & & & & & | + +### Parser generalizes well to English constituency parsing (Results are on Section 23 of WSJ) + +| Parser | Training | WSJ 23 F1 | +|-------------------------------------|--------------------------|-----------| +| Vinyals & Kaiser et al. (2014) [37] | WSJ only, discriminative | 88.3 | +| Petrov et al. (2006) [29] | WSJ only, discriminative | 90.4 | +| Zhu et al. (2013) [40] | WSJ only, discriminative | 90.4 | +| Dyer et al. (2016) [8] | WSJ only, discriminative | 91.7 | +| Transformer (4 layers) | WSJ only, discriminative | 91.3 | +| Zhu et al. (2013) [40] | semi-supervised | 91.3 | +| Huang & Harper (2009) [14] | semi-supervised | 91.3 | +| McClosky et al. (2006) [26] | semi-supervised | 92.1 | +| Vinyals & Kaiser et al. (2014) [37] | semi-supervised | 92.1 | +| Luong et al. (2015) [23] | multi-task | 93.0 | +| Dyer et al. (2016) [8] | generative | 93.3 | + +# Learning phrase representations using rnn encoder-decoder for statistical machine translation + +Kyunghyun Cho, Bart van Merrienboer, Caglar Gulcehre, Fethi Bougares, Holger Schwenk, and Yoshua Bengio + +###### Abstract + +In this paper, we present a novel approach to learn phrase representations using recurrent neural network (RNN) +encoder-decoder architectures. Our model learns to represent phrases as vectors that can be used for various tasks such +as translation, question answering, or information retrieval. We show empirically that our approach significantly +improves performance over state-of-the-art methods on several benchmark datasets. + +keywords: Statistical Machine Translation, Phrase Representations, Recurrent Neural Networks, Encoder-Decoder Models + +Footnote †: journalyear: 2014 + ++ + +Footnote †: journalyear: 2014 + +## 1 Introduction + +Learning representations of phrases is a fundamental problem in natural language processing (NLP). Phrases are groups of +words that often carry semantic meaning beyond individual words and play a crucial role in understanding and generating +natural language. For example, in the sentence "I saw a man walking his dog," the phrase "walking his dog" captures the +action being performed by the subject "man." Understanding and representing these phrases accurately can greatly improve +the performance of NLP systems on various tasks such as translation, question answering, and information retrieval. + +Recent work has shown that deep learning techniques, especially convolutional neural networks (CNNs) and recurrent +neural networks (RNNs), can effectively learn to represent phrases [?, ?]. CNNs capture local patterns in text by +applying filters across different window sizes, while RNNs can capture long-range dependencies between words in a +sequence. However, these models often require large amounts of labeled data for training, which can be expensive and +time-consuming to acquire. + +In this paper, we propose a novel approach to learn phrase representations using RNN encoder-decoder architectures. +Unlike traditional approaches that focus on single words or short sequences, our model learns to represent phrases as +vectors that can be used for various tasks. We show empirically that our approach significantly improves performance +over state-of-the-art methods on several benchmark datasets. + +The rest of the paper is organized as follows. In Section 2, we describe the proposed method in detail. Section 3 +presents the experimental setup and results. Finally, Section 4 concludes the paper with future directions. + +## 2 Methodology + +### Encoder-Decoder Architecture + +Our approach is based on the encoder-decoder architecture, which was first introduced by [?] for image caption +generation. The encoder-decoder framework consists of two parts: an encoder and a decoder. The encoder processes the +input sequence into a compact representation, while the decoder uses this representation to generate the output +sequence. + +**Encoder:** The encoder takes an input sequence\( x_1, x_2, ..., x_n\) and transforms it into a vector\( z\) that +represents the input sequence. This transformation is achieved through a series of RNN layers stacked together. At each +step, the encoder looks at the current word\( x_t\) and uses its hidden state to predict the next word in the sequence. + +**Decoder:** The decoder uses the encoded representation\( z\) to generate the output sequence\( y_1, y_2, ..., y_m\). +It also consists of a series of RNN layers, but unlike the encoder, the decoder uses a separate set of weights for each +layer. At each step, the decoder generates a new word based on the previous generated words and the current hidden +state. + +**Attention Mechanism:** To better capture the contextual information within the input sequence, we introduce an +attention mechanism inspired by [?]. Attention allows the decoder to focus on different parts of the input sequence when +generating a particular word. Specifically, during the decoding process, the decoder computes a score for each position +in the input sequence and uses softmax to distribute its attention across the sequence. + +**Multi-Head Attention:** Instead of having a single attention head, we use multi-head attention, which was introduced +by [?]. Multi-head attention allows the model to jointly attend to information from different representation subspaces +at different positions. Each head attends to a different subset of the input sequence, allowing the model to capture +more complex dependencies. + +**Feed Forward Network:** After computing the attention scores, we feed them back into the decoder along with the +original sequence and the previous hidden states. This is done through a feed + +# Building a large annotated corpus of english: The penn treebank + +Mitchell P Marcus +Mary Ann Marcinkiewicz +Beatrice Santorini +Computational Linguistics +1992 +Volume 19 +Issue 2 +Pages 313-338 + +## Effective self-training for parsing + +David McClosky +Eugene Charniak +Mark Johnson +Proceedings of the Human Language Technology Conference of the NAACL +2006 +Main Conference +Pages 152-159 + +## A decomposable attention model + +Ankur Parikh +Oscar Tackstrom +Dipanjan Das +Jakob Uszkoreit +Empirical Methods in Natural Language Processing +2016 + +## A deep reinforced model for abstractive summarization + +Romain Paulus +Caiming Xiong +Richard Socher +arXiv preprint arXiv:1705.04304 +2017 + +## Learning accurate, compact, and interpretable tree annotation + +Slav Petrov +Leon Barrett +Romain Thibaux +Dan Klein +Proceedings of the 21st International Conference on Computational Linguistics and 44th Annual Meeting of the ACL +2006 +Pages 433-440 + +## Using the output embedding to improve language models + +Ofir Press +Lior Wolf +arXiv preprint arXiv:1608.05859 +2016 + +## Neural machine translation of rare words with subword units + +Rico Sennrich +Barry Haddow +Alexandra Birch +arXiv preprint arXiv:1508.07909 +2015 + +## Outrageously large neural networks: The sparsely-gated mixture-of-experts layer + +Noam Shazeer +Azalia Mirhoseini +Krzysztof Mazurczak +Andy Davis +Quoc Le +Geoffrey Hinton +Jeff Dean +arXiv preprint arXiv:1701.06538 +2017 + +## Dropout: a simple way to prevent neural networks from overfitting + +Nitish Srivastava +Geoffrey E Hinton +Alex Krizhevsky +Ilya Sutskever +Ruslan Salakhutdinov +Journal of Machine Learning Research +2014 +Volume 15 +Issue 1 +Pages 1929-1958 + +## Sequence to sequence learning with neural networks + +Sainbayar Sukhbaatar +Arthur Szlam +Jason Weston +Rob Fergus +Advances in Neural Information Processing Systems +2015 +Volume 28 +Pages 2440-2448 + +## Sequence to sequence learning with neural networks + +Ilya Sutskever +Oriol Vinylas +Quoc VV Le +Advances in Neural Information Processing Systems +2014 +Volume 31 +Pages 3104-3112 + +## Rethinking the inception architecture for computer vision + +Christian Szegedy +Vincent Vanhoucke +Sergey Ioffe +Jonathan Shlens +Zbigniew Wojna +CoRR +abs/1512.00567 +2015 + +## Grammar as a foreign language + +Vinyals & Kaiser +Petrov +Sutskever +Hinton +Advances in Neural Information Processing Systems +2015 + +## Google's neural machine translation system: Bridging the gap between human and machine translation + +Yonghui Wu +Mike Schuster +Zhifeng Chen +Quoc V Le +Mohammad Norouzi +Wolfgang Macherey +Maxim Krikun +Yuan Cao +Qin Gao +Klaus Macherey +et al +arXiv preprint arXiv:1609.08144 +2016 + +## Deep recurrent convolutional models with fast-forward connections for neural machine translation + +Jie Zhou +Ying Cao +Xuguang Wang +Peng Li +Wei Xu +CoRR +abs/1606.04199 +2016 + +## Fast and accurate shift-reduce + +# Attention Visualizations + +It is in this spirit that a majority of American governments have passed new laws since 2009 making the registration or +voting process more difficult. EOS + +### The Law will never be perfect, but its application should be just this way. + +In my opinion, we are missing something in what we are doing. + +This is what we are missing in my opinion. + +# 14_0.png + +The Law will never be perfect. But its application should just be what this is - missing in my opinion. + + From 9efa386e396be176a1cf93d78c3028d0ab515dd5 Mon Sep 17 00:00:00 2001 From: zR <2448370773@qq.com> Date: Wed, 3 Jul 2024 15:24:36 +0800 Subject: [PATCH 2/2] prompt should be dict --- README.md | 14 +++++++++++++- README_CN.md | 18 ++++++++++++++---- gptpdf/parse.py | 6 +++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7f803e1..2620eff 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,19 @@ See more in [test/test.py](test/test.py) ### parse_pdf -**Function**: `parse_pdf(pdf_path, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False, gpt_worker=1)` +**Function**: +``` +def parse_pdf( + pdf_path: str, + output_dir: str = './', + prompt: Optional[Dict] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model: str = 'gpt-4o', + verbose: bool = False, + gpt_worker: int = 1 +) -> Tuple[str, List[str]]: +``` Parses a PDF file into a Markdown file and returns the Markdown content along with all image paths. diff --git a/README_CN.md b/README_CN.md index 23e7bc6..23d3606 100644 --- a/README_CN.md +++ b/README_CN.md @@ -25,8 +25,7 @@ ## 样例 -有关 -PDF，请参阅 [examples/attention_is_all_you_need/output.md](examples/attention_is_all_you_need/output.md) [examples/attention_is_all_you_need.pdf](examples/attention_is_all_you_need.pdf)。 +有关 PDF，请参阅 [examples/attention_is_all_you_need/output.md](examples/attention_is_all_you_need/output.md) [examples/attention_is_all_you_need.pdf](examples/attention_is_all_you_need.pdf)。 ## 安装 @@ -50,8 +49,19 @@ print(content) ### parse_pdf -**函数 -**：`parse_pdf(pdf_path, output_dir='./', api_key=None, base_url=None, model='gpt-4o', verbose=False, gpt_worker=1)` +**函数**： +``` +def parse_pdf( + pdf_path: str, + output_dir: str = './', + prompt: Optional[Dict] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + model: str = 'gpt-4o', + verbose: bool = False, + gpt_worker: int = 1 +) -> Tuple[str, List[str]]: +``` 将 PDF 文件解析为 Markdown 文件，并返回 Markdown 内容和所有图片路径列表。 diff --git a/gptpdf/parse.py b/gptpdf/parse.py index fa19cd2..1271c66 100644 --- a/gptpdf/parse.py +++ b/gptpdf/parse.py @@ -1,5 +1,5 @@ import os -from typing import List, Tuple, Optional +from typing import List, Tuple, Optional, Dict import logging logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') @@ -168,7 +168,7 @@ def _parse_pdf_to_images(pdf_path: str, output_dir: str = './') -> List[Tuple[st def _gpt_parse_images( image_infos: List[Tuple[str, List[str]]], - prompt: Optional[str], + prompt: Optional[Dict] = None, output_dir: str = './', api_key: Optional[str] = None, base_url: Optional[str] = None, @@ -224,7 +224,7 @@ def _process_page(index: int, image_info: Tuple[str, List[str]]) -> Tuple[int, s def parse_pdf( pdf_path: str, output_dir: str = './', - prompt: Optional[str] = None, + prompt: Optional[Dict] = None, api_key: Optional[str] = None, base_url: Optional[str] = None, model: str = 'gpt-4o',