Skip to content

Commit 16c4b84

Browse files
committed
chore: refactor handle_pdf_content to include save_image functionality and improve image handling
1 parent ef338e0 commit 16c4b84

1 file changed

Lines changed: 16 additions & 4 deletions

File tree

‎apps/common/handle/impl/text/pdf_split_handle.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,15 @@
1414
import traceback
1515
from typing import List
1616

17+
import uuid_utils.compat as uuid
1718
from django.utils.translation import gettext_lazy as _
1819
from pypdf import PdfReader
1920
from pypdf.generic import Destination
2021

2122
from common.handle.base_split_handle import BaseSplitHandle
2223
from common.utils.logger import maxkb_logger
2324
from common.utils.split_model import SplitModel, smart_split_paragraph
25+
from knowledge.models import File
2426

2527
default_pattern_list = [
2628
re.compile("(?<=^)# .*|(?<=\\n)# .*"),
@@ -81,7 +83,7 @@ def handle(
8183
return {"name": file.name, "content": result}
8284

8385
# 没有目录的pdf
84-
content = self.handle_pdf_content(file, pdf_document)
86+
content = self.handle_pdf_content(file, pdf_document, save_image)
8587

8688
if pattern_list is not None and len(pattern_list) > 0:
8789
split_model = SplitModel(pattern_list, with_filter, limit)
@@ -97,7 +99,7 @@ def handle(
9799
return {"name": file.name, "content": split_model.parse(content)}
98100

99101
@staticmethod
100-
def handle_pdf_content(file, pdf_document):
102+
def handle_pdf_content(file, pdf_document, save_image):
101103
# 第一步:收集所有字体大小
102104
font_sizes = []
103105
page_lines = []
@@ -118,6 +120,7 @@ def handle_pdf_content(file, pdf_document):
118120

119121
# 第二步:提取内容
120122
content = ""
123+
image_list = []
121124
for page_num, page in enumerate(pdf_document.pages):
122125
start_time = time.time()
123126

@@ -136,13 +139,22 @@ def handle_pdf_content(file, pdf_document):
136139
content += f"{text}\n"
137140

138141
for image_index in range(PdfSplitHandle.get_page_image_count(page)):
139-
content += f"![image](image_{page_num}_{image_index})\n\n"
142+
try:
143+
image = page.images[image_index]
144+
except Exception as e:
145+
maxkb_logger.warning(f"File: {file.name}, Page: {page_num + 1}, Image: {image_index}, error: {e}")
146+
continue
147+
image_id = uuid.uuid7()
148+
image_list.append(File(id=image_id, file_name=image.name, meta={"debug": False, "content": image.data}))
149+
content += f"![image](./oss/file/{image_id})\n\n"
140150

141151
content = content.replace("\0", "")
142152

143153
elapsed_time = time.time() - start_time
144154
maxkb_logger.debug(f"File: {file.name}, Page: {page_num + 1}, Time: {elapsed_time:.3f}s")
145155

156+
if image_list:
157+
save_image(image_list)
146158
return content
147159

148160
@staticmethod
@@ -613,7 +625,7 @@ def get_content(self, file, save_image):
613625
try:
614626
with open(temp_file_path, "rb") as pdf_file:
615627
pdf_document = PdfReader(pdf_file)
616-
return self.handle_pdf_content(file, pdf_document)
628+
return self.handle_pdf_content(file, pdf_document, save_image)
617629
except BaseException as e:
618630
traceback.print_exception(e)
619631
return f"{e}"

0 commit comments

Comments
 (0)