feat(chat,pdf): scanned PDF OCR fallback and authenticated attachment display

This commit is contained in:
matevip 2026-04-06 07:48:28 +08:00
parent cc0569440e
commit d8cf5acf0f
5 changed files with 364 additions and 24 deletions

View File

@ -153,31 +153,197 @@ public class DocumentExtractTool {
// ==================== PDF 提取链 ====================
private ExtractedContent extractPdf(Path path, String options, List<String> attempts) throws Exception {
String bestContent = null;
String bestMethod = null;
// 1. 尝试 pdftotext
long t0 = System.currentTimeMillis();
String content = tryPdftotext(path, options);
if (content != null && !content.isBlank()) {
attempts.add("pdftotext: 成功");
return new ExtractedContent(content, "pdftotext", estimatePages(content));
int pages = estimatePages(content);
if (!needsOcr(content, pages)) {
attempts.add("pdftotext: 成功 (" + (System.currentTimeMillis() - t0) + "ms)");
return new ExtractedContent(content, "pdftotext", pages);
}
attempts.add("pdftotext: 文本过少 (每页 " + charsPerPage(content, pages) + " 字符),可能是扫描版");
bestContent = content;
bestMethod = "pdftotext";
} else {
attempts.add("pdftotext: 失败或不可用");
}
attempts.add("pdftotext: 失败或不可用");
// 2. 尝试 Python pdfplumber/pypdf
long t1 = System.currentTimeMillis();
content = tryPythonPdfExtractor(path, options);
if (content != null && !content.isBlank()) {
attempts.add("python_pdf: 成功");
return new ExtractedContent(content, "python_pdfplumber", estimatePages(content));
int pages = estimatePages(content);
if (!needsOcr(content, pages)) {
attempts.add("python_pdf: 成功 (" + (System.currentTimeMillis() - t1) + "ms)");
return new ExtractedContent(content, "python_pdfplumber", pages);
}
attempts.add("python_pdf: 文本过少");
if (bestContent == null || content.strip().length() > bestContent.strip().length()) {
bestContent = content;
bestMethod = "python_pdfplumber";
}
} else {
attempts.add("python_pdf: 失败或不可用");
}
attempts.add("python_pdf: 失败或不可用");
// 3. Java 实现基于 Apache PDFBox 逻辑 Java
// 3. Java 实现
long t2 = System.currentTimeMillis();
content = extractPdfWithJava(path);
if (content != null && !content.isBlank()) {
attempts.add("java_pdf: 成功");
return new ExtractedContent(content, "java_pdfbox", estimatePages(content));
int pages = estimatePages(content);
if (!needsOcr(content, pages)) {
attempts.add("java_pdf: 成功 (" + (System.currentTimeMillis() - t2) + "ms)");
return new ExtractedContent(content, "java_pdfbox", pages);
}
attempts.add("java_pdf: 文本过少");
if (bestContent == null || content.strip().length() > bestContent.strip().length()) {
bestContent = content;
bestMethod = "java_pdfbox";
}
} else {
attempts.add("java_pdf: 失败");
}
attempts.add("java_pdf: 失败");
throw new Exception("所有 PDF 提取方法都失败");
// 4. OCR fallback扫描版/照片型 PDF
log.info("[DocumentExtract] 文本提取不足,尝试 OCR: {}", path.getFileName());
long t3 = System.currentTimeMillis();
content = tryOcrExtract(path, attempts);
if (content != null && !content.isBlank()) {
attempts.add("ocr_tesseract: 成功 (" + (System.currentTimeMillis() - t3) + "ms)");
return new ExtractedContent(content, "ocr_tesseract", estimatePages(content));
}
// attempts 已由 tryOcrExtract 内部记录失败原因
// 返回之前级别的部分结果如果有
if (bestContent != null) {
log.warn("[DocumentExtract] OCR 不可用,返回部分文本结果: method={}, length={}",
bestMethod, bestContent.strip().length());
return new ExtractedContent(bestContent, bestMethod + "_partial", estimatePages(bestContent));
}
throw new Exception("所有 PDF 提取方法都失败(包括 OCR");
}
/**
* 判断提取到的文本是否太少需要尝试 OCR
* 基于字符密度每页平均字符数而非总字符数避免误判短票据/证书类 PDF
*/
private boolean needsOcr(String text, int estimatedPages) {
if (text == null || text.isBlank()) return true;
String stripped = text.strip();
if (stripped.length() < 20) return true;
double perPage = charsPerPage(stripped, estimatedPages);
// 正常文本 PDF 每页至少数百字符每页不到 30 字符大概率是扫描版
return perPage < 30;
}
private double charsPerPage(String text, int pages) {
return (double) text.strip().length() / Math.max(1, pages);
}
/**
* OCR 提取pdftoppm 转图片 + tesseract 识别文字
* 仅依赖 Popplerpdftoppm tesseract 系统命令不引入新 Python 依赖
*/
private String tryOcrExtract(Path pdfPath, List<String> attempts) {
Path tempDir = null;
try {
long startTime = System.currentTimeMillis();
tempDir = Files.createTempDirectory("mc_ocr_");
// Step 1: 检查 pdftoppm 可用性
try {
executeCommand(List.of("pdftoppm", "-v"));
} catch (Exception e) {
attempts.add("ocr: pdftoppm 不可用,无法将 PDF 转为图片");
log.warn("[DocumentExtract] OCR: pdftoppm 不可用 - {}", e.getMessage());
return null;
}
// Step 2: PDF PNG200dpi足够 OCR 但不过大
long t1 = System.currentTimeMillis();
String pagePrefix = tempDir.resolve("page").toString();
executeCommand(List.of("pdftoppm", "-png", "-r", "200",
pdfPath.toString(), pagePrefix));
log.info("[DocumentExtract] OCR: pdftoppm 耗时 {}ms", System.currentTimeMillis() - t1);
File[] pageFiles = tempDir.toFile().listFiles((dir, name) -> name.endsWith(".png"));
if (pageFiles == null || pageFiles.length == 0) {
attempts.add("ocr: pdftoppm 未生成图片");
return null;
}
java.util.Arrays.sort(pageFiles);
// Step 3: 检查 tesseract 可用性并探测语言包
String langParam;
try {
String langOutput = executeCommand(List.of("tesseract", "--list-langs"));
langParam = buildTesseractLangParam(langOutput);
log.info("[DocumentExtract] OCR: tesseract 可用,语言参数: {}", langParam);
} catch (Exception e) {
attempts.add("ocr: tesseract 未安装");
log.warn("[DocumentExtract] OCR: tesseract 不可用 - {}", e.getMessage());
return null;
}
// Step 4: 对每页执行 OCR
long t2 = System.currentTimeMillis();
StringBuilder ocrText = new StringBuilder();
for (int i = 0; i < pageFiles.length; i++) {
ocrText.append("--- Page ").append(i + 1).append(" ---\n");
try {
List<String> cmd = new ArrayList<>();
cmd.add("tesseract");
cmd.add(pageFiles[i].getAbsolutePath());
cmd.add("stdout");
if (langParam != null) {
cmd.add("-l");
cmd.add(langParam);
}
String pageText = executeCommand(cmd);
ocrText.append(pageText != null ? pageText.trim() : "").append("\n\n");
} catch (Exception e) {
log.warn("[DocumentExtract] OCR: 页面 {} tesseract 失败: {}", i + 1, e.getMessage());
ocrText.append("[OCR 失败]\n\n");
}
}
log.info("[DocumentExtract] OCR: tesseract 处理 {} 页耗时 {}ms总耗时 {}ms",
pageFiles.length, System.currentTimeMillis() - t2, System.currentTimeMillis() - startTime);
String result = ocrText.toString().trim();
return result.isEmpty() ? null : result;
} catch (Exception e) {
log.warn("[DocumentExtract] OCR 失败: {}", e.getMessage());
attempts.add("ocr: 异常 - " + e.getMessage());
return null;
} finally {
if (tempDir != null) {
try {
Files.walk(tempDir)
.sorted(java.util.Comparator.reverseOrder())
.forEach(f -> { try { Files.delete(f); } catch (IOException ignored) {} });
} catch (IOException ignored) {}
}
}
}
/**
* tesseract --list-langs 输出中构建最优语言参数
* 优先 eng+chi_sim缺中文则只用 eng都缺则返回 null tesseract 默认
*/
private String buildTesseractLangParam(String listLangsOutput) {
if (listLangsOutput == null) return null;
boolean hasEng = listLangsOutput.contains("eng");
boolean hasChiSim = listLangsOutput.contains("chi_sim");
if (hasEng && hasChiSim) return "eng+chi_sim";
if (hasEng) return "eng";
if (hasChiSim) return "chi_sim";
return null; // tesseract 默认语言
}
private String tryPdftotext(Path path, String options) {
@ -551,7 +717,7 @@ public class DocumentExtractTool {
}
}
private String tryPythonScript(String script, String filePath) {
private String tryPythonScript(String script, String... args) {
Path tempScript = null;
try {
tempScript = Files.createTempFile("extract", ".py");
@ -559,7 +725,10 @@ public class DocumentExtractTool {
// Windows 通常只有 python没有 python3
String pythonCmd = IS_WINDOWS ? "python" : "python3";
List<String> command = List.of(pythonCmd, tempScript.toString(), filePath);
List<String> command = new ArrayList<>();
command.add(pythonCmd);
command.add(tempScript.toString());
for (String arg : args) command.add(arg);
return executeCommand(command);
} catch (Exception e) {
log.debug("Python 脚本失败: {}", e.getMessage());

View File

@ -42,6 +42,22 @@ http.interceptors.response.use(
}
)
// ==================== 受保护文件访问 ====================
/**
* 使 JWT fetch Blob
* <img> <a download> Authorization header
* 使 fetch axios baseURL R<T>
*/
export async function fetchAuthenticatedBlob(fileUrl: string): Promise<Blob> {
const token = localStorage.getItem('token')
const headers: Record<string, string> = {}
if (token) headers.Authorization = `Bearer ${token}`
const response = await fetch(fileUrl, { headers })
if (!response.ok) throw new Error(`Fetch failed: ${response.status}`)
return response.blob()
}
// ==================== Auth ====================
export const authApi = {
login: (data: { username: string; password: string }) =>

View File

@ -214,21 +214,19 @@
class="message-attachment-image"
>
<img
:src="attachment.url"
:src="getDisplayUrl(attachment)"
:alt="attachment.name"
loading="lazy"
@click="openAttachment(attachment.url)"
@click="openImage(getDisplayUrl(attachment))"
/>
<span class="message-attachment-image__name">{{ attachment.name }}</span>
</div>
<a
<button
v-for="attachment in fileAttachments"
:key="attachment.storedName"
class="message-attachment"
:href="attachment.url"
target="_blank"
rel="noreferrer"
:download="attachment.name"
type="button"
@click="downloadFile(attachment)"
>
<svg class="message-attachment__icon" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/>
@ -236,7 +234,7 @@
</svg>
<span class="message-attachment__name">{{ attachment.name }}</span>
<span class="message-attachment__meta">{{ formatFileSize(attachment.size) }}</span>
</a>
</button>
</div>
</div>
@ -288,12 +286,14 @@
import { computed, ref, watch, onBeforeUnmount } from 'vue'
import { useI18n } from 'vue-i18n'
import { useMarkdownRenderer } from '@/composables/useMarkdownRenderer'
import { useAuthenticatedAttachment } from '@/composables/useAuthenticatedAttachment'
import TypingCursor from './TypingCursor.vue'
import type { Message, ChatAttachment, ToolCallMeta, PlanMeta } from '@/types'
import type { ChatErrorInfo } from '@/types/chatError'
const { renderMarkdown } = useMarkdownRenderer()
const { t } = useI18n()
const { blobUrls, loadAllImages, downloadFile, openImage, getDisplayUrl, revokeAll } = useAuthenticatedAttachment()
interface Props {
message: Message
@ -456,6 +456,7 @@ function copyMessage() {
onBeforeUnmount(() => {
if (copyTimer) clearTimeout(copyTimer)
revokeAll()
})
// --- ---
@ -463,6 +464,11 @@ const attachments = computed(() => props.message.attachments || [])
const imageAttachments = computed(() => attachments.value.filter(a => a.contentType?.startsWith('image/')))
const fileAttachments = computed(() => attachments.value.filter(a => !a.contentType?.startsWith('image/')))
// blob URLwatch +
watch(imageAttachments, (atts) => {
if (atts.length > 0) loadAllImages(atts)
}, { immediate: true })
// --- ---
const formattedTime = computed(() => {
if (!props.message.createTime) return ''
@ -478,9 +484,7 @@ const formatFileSize = (size: number) => {
return `${(size / (1024 * 1024)).toFixed(1)} MB`
}
const openAttachment = (url: string) => {
window.open(url, '_blank')
}
// openAttachment useAuthenticatedAttachment openImage / downloadFile
// --- ---
const executionExpanded = ref(false)
@ -1331,6 +1335,10 @@ watch(isGenerating, (generating) => {
background: rgba(255, 255, 255, 0.14);
color: inherit;
text-decoration: none;
border: none;
font: inherit;
cursor: pointer;
width: 100%;
}
.user-bubble .message-attachment {

View File

@ -0,0 +1,134 @@
import { ref } from 'vue'
import { fetchAuthenticatedBlob } from '@/api/index'
import type { ChatAttachment } from '@/types'
/**
* Loader
*
* 使 Bearer token localStorage<img src> <a href>
* Authorization header composable fetch blob ObjectURL
*
* 使
* - loadAllImages() blobUrls[key] <img :src>
* - downloadFile()
* - openImage()
* - revokeAll() blob URL
*/
export function useAuthenticatedAttachment() {
/** 已加载的 blob URL 映射key = storedName || url */
const blobUrls = ref<Record<string, string>>({})
/** 跟踪所有创建的 blob URL用于清理 */
const trackedUrls: string[] = []
/** 正在加载的 URL 集合,防止重复请求 */
const loading = new Set<string>()
/**
* blob URL
*/
async function loadBlobUrl(url: string, key: string): Promise<string | null> {
// 跳过:已加载、正在加载、本地 blob
if (blobUrls.value[key] || loading.has(key) || url.startsWith('blob:')) return blobUrls.value[key] || url
loading.add(key)
try {
const blob = await fetchAuthenticatedBlob(url)
const objectUrl = URL.createObjectURL(blob)
blobUrls.value[key] = objectUrl
trackedUrls.push(objectUrl)
return objectUrl
} catch (e) {
console.warn('[useAuthenticatedAttachment] Failed to load:', url, e)
return null
} finally {
loading.delete(key)
}
}
/**
* blob URL
*/
async function loadAllImages(attachments: ChatAttachment[]) {
const imageAtts = attachments.filter(a => a.contentType?.startsWith('image/'))
for (const att of imageAtts) {
const key = att.storedName || att.url
if (!att.previewUrl && att.url && !blobUrls.value[key]) {
await loadBlobUrl(att.url, key)
}
}
}
/**
* fetch blob <a download>
*/
async function downloadFile(attachment: ChatAttachment) {
try {
const blob = await fetchAuthenticatedBlob(attachment.url)
const url = URL.createObjectURL(blob)
const a = document.createElement('a')
a.href = url
a.download = attachment.name || 'download'
document.body.appendChild(a)
a.click()
document.body.removeChild(a)
// 延迟释放,确保下载已启动
setTimeout(() => URL.revokeObjectURL(url), 10000)
} catch (e) {
console.error('[useAuthenticatedAttachment] Download failed:', attachment.name, e)
// fallback直接打开 URL可能 401 但至少给个反馈)
window.open(attachment.url, '_blank')
}
}
/**
*
* window.open blob URL
*/
async function openImage(url: string) {
if (url.startsWith('blob:')) {
window.open(url, '_blank')
return
}
// 同步开窗:必须在用户点击的同步调用栈中,否则被浏览器拦截
const win = window.open('about:blank', '_blank')
if (!win) return
try {
const blob = await fetchAuthenticatedBlob(url)
const blobUrl = URL.createObjectURL(blob)
win.location.href = blobUrl
// 页面关闭后释放
setTimeout(() => URL.revokeObjectURL(blobUrl), 300000)
} catch {
win.location.href = url // fallback
}
}
/**
* URL blob previewUrl url
*/
function getDisplayUrl(attachment: ChatAttachment): string {
const key = attachment.storedName || attachment.url
return blobUrls.value[key] || attachment.previewUrl || attachment.url || ''
}
/**
* blob URL
*/
function revokeAll() {
for (const url of trackedUrls) {
URL.revokeObjectURL(url)
}
trackedUrls.length = 0
blobUrls.value = {}
}
return {
blobUrls,
loadBlobUrl,
loadAllImages,
downloadFile,
openImage,
getDisplayUrl,
revokeAll,
}
}

View File

@ -1029,6 +1029,19 @@ function normalizeMessage(raw: Message): Message {
// interrupted interrupt-with-followup stopped
if (!msg.status) msg.status = 'completed'
// file-type contentParts attachments API attachments
const fileParts = msg.contentParts.filter(p => p.type === 'file' && p.fileUrl)
if (fileParts.length > 0 && (!msg.attachments || msg.attachments.length === 0)) {
msg.attachments = fileParts.map(p => ({
name: p.fileName || 'unknown',
size: typeof p.fileSize === 'number' ? p.fileSize : Number(p.fileSize) || 0,
url: p.fileUrl!,
storedName: p.storedName || '',
path: p.path || '',
contentType: p.contentType,
}))
}
// [] errorInfo使
if (msg.role === 'assistant' && !msg.errorInfo) {
const text = msg.content || msg.contentParts?.find(p => p.type === 'text')?.text || ''