diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ReadFileTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ReadFileTool.java index 817a3b10..dc81781c 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ReadFileTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ReadFileTool.java @@ -49,12 +49,15 @@ public class ReadFileTool { @Tool(description = """ Read the contents of a file. Supports line-range reading (1-based). \ Returns structured JSON with filePath, totalLines, readLines, content. \ - Auto-truncates large files with continuation hints. \ - Text files only; use extract_document_text for PDF/Office documents.""") + Auto-truncates large files with continuation hints: when the result has \ + truncated=true, continue with the returned nextStartLine (and \ + nextStartColumn when present, to resume reading the rest of a very long \ + line). Text files only; use extract_document_text for PDF/Office documents.""") public String read_file( @ToolParam(description = "Absolute or relative file path") String filePath, @ToolParam(description = "Start line number (1-based, inclusive). Omit to start from line 1", required = false) Integer startLine, @ToolParam(description = "End line number (1-based, inclusive). Omit to read to EOF or truncation limit", required = false) Integer endLine, + @ToolParam(description = "Start character position within startLine (1-based, inclusive). Used to resume reading the rest of a very long line; pass the nextStartColumn from a previous truncated result. Omit to start at the beginning of the line", required = false) Integer startColumn, // RFC-063r §2.5: hidden from LLM by JsonSchemaGenerator. @Nullable ToolContext ctx) { @@ -130,68 +133,108 @@ public class ReadFileTool { // 提取指定范围的行（转为 0-based） List selectedLines = allLines.subList(start - 1, end); + // Character offset into the FIRST selected line, used to resume reading + // the tail of a very long line across calls. 1-based on the wire, 0-based + // here. Only applies to the first line of the selection. + int firstLineOffset = (startColumn != null && startColumn > 1) ? startColumn - 1 : 0; + // Truncation control. Each output line carries a "%6d\t" prefix and a - // trailing newline, so the budget available for the line's own text is - // the remaining byte budget minus that overhead. + // trailing newline, so the budget available for a line's own text is the + // remaining budget minus that overhead. StringBuilder sb = new StringBuilder(); int linesRead = 0; boolean truncated = false; boolean lineTruncated = false; int truncatedLineNum = 0; + // Where a subsequent read_file call should resume. nextLine is 1-based; + // nextColumn is a 1-based char offset (1 = start of the line). + int nextLine = -1; + int nextColumn = 1; for (int i = 0; i < selectedLines.size(); i++) { - String line = selectedLines.get(i); int lineNum = start + i; + String fullLine = selectedLines.get(i); + // The offset only applies to the first line of the selection. + int offset = (i == 0) ? Math.min(firstLineOffset, fullLine.length()) : 0; + String line = offset > 0 ? fullLine.substring(offset) : fullLine; if (linesRead >= DEFAULT_MAX_LINES) { + // Hit the line-count cap; resume at this line from the same offset. truncated = true; + nextLine = lineNum; + nextColumn = offset + 1; break; } - String numberedLine = String.format("%6d\t%s\n", lineNum, line); - if (sb.length() + numberedLine.length() > MAX_OUTPUT_BYTES) { - // This line does not fit in the remaining budget. Normally we - // stop and let the caller continue from the next line. But when - // a single line is itself larger than the whole budget and we - // have read nothing yet, stopping here would return empty - // content with linesRead=0 — and the suggested continuation - // startLine never advances, producing an infinite retry loop. - // Guarantee progress by emitting as much of this oversized line - // as fits, flagged as truncated, then advancing past it. - if (linesRead == 0) { - String prefix = String.format("%6d\t", lineNum); - String marker = i18n.msg("tool.read_file.line_truncated_marker"); - int budget = MAX_OUTPUT_BYTES - prefix.length() - marker.length() - 1; // -1 for '\n' - String clipped = safeTruncate(line, Math.max(0, budget)); - sb.append(prefix).append(clipped).append(marker).append('\n'); - linesRead++; - lineTruncated = true; - truncatedLineNum = lineNum; - } + String prefix = String.format("%6d\t", lineNum); + int lineCost = prefix.length() + line.length() + 1; // +1 for '\n' + + if (sb.length() + lineCost <= MAX_OUTPUT_BYTES) { + sb.append(prefix).append(line).append('\n'); + linesRead++; + continue; + } + + // This line does not fit in the remaining budget. + boolean fitsAlone = prefix.length() + line.length() + 1 <= MAX_OUTPUT_BYTES; + if (fitsAlone || linesRead > 0) { + // EITHER the line would fit in a fresh budget (normal truncation + // at a clean line boundary), OR we have already emitted lines and + // defer this oversized line to the next call. Either way, resume + // at this line; for non-first lines offset is 0 so column is 1. truncated = true; + nextLine = lineNum; + nextColumn = offset + 1; break; } - sb.append(numberedLine); + + // linesRead == 0 AND the line is larger than the whole budget even on + // its own. Returning empty content here would yield readLines=0 with a + // continuation hint that never advances — the infinite retry loop from + // the original bug. Emit as much of this line as fits (a window), + // flagged truncated, and advance by exactly the chars consumed so the + // caller can page through the rest of the line with nextStartColumn. + String marker = i18n.msg("tool.read_file.line_truncated_marker"); + int windowBudget = MAX_OUTPUT_BYTES - prefix.length() - marker.length() - 1; + String window = safeTruncate(line, Math.max(0, windowBudget)); + sb.append(prefix).append(window).append(marker).append('\n'); linesRead++; + truncated = true; + lineTruncated = true; + truncatedLineNum = lineNum; + int consumed = offset + window.length(); + if (consumed < fullLine.length()) { + nextLine = lineNum; // more of this line remains + nextColumn = consumed + 1; + } else { + nextLine = lineNum + 1; // line exactly consumed; move on + nextColumn = 1; + } + break; } result.set("startLine", start); + result.set("startColumn", firstLineOffset + 1); result.set("endLine", start + linesRead - 1); result.set("readLines", linesRead); result.set("content", sb.toString()); if (truncated) { result.set("truncated", true); + result.set("nextStartLine", nextLine); int kb = MAX_OUTPUT_BYTES / 1024; - if (lineTruncated) { - // The oversized line was clipped in place; line ranges cannot - // recover its tail, so do not advertise a startLine that would - // silently skip the remainder. + if (lineTruncated && nextColumn > 1) { + // A long line was windowed and more of it remains. Surface the + // column so the caller can resume reading the same line's tail. result.set("lineTruncated", true); - result.set("message", i18n.msg("tool.read_file.line_truncated", truncatedLineNum, kb)); + result.set("nextStartColumn", nextColumn); + result.set("message", i18n.msg("tool.read_file.line_truncated", + truncatedLineNum, kb, nextLine, nextColumn, nextLine + 1)); } else { - int nextStart = start + linesRead; - result.set("message", i18n.msg("tool.read_file.truncated", DEFAULT_MAX_LINES, kb, nextStart)); + if (lineTruncated) { + result.set("lineTruncated", true); + } + result.set("message", i18n.msg("tool.read_file.truncated", DEFAULT_MAX_LINES, kb, nextLine)); } } else { result.set("truncated", false); diff --git a/mateclaw-server/src/main/resources/messages.properties b/mateclaw-server/src/main/resources/messages.properties index f60e53db..3b491805 100644 --- a/mateclaw-server/src/main/resources/messages.properties +++ b/mateclaw-server/src/main/resources/messages.properties @@ -61,7 +61,7 @@ tool.read_file.error.start_gt_end=\u8d77\u59cb\u884c {0} \u5927\u4e8e\u7ed3\u675 tool.read_file.error.read_exception=\u8bfb\u53d6\u6587\u4ef6\u5f02\u5e38: {0} tool.read_file.truncated=\u8f93\u51fa\u5df2\u622a\u65ad\uff08\u6700\u591a {0} \u884c / {1}KB\uff09\u3002\u4f7f\u7528 startLine={2} \u7ee7\u7eed\u8bfb\u53d6\u3002 tool.read_file.line_truncated_marker= ...[\u672c\u884c\u8fc7\u957f\uff0c\u5df2\u622a\u65ad] -tool.read_file.line_truncated=\u7b2c {0} \u884c\u957f\u5ea6\u8d85\u8fc7\u5355\u6b21\u8f93\u51fa\u4e0a\u9650\uff08{1}KB\uff09\uff0c\u5df2\u622a\u65ad\u663e\u793a\u3002\u8be5\u884c\u5b8c\u6574\u5185\u5bb9\u65e0\u6cd5\u901a\u8fc7\u884c\u53f7\u8303\u56f4\u9010\u6b65\u8bfb\u53d6\uff1b\u5982\u9700\u5b8c\u6574\u6570\u636e\u8bf7\u4f7f\u7528 execute_shell_command\u3002\u4e0d\u8981\u63a8\u65ad\u6216\u865a\u6784\u88ab\u622a\u65ad\u7684\u5185\u5bb9\u3002 +tool.read_file.line_truncated=\u7b2c {0} \u884c\u957f\u5ea6\u8d85\u8fc7\u5355\u6b21\u8f93\u51fa\u4e0a\u9650\uff08{1}KB\uff09\u3002\u8981\u7ee7\u7eed\u8bfb\u53d6\u8be5\u884c\u5269\u4f59\u5185\u5bb9\uff0c\u8bf7\u4f7f\u7528 startLine={2}\u3001startColumn={3}\uff1b\u6216\u4f7f\u7528 startLine={4} \u8df3\u5230\u4e0b\u4e00\u884c\u3002\u4e0d\u8981\u63a8\u65ad\u6216\u865a\u6784\u88ab\u622a\u65ad\u7684\u5185\u5bb9\u3002 tool.write_file.error.path_empty=\u6587\u4ef6\u8def\u5f84\u4e0d\u80fd\u4e3a\u7a7a tool.write_file.error.is_directory=\u8def\u5f84\u662f\u4e00\u4e2a\u5df2\u6709\u76ee\u5f55\uff0c\u65e0\u6cd5\u4f5c\u4e3a\u6587\u4ef6\u5199\u5165: {0} tool.write_file.error.write_exception=\u5199\u5165\u6587\u4ef6\u5f02\u5e38: {0} diff --git a/mateclaw-server/src/main/resources/messages_en.properties b/mateclaw-server/src/main/resources/messages_en.properties index d5aa8220..859c5a4c 100644 --- a/mateclaw-server/src/main/resources/messages_en.properties +++ b/mateclaw-server/src/main/resources/messages_en.properties @@ -61,7 +61,7 @@ tool.read_file.error.start_gt_end=Start line {0} is greater than end line {1} tool.read_file.error.read_exception=Read file exception: {0} tool.read_file.truncated=Output truncated (max {0} lines / {1}KB). Use startLine={2} to continue reading. tool.read_file.line_truncated_marker= ...[line too long, truncated] -tool.read_file.line_truncated=Line {0} exceeds the single-output limit ({1}KB) and was truncated. Its full content cannot be read via line ranges; use execute_shell_command if you need the complete data. Do NOT infer or fabricate the omitted content. +tool.read_file.line_truncated=Line {0} exceeds the {1}KB single-output limit. To read the rest of this line, continue with startLine={2}, startColumn={3}; or skip to startLine={4} for the next line. Do NOT infer or fabricate the omitted content. tool.write_file.error.path_empty=File path cannot be empty tool.write_file.error.is_directory=Path is an existing directory, cannot write as file: {0} tool.write_file.error.write_exception=Write file exception: {0} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/ReadFileToolLargeLineTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ReadFileToolLargeLineTest.java index dc54d941..cc847ea5 100644 --- a/mateclaw-server/src/test/java/vip/mate/tool/builtin/ReadFileToolLargeLineTest.java +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ReadFileToolLargeLineTest.java @@ -19,11 +19,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; /** - * Regression test for the single-line oversized-file bug: a file whose only - * line exceeds the output byte budget used to return empty content with - * readLines=0 and a continuation hint that never advanced — an infinite retry - * loop. The tool must instead return a clipped, clearly-flagged result and make - * progress. + * Regression tests for reading oversized content with {@link ReadFileTool}. + *

+ * The original bug: a file whose only line exceeds the output budget returned + * empty content with readLines=0 and a continuation hint (startLine) that never + * advanced — an infinite retry loop. These tests assert that the tool always + * makes progress, clearly flags truncation, and lets a caller page through both + * a very long single line (via nextStartColumn) and subsequent normal lines + * (via nextStartLine). */ class ReadFileToolLargeLineTest { @@ -45,29 +48,134 @@ class ReadFileToolLargeLineTest { ToolExecutionContext.clear(); } - @Test - @DisplayName("single line larger than the 30KB budget returns clipped content, not empty + infinite loop") - void singleOversizedLine_returnsClippedContent(@TempDir Path dir) throws Exception { - // ~40KB single-line JSON array on one physical line. + /** Build a single-line JSON array of {@code n} string elements. */ + private static String oneLineJsonArray(int n) { StringBuilder json = new StringBuilder("["); - for (int i = 0; i < 4000; i++) { + for (int i = 0; i < n; i++) { if (i > 0) json.append(','); json.append("\"item-").append(i).append("\""); } - json.append(']'); - Path file = dir.resolve("big.json"); - Files.writeString(file, json.toString(), StandardCharsets.UTF_8); + return json.append(']').toString(); + } - String raw = tool.read_file(file.toString(), null, null, null); + /** Strip the "%6d\t" line-number prefixes and the truncation marker from content. */ + private static String stripDecorations(String content) { + StringBuilder out = new StringBuilder(); + for (String l : content.split("\n", -1)) { + if (l.isEmpty()) continue; + int tab = l.indexOf('\t'); + String body = tab >= 0 ? l.substring(tab + 1) : l; + int marker = body.indexOf("tool.read_file.line_truncated_marker"); + if (marker >= 0) body = body.substring(0, marker); + out.append(body); + } + return out.toString(); + } + + @Test + @DisplayName("single line larger than the 30KB budget returns clipped content, not empty + infinite loop") + void singleOversizedLine_returnsClippedContent(@TempDir Path dir) throws Exception { + String json = oneLineJsonArray(4000); // ~40KB on one physical line + Path file = dir.resolve("big.json"); + Files.writeString(file, json, StandardCharsets.UTF_8); + + String raw = tool.read_file(file.toString(), null, null, null, null); JSONObject res = JSONUtil.parseObj(raw); assertFalse(res.getBool("error", false), "should not be an error result"); assertTrue(res.getBool("truncated"), "should be marked truncated"); assertTrue(res.getBool("lineTruncated", false), "should flag in-line truncation"); - // The bug: content was empty and readLines was 0. assertEquals(1, res.getInt("readLines"), "must count the clipped line as read"); assertTrue(res.getStr("content").length() > 1000, "content must carry the clipped line, not be empty"); - assertEquals("tool.read_file.line_truncated", res.getStr("message")); + // Continuation must advance into the same line, not loop on column 1. + assertEquals(1, res.getInt("nextStartLine")); + assertTrue(res.getInt("nextStartColumn") > 1, "nextStartColumn must advance past the head"); + } + + @Test + @DisplayName("a very long single line can be fully read by paging through nextStartColumn") + void oversizedLine_pagesToCompletionViaColumn(@TempDir Path dir) throws Exception { + String json = oneLineJsonArray(10000); // big enough to need several windows + Path file = dir.resolve("huge.json"); + Files.writeString(file, json, StandardCharsets.UTF_8); + + StringBuilder reassembled = new StringBuilder(); + Integer startLine = null; + Integer startColumn = null; + int guard = 0; + while (true) { + String raw = tool.read_file(file.toString(), startLine, null, startColumn, null); + JSONObject res = JSONUtil.parseObj(raw); + assertFalse(res.getBool("error", false), "no error while paging"); + reassembled.append(stripDecorations(res.getStr("content"))); + if (!res.getBool("truncated")) { + break; + } + startLine = res.getInt("nextStartLine"); + startColumn = res.containsKey("nextStartColumn") ? res.getInt("nextStartColumn") : 1; + assertTrue(++guard < 50, "must terminate, not loop forever"); + } + assertEquals(json, reassembled.toString(), "paging through columns must reconstruct the whole line"); + } + + @Test + @DisplayName("multi-line file with a huge first line still lets the caller reach later normal lines") + void hugeFirstLine_thenNormalLines_offerNextLine(@TempDir Path dir) throws Exception { + String first = oneLineJsonArray(4000); // oversized line 1 + Path file = dir.resolve("mixed.txt"); + Files.writeString(file, first + "\nsecond-line\nthird-line\n", StandardCharsets.UTF_8); + + // First read clips line 1 and must point both at the line's tail and the next line. + String raw = tool.read_file(file.toString(), null, null, null, null); + JSONObject res = JSONUtil.parseObj(raw); + assertTrue(res.getBool("truncated")); + assertTrue(res.getBool("lineTruncated", false)); + assertEquals(3, res.getInt("totalLines")); + + // The caller can skip the rest of the giant line and read the normal lines. + String raw2 = tool.read_file(file.toString(), 2, null, null, null); + JSONObject res2 = JSONUtil.parseObj(raw2); + assertFalse(res2.getBool("truncated")); + assertEquals(2, res2.getInt("readLines")); + assertTrue(res2.getStr("content").contains("second-line")); + assertTrue(res2.getStr("content").contains("third-line")); + } + + @Test + @DisplayName("single-line spill-style JSON {\"stdout\":\"...\"} is windowed, not dropped") + void spillStyleStdoutJson_isWindowed(@TempDir Path dir) throws Exception { + String payload = "x".repeat(50 * 1024); // 50KB payload on one line + String line = "{\"stdout\":\"" + payload + "\"}"; + Path file = dir.resolve("spill.json"); + Files.writeString(file, line, StandardCharsets.UTF_8); + + String raw = tool.read_file(file.toString(), null, null, null, null); + JSONObject res = JSONUtil.parseObj(raw); + assertTrue(res.getBool("truncated")); + assertTrue(res.getBool("lineTruncated", false)); + assertTrue(res.getStr("content").contains("{\"stdout\":"), "head of the line must be present"); + assertTrue(res.getInt("nextStartColumn") > 1); + } + + @Test + @DisplayName("normal truncation at a line boundary advertises nextStartLine for continuation") + void manyNormalLines_truncateAtLineBoundary(@TempDir Path dir) throws Exception { + // 2000 lines of ~50 chars each well exceeds the 30KB budget but no single + // line is oversized, so truncation must happen at a clean line boundary. + StringBuilder sb = new StringBuilder(); + for (int i = 1; i <= 2000; i++) { + sb.append("line-").append(i).append("-").append("y".repeat(40)).append('\n'); + } + Path file = dir.resolve("many.txt"); + Files.writeString(file, sb.toString(), StandardCharsets.UTF_8); + + String raw = tool.read_file(file.toString(), null, null, null, null); + JSONObject res = JSONUtil.parseObj(raw); + assertTrue(res.getBool("truncated")); + assertFalse(res.getBool("lineTruncated", false), "no individual line is oversized"); + int next = res.getInt("nextStartLine"); + assertEquals(res.getInt("endLine") + 1, next, "continuation must resume right after the last read line"); + assertFalse(res.containsKey("nextStartColumn"), "line-boundary truncation has no column"); } @Test @@ -76,7 +184,7 @@ class ReadFileToolLargeLineTest { Path file = dir.resolve("small.txt"); Files.writeString(file, "alpha\nbeta\ngamma\n", StandardCharsets.UTF_8); - String raw = tool.read_file(file.toString(), null, null, null); + String raw = tool.read_file(file.toString(), null, null, null, null); JSONObject res = JSONUtil.parseObj(raw); assertFalse(res.getBool("truncated"), "small file should not truncate");