opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(tool): read_file can page through an oversized single line via startColumn (#190)

Browse Source
This commit is contained in:
matevip 2026-05-21 22:27:09 +08:00
parent 9e93c52d9a
commit 81915ccfae
4 changed files with 203 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
mateclaw-server/src/main/java/vip/mate/tool/builtin/ReadFileTool.java
View File

@ -49,12 +49,15 @@ public class ReadFileTool {
@Tool(description = """
Read the contents of a file. Supports line-range reading (1-based). \
Returns structured JSON with filePath, totalLines, readLines, content. \
Auto-truncates large files with continuation hints. \
Text files only; use extract_document_text for PDF/Office documents.""")
Auto-truncates large files with continuation hints: when the result has \
truncated=true, continue with the returned nextStartLine (and \
nextStartColumn when present, to resume reading the rest of a very long \
line). Text files only; use extract_document_text for PDF/Office documents.""")
public String read_file(
@ToolParam(description = "Absolute or relative file path") String filePath,
@ToolParam(description = "Start line number (1-based, inclusive). Omit to start from line 1", required = false) Integer startLine,
@ToolParam(description = "End line number (1-based, inclusive). Omit to read to EOF or truncation limit", required = false) Integer endLine,
@ToolParam(description = "Start character position within startLine (1-based, inclusive). Used to resume reading the rest of a very long line; pass the nextStartColumn from a previous truncated result. Omit to start at the beginning of the line", required = false) Integer startColumn,
// RFC-063r §2.5: hidden from LLM by JsonSchemaGenerator.
@Nullable ToolContext ctx) {
@ -130,68 +133,108 @@ public class ReadFileTool {
// 提取指定范围的行（转为 0-based）
List<String> selectedLines = allLines.subList(start - 1, end);
// Character offset into the FIRST selected line, used to resume reading
// the tail of a very long line across calls. 1-based on the wire, 0-based
// here. Only applies to the first line of the selection.
int firstLineOffset = (startColumn != null && startColumn > 1) ? startColumn - 1 : 0;
// Truncation control. Each output line carries a "%6d\t" prefix and a
// trailing newline, so the budget available for the line's own text is
// the remaining byte budget minus that overhead.
// trailing newline, so the budget available for a line's own text is the
// remaining budget minus that overhead.
StringBuilder sb = new StringBuilder();
int linesRead = 0;
boolean truncated = false;
boolean lineTruncated = false;
int truncatedLineNum = 0;
// Where a subsequent read_file call should resume. nextLine is 1-based;
// nextColumn is a 1-based char offset (1 = start of the line).
int nextLine = -1;
int nextColumn = 1;
for (int i = 0; i < selectedLines.size(); i++) {
String line = selectedLines.get(i);
int lineNum = start + i;
String fullLine = selectedLines.get(i);
// The offset only applies to the first line of the selection.
int offset = (i == 0) ? Math.min(firstLineOffset, fullLine.length()) : 0;
String line = offset > 0 ? fullLine.substring(offset) : fullLine;
if (linesRead >= DEFAULT_MAX_LINES) {
// Hit the line-count cap; resume at this line from the same offset.
truncated = true;
nextLine = lineNum;
nextColumn = offset + 1;
break;
}
String numberedLine = String.format("%6d\t%s\n", lineNum, line);
if (sb.length() + numberedLine.length() > MAX_OUTPUT_BYTES) {
// This line does not fit in the remaining budget. Normally we
// stop and let the caller continue from the next line. But when
// a single line is itself larger than the whole budget and we
// have read nothing yet, stopping here would return empty
// content with linesRead=0 — and the suggested continuation
// startLine never advances, producing an infinite retry loop.
// Guarantee progress by emitting as much of this oversized line
// as fits, flagged as truncated, then advancing past it.
if (linesRead == 0) {
String prefix = String.format("%6d\t", lineNum);
String marker = i18n.msg("tool.read_file.line_truncated_marker");
int budget = MAX_OUTPUT_BYTES - prefix.length() - marker.length() - 1; // -1 for '\n'
String clipped = safeTruncate(line, Math.max(0, budget));
sb.append(prefix).append(clipped).append(marker).append('\n');
linesRead++;
lineTruncated = true;
truncatedLineNum = lineNum;
}
String prefix = String.format("%6d\t", lineNum);
int lineCost = prefix.length() + line.length() + 1; // +1 for '\n'
if (sb.length() + lineCost <= MAX_OUTPUT_BYTES) {
sb.append(prefix).append(line).append('\n');
linesRead++;
continue;
}
// This line does not fit in the remaining budget.
boolean fitsAlone = prefix.length() + line.length() + 1 <= MAX_OUTPUT_BYTES;
if (fitsAlone || linesRead > 0) {
// EITHER the line would fit in a fresh budget (normal truncation
// at a clean line boundary), OR we have already emitted lines and
// defer this oversized line to the next call. Either way, resume
// at this line; for non-first lines offset is 0 so column is 1.
truncated = true;
nextLine = lineNum;
nextColumn = offset + 1;
break;
}
sb.append(numberedLine);
// linesRead == 0 AND the line is larger than the whole budget even on
// its own. Returning empty content here would yield readLines=0 with a
// continuation hint that never advances — the infinite retry loop from
// the original bug. Emit as much of this line as fits (a window),
// flagged truncated, and advance by exactly the chars consumed so the
// caller can page through the rest of the line with nextStartColumn.
String marker = i18n.msg("tool.read_file.line_truncated_marker");
int windowBudget = MAX_OUTPUT_BYTES - prefix.length() - marker.length() - 1;
String window = safeTruncate(line, Math.max(0, windowBudget));
sb.append(prefix).append(window).append(marker).append('\n');
linesRead++;
truncated = true;
lineTruncated = true;
truncatedLineNum = lineNum;
int consumed = offset + window.length();
if (consumed < fullLine.length()) {
nextLine = lineNum; // more of this line remains
nextColumn = consumed + 1;
} else {
nextLine = lineNum + 1; // line exactly consumed; move on
nextColumn = 1;
}
break;
}
result.set("startLine", start);
result.set("startColumn", firstLineOffset + 1);
result.set("endLine", start + linesRead - 1);
result.set("readLines", linesRead);
result.set("content", sb.toString());
if (truncated) {
result.set("truncated", true);
result.set("nextStartLine", nextLine);
int kb = MAX_OUTPUT_BYTES / 1024;
if (lineTruncated) {
// The oversized line was clipped in place; line ranges cannot
// recover its tail, so do not advertise a startLine that would
// silently skip the remainder.
if (lineTruncated && nextColumn > 1) {
// A long line was windowed and more of it remains. Surface the
// column so the caller can resume reading the same line's tail.
result.set("lineTruncated", true);
result.set("message", i18n.msg("tool.read_file.line_truncated", truncatedLineNum, kb));
result.set("nextStartColumn", nextColumn);
result.set("message", i18n.msg("tool.read_file.line_truncated",
truncatedLineNum, kb, nextLine, nextColumn, nextLine + 1));
} else {
int nextStart = start + linesRead;
result.set("message", i18n.msg("tool.read_file.truncated", DEFAULT_MAX_LINES, kb, nextStart));
if (lineTruncated) {
result.set("lineTruncated", true);
}
result.set("message", i18n.msg("tool.read_file.truncated", DEFAULT_MAX_LINES, kb, nextLine));
}
} else {
result.set("truncated", false);

2
mateclaw-server/src/main/resources/messages.properties
View File

@ -61,7 +61,7 @@ tool.read_file.error.start_gt_end=\u8d77\u59cb\u884c {0} \u5927\u4e8e\u7ed3\u675
tool.read_file.error.read_exception=\u8bfb\u53d6\u6587\u4ef6\u5f02\u5e38: {0}
tool.read_file.truncated=\u8f93\u51fa\u5df2\u622a\u65ad\uff08\u6700\u591a {0} \u884c / {1}KB\uff09\u3002\u4f7f\u7528 startLine={2} \u7ee7\u7eed\u8bfb\u53d6\u3002
tool.read_file.line_truncated_marker= ...[\u672c\u884c\u8fc7\u957f\uff0c\u5df2\u622a\u65ad]
tool.read_file.line_truncated=\u7b2c {0} \u884c\u957f\u5ea6\u8d85\u8fc7\u5355\u6b21\u8f93\u51fa\u4e0a\u9650\uff08{1}KB\uff09\uff0c\u5df2\u622a\u65ad\u663e\u793a\u3002\u8be5\u884c\u5b8c\u6574\u5185\u5bb9\u65e0\u6cd5\u901a\u8fc7\u884c\u53f7\u8303\u56f4\u9010\u6b65\u8bfb\u53d6\uff1b\u5982\u9700\u5b8c\u6574\u6570\u636e\u8bf7\u4f7f\u7528 execute_shell_command\u3002\u4e0d\u8981\u63a8\u65ad\u6216\u865a\u6784\u88ab\u622a\u65ad\u7684\u5185\u5bb9\u3002
tool.read_file.line_truncated=\u7b2c {0} \u884c\u957f\u5ea6\u8d85\u8fc7\u5355\u6b21\u8f93\u51fa\u4e0a\u9650\uff08{1}KB\uff09\u3002\u8981\u7ee7\u7eed\u8bfb\u53d6\u8be5\u884c\u5269\u4f59\u5185\u5bb9\uff0c\u8bf7\u4f7f\u7528 startLine={2}\u3001startColumn={3}\uff1b\u6216\u4f7f\u7528 startLine={4} \u8df3\u5230\u4e0b\u4e00\u884c\u3002\u4e0d\u8981\u63a8\u65ad\u6216\u865a\u6784\u88ab\u622a\u65ad\u7684\u5185\u5bb9\u3002
tool.write_file.error.path_empty=\u6587\u4ef6\u8def\u5f84\u4e0d\u80fd\u4e3a\u7a7a
tool.write_file.error.is_directory=\u8def\u5f84\u662f\u4e00\u4e2a\u5df2\u6709\u76ee\u5f55\uff0c\u65e0\u6cd5\u4f5c\u4e3a\u6587\u4ef6\u5199\u5165: {0}
tool.write_file.error.write_exception=\u5199\u5165\u6587\u4ef6\u5f02\u5e38: {0}

2
mateclaw-server/src/main/resources/messages_en.properties
View File

@ -61,7 +61,7 @@ tool.read_file.error.start_gt_end=Start line {0} is greater than end line {1}
tool.read_file.error.read_exception=Read file exception: {0}
tool.read_file.truncated=Output truncated (max {0} lines / {1}KB). Use startLine={2} to continue reading.
tool.read_file.line_truncated_marker= ...[line too long, truncated]
tool.read_file.line_truncated=Line {0} exceeds the single-output limit ({1}KB) and was truncated. Its full content cannot be read via line ranges; use execute_shell_command if you need the complete data. Do NOT infer or fabricate the omitted content.
tool.read_file.line_truncated=Line {0} exceeds the {1}KB single-output limit. To read the rest of this line, continue with startLine={2}, startColumn={3}; or skip to startLine={4} for the next line. Do NOT infer or fabricate the omitted content.
tool.write_file.error.path_empty=File path cannot be empty
tool.write_file.error.is_directory=Path is an existing directory, cannot write as file: {0}
tool.write_file.error.write_exception=Write file exception: {0}

142
mateclaw-server/src/test/java/vip/mate/tool/builtin/ReadFileToolLargeLineTest.java
View File

@ -19,11 +19,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Regression test for the single-line oversized-file bug: a file whose only
* line exceeds the output byte budget used to return empty content with
* readLines=0 and a continuation hint that never advanced — an infinite retry
* loop. The tool must instead return a clipped, clearly-flagged result and make
* progress.
* Regression tests for reading oversized content with {@link ReadFileTool}.
* <p>
* The original bug: a file whose only line exceeds the output budget returned
* empty content with readLines=0 and a continuation hint (startLine) that never
* advanced — an infinite retry loop. These tests assert that the tool always
* makes progress, clearly flags truncation, and lets a caller page through both
* a very long single line (via nextStartColumn) and subsequent normal lines
* (via nextStartLine).
*/
class ReadFileToolLargeLineTest {
@ -45,29 +48,134 @@ class ReadFileToolLargeLineTest {
ToolExecutionContext.clear();
}
@Test
@DisplayName("single line larger than the 30KB budget returns clipped content, not empty + infinite loop")
void singleOversizedLine_returnsClippedContent(@TempDir Path dir) throws Exception {
// ~40KB single-line JSON array on one physical line.
/** Build a single-line JSON array of {@code n} string elements. */
private static String oneLineJsonArray(int n) {
StringBuilder json = new StringBuilder("[");
for (int i = 0; i < 4000; i++) {
for (int i = 0; i < n; i++) {
if (i > 0) json.append(',');
json.append("\"item-").append(i).append("\"");
}
json.append(']');
Path file = dir.resolve("big.json");
Files.writeString(file, json.toString(), StandardCharsets.UTF_8);
return json.append(']').toString();
}
String raw = tool.read_file(file.toString(), null, null, null);
/** Strip the "%6d\t" line-number prefixes and the truncation marker from content. */
private static String stripDecorations(String content) {
StringBuilder out = new StringBuilder();
for (String l : content.split("\n", -1)) {
if (l.isEmpty()) continue;
int tab = l.indexOf('\t');
String body = tab >= 0 ? l.substring(tab + 1) : l;
int marker = body.indexOf("tool.read_file.line_truncated_marker");
if (marker >= 0) body = body.substring(0, marker);
out.append(body);
}
return out.toString();
}
@Test
@DisplayName("single line larger than the 30KB budget returns clipped content, not empty + infinite loop")
void singleOversizedLine_returnsClippedContent(@TempDir Path dir) throws Exception {
String json = oneLineJsonArray(4000); // ~40KB on one physical line
Path file = dir.resolve("big.json");
Files.writeString(file, json, StandardCharsets.UTF_8);
String raw = tool.read_file(file.toString(), null, null, null, null);
JSONObject res = JSONUtil.parseObj(raw);
assertFalse(res.getBool("error", false), "should not be an error result");
assertTrue(res.getBool("truncated"), "should be marked truncated");
assertTrue(res.getBool("lineTruncated", false), "should flag in-line truncation");
// The bug: content was empty and readLines was 0.
assertEquals(1, res.getInt("readLines"), "must count the clipped line as read");
assertTrue(res.getStr("content").length() > 1000, "content must carry the clipped line, not be empty");
assertEquals("tool.read_file.line_truncated", res.getStr("message"));
// Continuation must advance into the same line, not loop on column 1.
assertEquals(1, res.getInt("nextStartLine"));
assertTrue(res.getInt("nextStartColumn") > 1, "nextStartColumn must advance past the head");
}
@Test
@DisplayName("a very long single line can be fully read by paging through nextStartColumn")
void oversizedLine_pagesToCompletionViaColumn(@TempDir Path dir) throws Exception {
String json = oneLineJsonArray(10000); // big enough to need several windows
Path file = dir.resolve("huge.json");
Files.writeString(file, json, StandardCharsets.UTF_8);
StringBuilder reassembled = new StringBuilder();
Integer startLine = null;
Integer startColumn = null;
int guard = 0;
while (true) {
String raw = tool.read_file(file.toString(), startLine, null, startColumn, null);
JSONObject res = JSONUtil.parseObj(raw);
assertFalse(res.getBool("error", false), "no error while paging");
reassembled.append(stripDecorations(res.getStr("content")));
if (!res.getBool("truncated")) {
break;
}
startLine = res.getInt("nextStartLine");
startColumn = res.containsKey("nextStartColumn") ? res.getInt("nextStartColumn") : 1;
assertTrue(++guard < 50, "must terminate, not loop forever");
}
assertEquals(json, reassembled.toString(), "paging through columns must reconstruct the whole line");
}
@Test
@DisplayName("multi-line file with a huge first line still lets the caller reach later normal lines")
void hugeFirstLine_thenNormalLines_offerNextLine(@TempDir Path dir) throws Exception {
String first = oneLineJsonArray(4000); // oversized line 1
Path file = dir.resolve("mixed.txt");
Files.writeString(file, first + "\nsecond-line\nthird-line\n", StandardCharsets.UTF_8);
// First read clips line 1 and must point both at the line's tail and the next line.
String raw = tool.read_file(file.toString(), null, null, null, null);
JSONObject res = JSONUtil.parseObj(raw);
assertTrue(res.getBool("truncated"));
assertTrue(res.getBool("lineTruncated", false));
assertEquals(3, res.getInt("totalLines"));
// The caller can skip the rest of the giant line and read the normal lines.
String raw2 = tool.read_file(file.toString(), 2, null, null, null);
JSONObject res2 = JSONUtil.parseObj(raw2);
assertFalse(res2.getBool("truncated"));
assertEquals(2, res2.getInt("readLines"));
assertTrue(res2.getStr("content").contains("second-line"));
assertTrue(res2.getStr("content").contains("third-line"));
}
@Test
@DisplayName("single-line spill-style JSON {\"stdout\":\"...\"} is windowed, not dropped")
void spillStyleStdoutJson_isWindowed(@TempDir Path dir) throws Exception {
String payload = "x".repeat(50 * 1024); // 50KB payload on one line
String line = "{\"stdout\":\"" + payload + "\"}";
Path file = dir.resolve("spill.json");
Files.writeString(file, line, StandardCharsets.UTF_8);
String raw = tool.read_file(file.toString(), null, null, null, null);
JSONObject res = JSONUtil.parseObj(raw);
assertTrue(res.getBool("truncated"));
assertTrue(res.getBool("lineTruncated", false));
assertTrue(res.getStr("content").contains("{\"stdout\":"), "head of the line must be present");
assertTrue(res.getInt("nextStartColumn") > 1);
}
@Test
@DisplayName("normal truncation at a line boundary advertises nextStartLine for continuation")
void manyNormalLines_truncateAtLineBoundary(@TempDir Path dir) throws Exception {
// 2000 lines of ~50 chars each well exceeds the 30KB budget but no single
// line is oversized, so truncation must happen at a clean line boundary.
StringBuilder sb = new StringBuilder();
for (int i = 1; i <= 2000; i++) {
sb.append("line-").append(i).append("-").append("y".repeat(40)).append('\n');
}
Path file = dir.resolve("many.txt");
Files.writeString(file, sb.toString(), StandardCharsets.UTF_8);
String raw = tool.read_file(file.toString(), null, null, null, null);
JSONObject res = JSONUtil.parseObj(raw);
assertTrue(res.getBool("truncated"));
assertFalse(res.getBool("lineTruncated", false), "no individual line is oversized");
int next = res.getInt("nextStartLine");
assertEquals(res.getInt("endLine") + 1, next, "continuation must resume right after the last read line");
assertFalse(res.containsKey("nextStartColumn"), "line-boundary truncation has no column");
}
@Test
@ -76,7 +184,7 @@ class ReadFileToolLargeLineTest {
Path file = dir.resolve("small.txt");
Files.writeString(file, "alpha\nbeta\ngamma\n", StandardCharsets.UTF_8);
String raw = tool.read_file(file.toString(), null, null, null);
String raw = tool.read_file(file.toString(), null, null, null, null);
JSONObject res = JSONUtil.parseObj(raw);
assertFalse(res.getBool("truncated"), "small file should not truncate");