opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(repetition-detector): catch markdown-list duplication separated by transition paragraph

Browse Source
This commit is contained in:
matevip 2026-05-03 17:15:33 +08:00
parent a5e7f82a69
commit a2c96946e3
1 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
mateclaw-server/src/main/java/vip/mate/agent/graph/RepetitionDetector.java
View File

@ -53,7 +53,14 @@ public class RepetitionDetector {
* (other) ABC (other)" duplication while keeping the cost bounded.
*/
private static final int SENTENCE_TAIL_COUNT = 3;
private static final int SENTENCE_LOOKBACK = 10;
/**
* Number of historical sentences the tail compares against. Bumped from 10
* to 30 to catch markdown-list duplication where the model emits the same
* 4-6 item list twice with a transition sentence in between — at 10 the
* tail of the second list could not see the corresponding sentence of the
* first list, leaving the duplicate undetected.
*/
private static final int SENTENCE_LOOKBACK = 30;
private static final double JACCARD_THRESHOLD = 0.85;
/**
* Lower bound on the per-sentence token count. Below this both the
@ -68,11 +75,15 @@ public class RepetitionDetector {
private static final int SENTENCE_MIN_BUFFER = 1500;
/**
* Sentence delimiters: full-width Chinese punctuation plus ASCII end-of
* -sentence punctuation and newline. Splitting greedily on any of these
* is good enough for Jaccard's set-of-tokens semantics.
* -sentence punctuation and newline. ASCII '.' '!' '?' are NOT treated as
* sentence boundaries when preceded by a digit — that prevents markdown
* list ordinals like "1." / "2." / "3." from fragmenting a paragraph into
* noise sentences, which used to push the tail past SENTENCE_LOOKBACK and
* mask whole-list duplication. Decimal numbers (e.g. "1.5") are also
* spared by the same rule, which is the desired behavior.
*/
private static final Pattern SENTENCE_SPLIT =
Pattern.compile("[\\u3002\\uff01\\uff1f.!?\\n]+");
Pattern.compile("[\\u3002\\uff01\\uff1f\\n]+|(?<!\\d)[.!?]+");
private final StringBuilder buffer = new StringBuilder();
private boolean repetitionDetected = false;