opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-15 20:08:18 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(repetition-detector): catch markdown-list duplication separated by transition paragraph

Browse Source
This commit is contained in:
matevip 2026-05-03 17:15:33 +08:00
parent a5e7f82a69
commit a2c96946e3
1 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
mateclaw-server/src/main/java/vip/mate/agent/graph/RepetitionDetector.java
View File

@ -53,7 +53,14 @@ public class RepetitionDetector {
* (other) ABC (other)" duplication while keeping the cost bounded. * (other) ABC (other)" duplication while keeping the cost bounded.
*/ */
private static final int SENTENCE_TAIL_COUNT = 3; private static final int SENTENCE_TAIL_COUNT = 3;
private static final int SENTENCE_LOOKBACK = 10; /**
* Number of historical sentences the tail compares against. Bumped from 10
* to 30 to catch markdown-list duplication where the model emits the same
* 4-6 item list twice with a transition sentence in between — at 10 the
* tail of the second list could not see the corresponding sentence of the
* first list, leaving the duplicate undetected.
*/
private static final int SENTENCE_LOOKBACK = 30;
private static final double JACCARD_THRESHOLD = 0.85; private static final double JACCARD_THRESHOLD = 0.85;
/** /**
* Lower bound on the per-sentence token count. Below this both the * Lower bound on the per-sentence token count. Below this both the
@ -68,11 +75,15 @@ public class RepetitionDetector {
private static final int SENTENCE_MIN_BUFFER = 1500; private static final int SENTENCE_MIN_BUFFER = 1500;
/** /**
* Sentence delimiters: full-width Chinese punctuation plus ASCII end-of * Sentence delimiters: full-width Chinese punctuation plus ASCII end-of
* -sentence punctuation and newline. Splitting greedily on any of these * -sentence punctuation and newline. ASCII '.' '!' '?' are NOT treated as
* is good enough for Jaccard's set-of-tokens semantics. * sentence boundaries when preceded by a digit — that prevents markdown
* list ordinals like "1." / "2." / "3." from fragmenting a paragraph into
* noise sentences, which used to push the tail past SENTENCE_LOOKBACK and
* mask whole-list duplication. Decimal numbers (e.g. "1.5") are also
* spared by the same rule, which is the desired behavior.
*/ */
private static final Pattern SENTENCE_SPLIT = private static final Pattern SENTENCE_SPLIT =
Pattern.compile("[\\u3002\\uff01\\uff1f.!?\\n]+"); Pattern.compile("[\\u3002\\uff01\\uff1f\\n]+|(?<!\\d)[.!?]+");
private final StringBuilder buffer = new StringBuilder(); private final StringBuilder buffer = new StringBuilder();
private boolean repetitionDetected = false; private boolean repetitionDetected = false;