diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/profile/WikiMetadataValidator.java b/mateclaw-server/src/main/java/vip/mate/wiki/profile/WikiMetadataValidator.java new file mode 100644 index 00000000..7677fa83 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/profile/WikiMetadataValidator.java @@ -0,0 +1,183 @@ +package vip.mate.wiki.profile; + +import lombok.Data; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Validates a page's raw metadata against its pageType field schema. + * + *

Policy (non-blocking — ingest is never failed by metadata issues): + *

+ * + * @author MateClaw Team + */ +@Service +public class WikiMetadataValidator { + + private static final Pattern ISO_DATE = Pattern.compile("\\d{4}-\\d{2}-\\d{2}"); + + /** Validation status persisted to {@code metadata_validation_status}. */ + public static final String OK = "ok"; + public static final String WARNING = "warning"; + + /** One validation warning. Serialized to {@code metadata_validation_json}. */ + @Data + public static class FieldWarning { + private final String field; + private final String reason; + private final String source; + private final String rawValuePreview; + } + + @Data + public static class ValidationResult { + private final Map cleaned; + private final String status; + private final List warnings; + } + + /** + * Validate {@code raw} metadata against {@code def}'s schema. + * + * @param def the pageType definition (may be {@code null} → no schema) + * @param raw the raw metadata produced by the LLM (may be {@code null}) + * @param allowAdditional whether undeclared fields are kept + * @param source stage label recorded on each warning (route/create/merge) + */ + public ValidationResult validate(WikiPageTypeDef def, Map raw, + boolean allowAdditional, String source) { + Map cleaned = new LinkedHashMap<>(); + List warnings = new ArrayList<>(); + Map input = raw == null ? Map.of() : raw; + Map schema = (def == null || def.getSchema() == null) + ? Map.of() : def.getSchema(); + + // 1. Declared fields: validate / coerce in schema order. + for (Map.Entry entry : schema.entrySet()) { + String field = entry.getKey(); + WikiFieldSchema fieldSchema = entry.getValue(); + boolean present = input.containsKey(field) && input.get(field) != null; + if (!present) { + if (fieldSchema.isRequired()) { + warnings.add(new FieldWarning(field, "required field missing", source, null)); + } + continue; + } + Object value = input.get(field); + Coerced coerced = coerce(fieldSchema, value); + if (coerced.warning != null) { + warnings.add(new FieldWarning(field, coerced.warning, source, preview(value))); + } + cleaned.put(field, coerced.value); + } + + // 2. Undeclared fields: keep or drop. + for (Map.Entry entry : input.entrySet()) { + String field = entry.getKey(); + if (schema.containsKey(field)) { + continue; + } + if (allowAdditional) { + cleaned.put(field, entry.getValue()); + } else { + warnings.add(new FieldWarning(field, "dropped: not declared in schema", + source, preview(entry.getValue()))); + } + } + + String status = warnings.isEmpty() ? OK : WARNING; + return new ValidationResult(cleaned, status, warnings); + } + + private record Coerced(Object value, String warning) {} + + private Coerced coerce(WikiFieldSchema schema, Object value) { + String type = schema.getType() == null ? "string" : schema.getType().trim().toLowerCase(); + return switch (type) { + case "string" -> new Coerced(String.valueOf(value), null); + case "number" -> coerceNumber(value); + case "boolean" -> coerceBoolean(value); + case "date" -> coerceDate(value); + case "enum" -> coerceEnum(schema, value); + case "string_array" -> coerceStringArray(value); + default -> new Coerced(value, null); + }; + } + + private Coerced coerceNumber(Object value) { + if (value instanceof Number) { + return new Coerced(value, null); + } + try { + String s = String.valueOf(value).trim(); + if (s.contains(".")) { + return new Coerced(Double.parseDouble(s), null); + } + return new Coerced(Long.parseLong(s), null); + } catch (NumberFormatException e) { + return new Coerced(value, "expected number"); + } + } + + private Coerced coerceBoolean(Object value) { + if (value instanceof Boolean) { + return new Coerced(value, null); + } + String s = String.valueOf(value).trim().toLowerCase(); + if ("true".equals(s) || "false".equals(s)) { + return new Coerced(Boolean.valueOf(s), null); + } + return new Coerced(value, "expected boolean"); + } + + private Coerced coerceDate(Object value) { + String s = String.valueOf(value).trim(); + if (ISO_DATE.matcher(s).matches()) { + return new Coerced(s, null); + } + return new Coerced(value, "expected ISO date YYYY-MM-DD"); + } + + private Coerced coerceEnum(WikiFieldSchema schema, Object value) { + String s = String.valueOf(value); + List values = schema.getValues(); + if (values == null || values.contains(s)) { + return new Coerced(s, null); + } + return new Coerced(s, "value not in allowed enum set"); + } + + private Coerced coerceStringArray(Object value) { + if (value instanceof List list) { + List out = new ArrayList<>(list.size()); + for (Object o : list) { + out.add(String.valueOf(o)); + } + return new Coerced(out, null); + } + // A single scalar is accepted as a one-element array. + return new Coerced(List.of(String.valueOf(value)), null); + } + + private static String preview(Object value) { + if (value == null) { + return null; + } + String s = String.valueOf(value); + return s.length() > 120 ? s.substring(0, 120) + "…" : s; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/profile/WikiMetadataValidatorTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/profile/WikiMetadataValidatorTest.java new file mode 100644 index 00000000..da81b857 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/profile/WikiMetadataValidatorTest.java @@ -0,0 +1,137 @@ +package vip.mate.wiki.profile; + +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for {@link WikiMetadataValidator} covering required/type/enum/date + * rules, coercion, and undeclared-field handling. + */ +class WikiMetadataValidatorTest { + + private final WikiMetadataValidator validator = new WikiMetadataValidator(); + + private WikiPageTypeDef episodeDef() { + WikiPageTypeDef def = new WikiPageTypeDef(); + Map schema = new LinkedHashMap<>(); + schema.put("event_type", field("string", true, null)); + schema.put("event_date", field("date", true, null)); + schema.put("significance", field("enum", false, List.of("low", "medium", "high"))); + schema.put("cited_count", field("number", false, null)); + def.setSchema(schema); + return def; + } + + private WikiFieldSchema field(String type, boolean required, List values) { + WikiFieldSchema f = new WikiFieldSchema(); + f.setType(type); + f.setRequired(required); + f.setValues(values); + return f; + } + + @Test + void validMetadata_isOk() { + Map raw = new LinkedHashMap<>(); + raw.put("event_type", "liquidity_shock"); + raw.put("event_date", "2024-09-18"); + raw.put("significance", "high"); + raw.put("cited_count", 12); + + WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create"); + + assertEquals(WikiMetadataValidator.OK, r.getStatus()); + assertTrue(r.getWarnings().isEmpty()); + assertEquals("liquidity_shock", r.getCleaned().get("event_type")); + } + + @Test + void requiredMissing_warnsButKeepsGoing() { + Map raw = new LinkedHashMap<>(); + raw.put("event_type", "x"); + // event_date missing + WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create"); + + assertEquals(WikiMetadataValidator.WARNING, r.getStatus()); + assertTrue(r.getWarnings().stream().anyMatch(w -> + w.getField().equals("event_date") && w.getReason().contains("required"))); + } + + @Test + void numberCoercedFromString() { + Map raw = new LinkedHashMap<>(); + raw.put("event_type", "x"); + raw.put("event_date", "2024-01-01"); + raw.put("cited_count", "42"); + WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create"); + + assertEquals(42L, r.getCleaned().get("cited_count")); + assertEquals(WikiMetadataValidator.OK, r.getStatus()); + } + + @Test + void badDate_warns() { + Map raw = new LinkedHashMap<>(); + raw.put("event_type", "x"); + raw.put("event_date", "Sept 2024"); + WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create"); + + assertTrue(r.getWarnings().stream().anyMatch(w -> + w.getField().equals("event_date") && w.getReason().contains("ISO date"))); + } + + @Test + void enumOutOfRange_warns() { + Map raw = new LinkedHashMap<>(); + raw.put("event_type", "x"); + raw.put("event_date", "2024-01-01"); + raw.put("significance", "catastrophic"); + WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create"); + + assertTrue(r.getWarnings().stream().anyMatch(w -> + w.getField().equals("significance") && w.getReason().contains("enum"))); + } + + @Test + void undeclaredField_droppedWithWarning_whenNotAllowed() { + Map raw = new LinkedHashMap<>(); + raw.put("event_type", "x"); + raw.put("event_date", "2024-01-01"); + raw.put("rogue", "surprise"); + WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "route"); + + assertFalse(r.getCleaned().containsKey("rogue")); + WikiMetadataValidator.FieldWarning w = r.getWarnings().stream() + .filter(x -> x.getField().equals("rogue")).findFirst().orElseThrow(); + assertTrue(w.getReason().contains("dropped")); + assertEquals("route", w.getSource()); + assertEquals("surprise", w.getRawValuePreview()); + } + + @Test + void undeclaredField_keptWhenAllowed() { + Map raw = new LinkedHashMap<>(); + raw.put("event_type", "x"); + raw.put("event_date", "2024-01-01"); + raw.put("extra", "kept"); + WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, true, "create"); + + assertEquals("kept", r.getCleaned().get("extra")); + } + + @Test + void nullDef_keepsAllFields() { + Map raw = new LinkedHashMap<>(); + raw.put("anything", "goes"); + WikiMetadataValidator.ValidationResult r = validator.validate(null, raw, false, "create"); + // No schema → additional fields dropped (allowAdditional=false) with warning + assertTrue(r.getWarnings().stream().anyMatch(w -> w.getField().equals("anything"))); + } +}