mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 03:55:09 +08:00
feat(wiki): schema validator for structured page metadata
This commit is contained in:
matevip
parent
7f4987c30a
commit
7cd9971596
@ -0,0 +1,183 @@
|
|||||||
|
package vip.mate.wiki.profile;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validates a page's raw metadata against its pageType field schema.
|
||||||
|
*
|
||||||
|
* <p>Policy (non-blocking — ingest is never failed by metadata issues):
|
||||||
|
* <ul>
|
||||||
|
* <li>Required field missing → {@code warning}, field omitted.</li>
|
||||||
|
* <li>Type mismatch → attempt light coercion; on failure keep the raw value
|
||||||
|
* and emit a warning.</li>
|
||||||
|
* <li>Undeclared field → dropped unless the profile allows additional
|
||||||
|
* fields; a dropped field always emits a warning.</li>
|
||||||
|
* <li>Enum value outside the allowed set / malformed date → warning, value
|
||||||
|
* kept as-is.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Service
|
||||||
|
public class WikiMetadataValidator {
|
||||||
|
|
||||||
|
private static final Pattern ISO_DATE = Pattern.compile("\\d{4}-\\d{2}-\\d{2}");
|
||||||
|
|
||||||
|
/** Validation status persisted to {@code metadata_validation_status}. */
|
||||||
|
public static final String OK = "ok";
|
||||||
|
public static final String WARNING = "warning";
|
||||||
|
|
||||||
|
/** One validation warning. Serialized to {@code metadata_validation_json}. */
|
||||||
|
@Data
|
||||||
|
public static class FieldWarning {
|
||||||
|
private final String field;
|
||||||
|
private final String reason;
|
||||||
|
private final String source;
|
||||||
|
private final String rawValuePreview;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public static class ValidationResult {
|
||||||
|
private final Map<String, Object> cleaned;
|
||||||
|
private final String status;
|
||||||
|
private final List<FieldWarning> warnings;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate {@code raw} metadata against {@code def}'s schema.
|
||||||
|
*
|
||||||
|
* @param def the pageType definition (may be {@code null} → no schema)
|
||||||
|
* @param raw the raw metadata produced by the LLM (may be {@code null})
|
||||||
|
* @param allowAdditional whether undeclared fields are kept
|
||||||
|
* @param source stage label recorded on each warning (route/create/merge)
|
||||||
|
*/
|
||||||
|
public ValidationResult validate(WikiPageTypeDef def, Map<String, Object> raw,
|
||||||
|
boolean allowAdditional, String source) {
|
||||||
|
Map<String, Object> cleaned = new LinkedHashMap<>();
|
||||||
|
List<FieldWarning> warnings = new ArrayList<>();
|
||||||
|
Map<String, Object> input = raw == null ? Map.of() : raw;
|
||||||
|
Map<String, WikiFieldSchema> schema = (def == null || def.getSchema() == null)
|
||||||
|
? Map.of() : def.getSchema();
|
||||||
|
|
||||||
|
// 1. Declared fields: validate / coerce in schema order.
|
||||||
|
for (Map.Entry<String, WikiFieldSchema> entry : schema.entrySet()) {
|
||||||
|
String field = entry.getKey();
|
||||||
|
WikiFieldSchema fieldSchema = entry.getValue();
|
||||||
|
boolean present = input.containsKey(field) && input.get(field) != null;
|
||||||
|
if (!present) {
|
||||||
|
if (fieldSchema.isRequired()) {
|
||||||
|
warnings.add(new FieldWarning(field, "required field missing", source, null));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Object value = input.get(field);
|
||||||
|
Coerced coerced = coerce(fieldSchema, value);
|
||||||
|
if (coerced.warning != null) {
|
||||||
|
warnings.add(new FieldWarning(field, coerced.warning, source, preview(value)));
|
||||||
|
}
|
||||||
|
cleaned.put(field, coerced.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Undeclared fields: keep or drop.
|
||||||
|
for (Map.Entry<String, Object> entry : input.entrySet()) {
|
||||||
|
String field = entry.getKey();
|
||||||
|
if (schema.containsKey(field)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (allowAdditional) {
|
||||||
|
cleaned.put(field, entry.getValue());
|
||||||
|
} else {
|
||||||
|
warnings.add(new FieldWarning(field, "dropped: not declared in schema",
|
||||||
|
source, preview(entry.getValue())));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String status = warnings.isEmpty() ? OK : WARNING;
|
||||||
|
return new ValidationResult(cleaned, status, warnings);
|
||||||
|
}
|
||||||
|
|
||||||
|
private record Coerced(Object value, String warning) {}
|
||||||
|
|
||||||
|
private Coerced coerce(WikiFieldSchema schema, Object value) {
|
||||||
|
String type = schema.getType() == null ? "string" : schema.getType().trim().toLowerCase();
|
||||||
|
return switch (type) {
|
||||||
|
case "string" -> new Coerced(String.valueOf(value), null);
|
||||||
|
case "number" -> coerceNumber(value);
|
||||||
|
case "boolean" -> coerceBoolean(value);
|
||||||
|
case "date" -> coerceDate(value);
|
||||||
|
case "enum" -> coerceEnum(schema, value);
|
||||||
|
case "string_array" -> coerceStringArray(value);
|
||||||
|
default -> new Coerced(value, null);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private Coerced coerceNumber(Object value) {
|
||||||
|
if (value instanceof Number) {
|
||||||
|
return new Coerced(value, null);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String s = String.valueOf(value).trim();
|
||||||
|
if (s.contains(".")) {
|
||||||
|
return new Coerced(Double.parseDouble(s), null);
|
||||||
|
}
|
||||||
|
return new Coerced(Long.parseLong(s), null);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
return new Coerced(value, "expected number");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Coerced coerceBoolean(Object value) {
|
||||||
|
if (value instanceof Boolean) {
|
||||||
|
return new Coerced(value, null);
|
||||||
|
}
|
||||||
|
String s = String.valueOf(value).trim().toLowerCase();
|
||||||
|
if ("true".equals(s) || "false".equals(s)) {
|
||||||
|
return new Coerced(Boolean.valueOf(s), null);
|
||||||
|
}
|
||||||
|
return new Coerced(value, "expected boolean");
|
||||||
|
}
|
||||||
|
|
||||||
|
private Coerced coerceDate(Object value) {
|
||||||
|
String s = String.valueOf(value).trim();
|
||||||
|
if (ISO_DATE.matcher(s).matches()) {
|
||||||
|
return new Coerced(s, null);
|
||||||
|
}
|
||||||
|
return new Coerced(value, "expected ISO date YYYY-MM-DD");
|
||||||
|
}
|
||||||
|
|
||||||
|
private Coerced coerceEnum(WikiFieldSchema schema, Object value) {
|
||||||
|
String s = String.valueOf(value);
|
||||||
|
List<String> values = schema.getValues();
|
||||||
|
if (values == null || values.contains(s)) {
|
||||||
|
return new Coerced(s, null);
|
||||||
|
}
|
||||||
|
return new Coerced(s, "value not in allowed enum set");
|
||||||
|
}
|
||||||
|
|
||||||
|
private Coerced coerceStringArray(Object value) {
|
||||||
|
if (value instanceof List<?> list) {
|
||||||
|
List<String> out = new ArrayList<>(list.size());
|
||||||
|
for (Object o : list) {
|
||||||
|
out.add(String.valueOf(o));
|
||||||
|
}
|
||||||
|
return new Coerced(out, null);
|
||||||
|
}
|
||||||
|
// A single scalar is accepted as a one-element array.
|
||||||
|
return new Coerced(List.of(String.valueOf(value)), null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String preview(Object value) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String s = String.valueOf(value);
|
||||||
|
return s.length() > 120 ? s.substring(0, 120) + "…" : s;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,137 @@
|
|||||||
|
package vip.mate.wiki.profile;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit tests for {@link WikiMetadataValidator} covering required/type/enum/date
|
||||||
|
* rules, coercion, and undeclared-field handling.
|
||||||
|
*/
|
||||||
|
class WikiMetadataValidatorTest {
|
||||||
|
|
||||||
|
private final WikiMetadataValidator validator = new WikiMetadataValidator();
|
||||||
|
|
||||||
|
private WikiPageTypeDef episodeDef() {
|
||||||
|
WikiPageTypeDef def = new WikiPageTypeDef();
|
||||||
|
Map<String, WikiFieldSchema> schema = new LinkedHashMap<>();
|
||||||
|
schema.put("event_type", field("string", true, null));
|
||||||
|
schema.put("event_date", field("date", true, null));
|
||||||
|
schema.put("significance", field("enum", false, List.of("low", "medium", "high")));
|
||||||
|
schema.put("cited_count", field("number", false, null));
|
||||||
|
def.setSchema(schema);
|
||||||
|
return def;
|
||||||
|
}
|
||||||
|
|
||||||
|
private WikiFieldSchema field(String type, boolean required, List<String> values) {
|
||||||
|
WikiFieldSchema f = new WikiFieldSchema();
|
||||||
|
f.setType(type);
|
||||||
|
f.setRequired(required);
|
||||||
|
f.setValues(values);
|
||||||
|
return f;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void validMetadata_isOk() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("event_type", "liquidity_shock");
|
||||||
|
raw.put("event_date", "2024-09-18");
|
||||||
|
raw.put("significance", "high");
|
||||||
|
raw.put("cited_count", 12);
|
||||||
|
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||||
|
|
||||||
|
assertEquals(WikiMetadataValidator.OK, r.getStatus());
|
||||||
|
assertTrue(r.getWarnings().isEmpty());
|
||||||
|
assertEquals("liquidity_shock", r.getCleaned().get("event_type"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void requiredMissing_warnsButKeepsGoing() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("event_type", "x");
|
||||||
|
// event_date missing
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||||
|
|
||||||
|
assertEquals(WikiMetadataValidator.WARNING, r.getStatus());
|
||||||
|
assertTrue(r.getWarnings().stream().anyMatch(w ->
|
||||||
|
w.getField().equals("event_date") && w.getReason().contains("required")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void numberCoercedFromString() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("event_type", "x");
|
||||||
|
raw.put("event_date", "2024-01-01");
|
||||||
|
raw.put("cited_count", "42");
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||||
|
|
||||||
|
assertEquals(42L, r.getCleaned().get("cited_count"));
|
||||||
|
assertEquals(WikiMetadataValidator.OK, r.getStatus());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void badDate_warns() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("event_type", "x");
|
||||||
|
raw.put("event_date", "Sept 2024");
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||||
|
|
||||||
|
assertTrue(r.getWarnings().stream().anyMatch(w ->
|
||||||
|
w.getField().equals("event_date") && w.getReason().contains("ISO date")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void enumOutOfRange_warns() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("event_type", "x");
|
||||||
|
raw.put("event_date", "2024-01-01");
|
||||||
|
raw.put("significance", "catastrophic");
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||||
|
|
||||||
|
assertTrue(r.getWarnings().stream().anyMatch(w ->
|
||||||
|
w.getField().equals("significance") && w.getReason().contains("enum")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void undeclaredField_droppedWithWarning_whenNotAllowed() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("event_type", "x");
|
||||||
|
raw.put("event_date", "2024-01-01");
|
||||||
|
raw.put("rogue", "surprise");
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "route");
|
||||||
|
|
||||||
|
assertFalse(r.getCleaned().containsKey("rogue"));
|
||||||
|
WikiMetadataValidator.FieldWarning w = r.getWarnings().stream()
|
||||||
|
.filter(x -> x.getField().equals("rogue")).findFirst().orElseThrow();
|
||||||
|
assertTrue(w.getReason().contains("dropped"));
|
||||||
|
assertEquals("route", w.getSource());
|
||||||
|
assertEquals("surprise", w.getRawValuePreview());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void undeclaredField_keptWhenAllowed() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("event_type", "x");
|
||||||
|
raw.put("event_date", "2024-01-01");
|
||||||
|
raw.put("extra", "kept");
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, true, "create");
|
||||||
|
|
||||||
|
assertEquals("kept", r.getCleaned().get("extra"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void nullDef_keepsAllFields() {
|
||||||
|
Map<String, Object> raw = new LinkedHashMap<>();
|
||||||
|
raw.put("anything", "goes");
|
||||||
|
WikiMetadataValidator.ValidationResult r = validator.validate(null, raw, false, "create");
|
||||||
|
// No schema → additional fields dropped (allowAdditional=false) with warning
|
||||||
|
assertTrue(r.getWarnings().stream().anyMatch(w -> w.getField().equals("anything")));
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user