mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(wiki): schema validator for structured page metadata
This commit is contained in:
matevip
parent
7f4987c30a
commit
7cd9971596
@ -0,0 +1,183 @@
|
||||
package vip.mate.wiki.profile;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Validates a page's raw metadata against its pageType field schema.
|
||||
*
|
||||
* <p>Policy (non-blocking — ingest is never failed by metadata issues):
|
||||
* <ul>
|
||||
* <li>Required field missing → {@code warning}, field omitted.</li>
|
||||
* <li>Type mismatch → attempt light coercion; on failure keep the raw value
|
||||
* and emit a warning.</li>
|
||||
* <li>Undeclared field → dropped unless the profile allows additional
|
||||
* fields; a dropped field always emits a warning.</li>
|
||||
* <li>Enum value outside the allowed set / malformed date → warning, value
|
||||
* kept as-is.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Service
|
||||
public class WikiMetadataValidator {
|
||||
|
||||
private static final Pattern ISO_DATE = Pattern.compile("\\d{4}-\\d{2}-\\d{2}");
|
||||
|
||||
/** Validation status persisted to {@code metadata_validation_status}. */
|
||||
public static final String OK = "ok";
|
||||
public static final String WARNING = "warning";
|
||||
|
||||
/** One validation warning. Serialized to {@code metadata_validation_json}. */
|
||||
@Data
|
||||
public static class FieldWarning {
|
||||
private final String field;
|
||||
private final String reason;
|
||||
private final String source;
|
||||
private final String rawValuePreview;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class ValidationResult {
|
||||
private final Map<String, Object> cleaned;
|
||||
private final String status;
|
||||
private final List<FieldWarning> warnings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate {@code raw} metadata against {@code def}'s schema.
|
||||
*
|
||||
* @param def the pageType definition (may be {@code null} → no schema)
|
||||
* @param raw the raw metadata produced by the LLM (may be {@code null})
|
||||
* @param allowAdditional whether undeclared fields are kept
|
||||
* @param source stage label recorded on each warning (route/create/merge)
|
||||
*/
|
||||
public ValidationResult validate(WikiPageTypeDef def, Map<String, Object> raw,
|
||||
boolean allowAdditional, String source) {
|
||||
Map<String, Object> cleaned = new LinkedHashMap<>();
|
||||
List<FieldWarning> warnings = new ArrayList<>();
|
||||
Map<String, Object> input = raw == null ? Map.of() : raw;
|
||||
Map<String, WikiFieldSchema> schema = (def == null || def.getSchema() == null)
|
||||
? Map.of() : def.getSchema();
|
||||
|
||||
// 1. Declared fields: validate / coerce in schema order.
|
||||
for (Map.Entry<String, WikiFieldSchema> entry : schema.entrySet()) {
|
||||
String field = entry.getKey();
|
||||
WikiFieldSchema fieldSchema = entry.getValue();
|
||||
boolean present = input.containsKey(field) && input.get(field) != null;
|
||||
if (!present) {
|
||||
if (fieldSchema.isRequired()) {
|
||||
warnings.add(new FieldWarning(field, "required field missing", source, null));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Object value = input.get(field);
|
||||
Coerced coerced = coerce(fieldSchema, value);
|
||||
if (coerced.warning != null) {
|
||||
warnings.add(new FieldWarning(field, coerced.warning, source, preview(value)));
|
||||
}
|
||||
cleaned.put(field, coerced.value);
|
||||
}
|
||||
|
||||
// 2. Undeclared fields: keep or drop.
|
||||
for (Map.Entry<String, Object> entry : input.entrySet()) {
|
||||
String field = entry.getKey();
|
||||
if (schema.containsKey(field)) {
|
||||
continue;
|
||||
}
|
||||
if (allowAdditional) {
|
||||
cleaned.put(field, entry.getValue());
|
||||
} else {
|
||||
warnings.add(new FieldWarning(field, "dropped: not declared in schema",
|
||||
source, preview(entry.getValue())));
|
||||
}
|
||||
}
|
||||
|
||||
String status = warnings.isEmpty() ? OK : WARNING;
|
||||
return new ValidationResult(cleaned, status, warnings);
|
||||
}
|
||||
|
||||
private record Coerced(Object value, String warning) {}
|
||||
|
||||
private Coerced coerce(WikiFieldSchema schema, Object value) {
|
||||
String type = schema.getType() == null ? "string" : schema.getType().trim().toLowerCase();
|
||||
return switch (type) {
|
||||
case "string" -> new Coerced(String.valueOf(value), null);
|
||||
case "number" -> coerceNumber(value);
|
||||
case "boolean" -> coerceBoolean(value);
|
||||
case "date" -> coerceDate(value);
|
||||
case "enum" -> coerceEnum(schema, value);
|
||||
case "string_array" -> coerceStringArray(value);
|
||||
default -> new Coerced(value, null);
|
||||
};
|
||||
}
|
||||
|
||||
private Coerced coerceNumber(Object value) {
|
||||
if (value instanceof Number) {
|
||||
return new Coerced(value, null);
|
||||
}
|
||||
try {
|
||||
String s = String.valueOf(value).trim();
|
||||
if (s.contains(".")) {
|
||||
return new Coerced(Double.parseDouble(s), null);
|
||||
}
|
||||
return new Coerced(Long.parseLong(s), null);
|
||||
} catch (NumberFormatException e) {
|
||||
return new Coerced(value, "expected number");
|
||||
}
|
||||
}
|
||||
|
||||
private Coerced coerceBoolean(Object value) {
|
||||
if (value instanceof Boolean) {
|
||||
return new Coerced(value, null);
|
||||
}
|
||||
String s = String.valueOf(value).trim().toLowerCase();
|
||||
if ("true".equals(s) || "false".equals(s)) {
|
||||
return new Coerced(Boolean.valueOf(s), null);
|
||||
}
|
||||
return new Coerced(value, "expected boolean");
|
||||
}
|
||||
|
||||
private Coerced coerceDate(Object value) {
|
||||
String s = String.valueOf(value).trim();
|
||||
if (ISO_DATE.matcher(s).matches()) {
|
||||
return new Coerced(s, null);
|
||||
}
|
||||
return new Coerced(value, "expected ISO date YYYY-MM-DD");
|
||||
}
|
||||
|
||||
private Coerced coerceEnum(WikiFieldSchema schema, Object value) {
|
||||
String s = String.valueOf(value);
|
||||
List<String> values = schema.getValues();
|
||||
if (values == null || values.contains(s)) {
|
||||
return new Coerced(s, null);
|
||||
}
|
||||
return new Coerced(s, "value not in allowed enum set");
|
||||
}
|
||||
|
||||
private Coerced coerceStringArray(Object value) {
|
||||
if (value instanceof List<?> list) {
|
||||
List<String> out = new ArrayList<>(list.size());
|
||||
for (Object o : list) {
|
||||
out.add(String.valueOf(o));
|
||||
}
|
||||
return new Coerced(out, null);
|
||||
}
|
||||
// A single scalar is accepted as a one-element array.
|
||||
return new Coerced(List.of(String.valueOf(value)), null);
|
||||
}
|
||||
|
||||
private static String preview(Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
String s = String.valueOf(value);
|
||||
return s.length() > 120 ? s.substring(0, 120) + "…" : s;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,137 @@
|
||||
package vip.mate.wiki.profile;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Unit tests for {@link WikiMetadataValidator} covering required/type/enum/date
|
||||
* rules, coercion, and undeclared-field handling.
|
||||
*/
|
||||
class WikiMetadataValidatorTest {
|
||||
|
||||
private final WikiMetadataValidator validator = new WikiMetadataValidator();
|
||||
|
||||
private WikiPageTypeDef episodeDef() {
|
||||
WikiPageTypeDef def = new WikiPageTypeDef();
|
||||
Map<String, WikiFieldSchema> schema = new LinkedHashMap<>();
|
||||
schema.put("event_type", field("string", true, null));
|
||||
schema.put("event_date", field("date", true, null));
|
||||
schema.put("significance", field("enum", false, List.of("low", "medium", "high")));
|
||||
schema.put("cited_count", field("number", false, null));
|
||||
def.setSchema(schema);
|
||||
return def;
|
||||
}
|
||||
|
||||
private WikiFieldSchema field(String type, boolean required, List<String> values) {
|
||||
WikiFieldSchema f = new WikiFieldSchema();
|
||||
f.setType(type);
|
||||
f.setRequired(required);
|
||||
f.setValues(values);
|
||||
return f;
|
||||
}
|
||||
|
||||
@Test
|
||||
void validMetadata_isOk() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("event_type", "liquidity_shock");
|
||||
raw.put("event_date", "2024-09-18");
|
||||
raw.put("significance", "high");
|
||||
raw.put("cited_count", 12);
|
||||
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||
|
||||
assertEquals(WikiMetadataValidator.OK, r.getStatus());
|
||||
assertTrue(r.getWarnings().isEmpty());
|
||||
assertEquals("liquidity_shock", r.getCleaned().get("event_type"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void requiredMissing_warnsButKeepsGoing() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("event_type", "x");
|
||||
// event_date missing
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||
|
||||
assertEquals(WikiMetadataValidator.WARNING, r.getStatus());
|
||||
assertTrue(r.getWarnings().stream().anyMatch(w ->
|
||||
w.getField().equals("event_date") && w.getReason().contains("required")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void numberCoercedFromString() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("event_type", "x");
|
||||
raw.put("event_date", "2024-01-01");
|
||||
raw.put("cited_count", "42");
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||
|
||||
assertEquals(42L, r.getCleaned().get("cited_count"));
|
||||
assertEquals(WikiMetadataValidator.OK, r.getStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
void badDate_warns() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("event_type", "x");
|
||||
raw.put("event_date", "Sept 2024");
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||
|
||||
assertTrue(r.getWarnings().stream().anyMatch(w ->
|
||||
w.getField().equals("event_date") && w.getReason().contains("ISO date")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void enumOutOfRange_warns() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("event_type", "x");
|
||||
raw.put("event_date", "2024-01-01");
|
||||
raw.put("significance", "catastrophic");
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "create");
|
||||
|
||||
assertTrue(r.getWarnings().stream().anyMatch(w ->
|
||||
w.getField().equals("significance") && w.getReason().contains("enum")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void undeclaredField_droppedWithWarning_whenNotAllowed() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("event_type", "x");
|
||||
raw.put("event_date", "2024-01-01");
|
||||
raw.put("rogue", "surprise");
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, false, "route");
|
||||
|
||||
assertFalse(r.getCleaned().containsKey("rogue"));
|
||||
WikiMetadataValidator.FieldWarning w = r.getWarnings().stream()
|
||||
.filter(x -> x.getField().equals("rogue")).findFirst().orElseThrow();
|
||||
assertTrue(w.getReason().contains("dropped"));
|
||||
assertEquals("route", w.getSource());
|
||||
assertEquals("surprise", w.getRawValuePreview());
|
||||
}
|
||||
|
||||
@Test
|
||||
void undeclaredField_keptWhenAllowed() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("event_type", "x");
|
||||
raw.put("event_date", "2024-01-01");
|
||||
raw.put("extra", "kept");
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(episodeDef(), raw, true, "create");
|
||||
|
||||
assertEquals("kept", r.getCleaned().get("extra"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullDef_keepsAllFields() {
|
||||
Map<String, Object> raw = new LinkedHashMap<>();
|
||||
raw.put("anything", "goes");
|
||||
WikiMetadataValidator.ValidationResult r = validator.validate(null, raw, false, "create");
|
||||
// No schema → additional fields dropped (allowAdditional=false) with warning
|
||||
assertTrue(r.getWarnings().stream().anyMatch(w -> w.getField().equals("anything")));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user