From 704c6317e3c2a5d97baf67e94347e670f84eb4a5 Mon Sep 17 00:00:00 2001 From: matevip Date: Tue, 5 May 2026 13:06:19 +0800 Subject: [PATCH] feat(llm): live model discovery for ChatGPT OAuth provider --- .../llm/service/ModelDiscoveryService.java | 147 +++++++++++++++++- .../src/main/resources/db/data-en.sql | 3 +- .../src/main/resources/db/data-zh.sql | 3 +- .../h2/V89__chatgpt_oauth_model_discovery.sql | 14 ++ .../V89__chatgpt_oauth_model_discovery.sql | 17 ++ .../src/main/resources/messages.properties | 1 + .../src/main/resources/messages_en.properties | 1 + 7 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V89__chatgpt_oauth_model_discovery.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V89__chatgpt_oauth_model_discovery.sql diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java index 417fb6dc..3319ae1d 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java @@ -11,6 +11,7 @@ import org.springframework.util.StringUtils; import org.springframework.web.client.RestClient; import vip.mate.exception.MateClawException; import vip.mate.llm.model.*; +import vip.mate.llm.oauth.OpenAIOAuthService; import java.time.Duration; import java.util.*; @@ -30,6 +31,36 @@ public class ModelDiscoveryService { private final ModelProviderService modelProviderService; private final ModelConfigService modelConfigService; private final ObjectMapper objectMapper; + private final OpenAIOAuthService openAIOAuthService; + + /** + * The Codex models endpoint that the ChatGPT subscription OAuth path exposes. + * Returns a JSON object {@code {"models": [{slug, supported_in_api, visibility, + * priority, ...}]}} once authenticated with a Bearer access token. + */ + static final String CHATGPT_CODEX_MODELS_URL = + "https://chatgpt.com/backend-api/codex/models?client_version=1.0.0"; + + /** + * Synthetic forward-compat catalog: when a newer Codex slug is not surfaced + * by the live API but a known older sibling is, append the newer slug so + * users can opt into models OpenAI is rolling out without waiting for the + * metadata to flip. Mirrors the upstream Codex CLI behaviour. + */ + private static final List>> CHATGPT_FORWARD_COMPAT = + List.of( + Map.entry("gpt-5.5", List.of("gpt-5.4", "gpt-5.4-mini", "gpt-5.3-codex")), + Map.entry("gpt-5.4-mini", List.of("gpt-5.3-codex", "gpt-5.2-codex")), + Map.entry("gpt-5.4", List.of("gpt-5.3-codex", "gpt-5.2-codex")), + Map.entry("gpt-5.3-codex", List.of("gpt-5.2-codex")) + ); + + private RestClient chatgptCodexClient = RestClient.create(); + + /** Test seam — let unit tests point this at a {@link org.springframework.test.web.client.MockRestServiceServer}. */ + void setChatgptCodexClient(RestClient client) { + this.chatgptCodexClient = client; + } private static final Duration TIMEOUT = Duration.ofSeconds(10); @@ -364,10 +395,12 @@ public class ModelDiscoveryService { case DASHSCOPE_NATIVE -> fetchDashScopeModels(provider); case GEMINI_NATIVE -> fetchGeminiModels(provider); case ANTHROPIC_MESSAGES -> fetchAnthropicModels(provider); - // Claude Code OAuth provider has a fixed model catalog (Anthropic - // doesn't expose model discovery on Bearer-auth requests). Models - // are seeded via Flyway, not discovered. - case OPENAI_CHATGPT, ANTHROPIC_CLAUDE_CODE -> + // ChatGPT OAuth has its own discovery endpoint at chatgpt.com/backend-api/codex. + case OPENAI_CHATGPT -> fetchChatGPTOAuthModels(provider); + // Claude Code OAuth has a fixed model catalog — Anthropic doesn't + // expose a discovery endpoint to Bearer-auth requests, so models + // are seeded via Flyway. + case ANTHROPIC_CLAUDE_CODE -> throw new MateClawException("err.llm.oauth_no_discovery", "OAuth provider 不支持模型发现"); }; @@ -453,6 +486,112 @@ public class ModelDiscoveryService { return parseAnthropicModelsResponse(body); } + /** + * Fetch the ChatGPT subscription OAuth model catalog. Uses the user's + * already-stored OAuth access token (auto-refreshed if it's near expiry) + * and hits the same Codex endpoint the upstream client uses. Filters out + * models the API marks as not exposed ({@code supported_in_api == false}) + * or hidden, sorts by priority, then layers in synthetic forward-compat + * entries (e.g. surface {@code gpt-5.5} when only older siblings are + * returned). + */ + private List fetchChatGPTOAuthModels(ModelProviderEntity provider) { + String accessToken; + try { + accessToken = openAIOAuthService.ensureValidAccessToken(); + } catch (MateClawException e) { + // Surface the precise i18n key from OpenAIOAuthService (e.g. + // err.llm.oauth_not_connected) so the UI can prompt the user to + // sign in. Wrapping would lose that signal. + throw e; + } + + String body; + try { + body = chatgptCodexClient.get() + .uri(CHATGPT_CODEX_MODELS_URL) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken) + .header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) + .retrieve() + .body(String.class); + } catch (Exception e) { + log.warn("[ModelDiscovery] ChatGPT OAuth models fetch failed: {}", e.getMessage()); + throw new MateClawException("err.llm.chatgpt_models_fetch_failed", + "拉取 ChatGPT 可用模型失败: " + e.getMessage()); + } + + return addChatGPTForwardCompatModels(parseChatGPTCodexModelsResponse(body)); + } + + /** + * Parse the {@code {"models": [{slug, supported_in_api, visibility, priority}]}} + * response. Drops entries the API hides from the OAuth catalog and orders + * the rest by ascending {@code priority} (lower = higher precedence in + * the upstream client's UX). + */ + List parseChatGPTCodexModelsResponse(String body) { + if (body == null || body.isBlank()) return List.of(); + try { + JsonNode root = objectMapper.readTree(body); + JsonNode entries = root.path("models"); + if (!entries.isArray()) return List.of(); + + // Sort by priority ascending while preserving the API-listed slug + List indices = new ArrayList<>(); + for (int i = 0; i < entries.size(); i++) { + JsonNode item = entries.get(i); + if (!item.isObject()) continue; + String slug = item.path("slug").asText("").trim(); + if (slug.isEmpty()) continue; + if (item.path("supported_in_api").asBoolean(true) == false) continue; + String vis = item.path("visibility").asText("").trim().toLowerCase(); + if ("hide".equals(vis) || "hidden".equals(vis)) continue; + int priority = item.has("priority") && item.get("priority").isNumber() + ? item.get("priority").asInt() + : 10_000; + indices.add(new int[]{priority, i}); + } + indices.sort(Comparator.comparingInt((int[] a) -> a[0]).thenComparingInt(a -> a[1])); + + Set seen = new LinkedHashSet<>(); + List out = new ArrayList<>(); + for (int[] idx : indices) { + JsonNode item = entries.get(idx[1]); + String slug = item.path("slug").asText("").trim(); + if (!seen.add(slug)) continue; + out.add(new ModelInfoDTO(slug, slug)); + } + return out; + } catch (Exception e) { + log.warn("[ModelDiscovery] Failed to parse ChatGPT codex models response: {}", + e.getMessage()); + return List.of(); + } + } + + /** + * Append synthetic forward-compat entries for newer slugs that the API + * has not yet surfaced but a known older sibling is present for. Mirrors + * the reference client's behaviour so users can opt into {@code gpt-5.5} + * during a staged rollout. + */ + static List addChatGPTForwardCompatModels(List input) { + Set seen = new LinkedHashSet<>(); + List out = new ArrayList<>(input.size() + CHATGPT_FORWARD_COMPAT.size()); + for (ModelInfoDTO m : input) { + if (m.getId() != null && seen.add(m.getId())) out.add(m); + } + for (Map.Entry> e : CHATGPT_FORWARD_COMPAT) { + String synthetic = e.getKey(); + if (seen.contains(synthetic)) continue; + if (e.getValue().stream().anyMatch(seen::contains)) { + seen.add(synthetic); + out.add(new ModelInfoDTO(synthetic, synthetic)); + } + } + return out; + } + // ==================== 协议分派：单模型测试 ==================== private String sendTestPrompt(ModelProviderEntity provider, ModelProtocol protocol, String modelId) { diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index be6734b9..e59a4ad1 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -120,7 +120,7 @@ VALUES ('volcengine-plan', 'Volcano Engine Coding Plan', '', 'OpenAIChatModel', MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, auth_type, create_time, update_time) KEY (provider_id) -VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW()); +VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, TRUE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW()); -- RFC-062: Anthropic Claude Code OAuth provider. Credentials live on local -- disk (Keychain / ~/.claude/.credentials.json), not in this row — leave @@ -270,6 +270,7 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe (1000000240, 'Kimi for Coding', 'kimi-code', 'kimi-for-coding', 'Kimi Code dedicated coding model', 0.2, 32768, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000250, 'GPT-5.4', 'openai-chatgpt', 'gpt-5.4', 'ChatGPT Plus/Pro member model (OAuth login)', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000251, 'GPT-5.4 Mini', 'openai-chatgpt', 'gpt-5.4-mini', 'ChatGPT member lightweight model', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro flagship model', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0), -- GPT-5.5 series (direct OpenAI / Azure / OpenRouter / ChatGPT) (1000000260, 'GPT-5.5', 'openai', 'gpt-5.5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000261, 'GPT-5.5 Mini', 'openai', 'gpt-5.5-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index ac19bd5c..dd96079d 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -120,7 +120,7 @@ VALUES ('volcengine-plan', 'Volcano Engine Coding Plan (火山方舟代码计划 MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, auth_type, create_time, update_time) KEY (provider_id) -VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW()); +VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, TRUE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW()); -- RFC-062：Anthropic Claude Code OAuth 订阅 provider。凭据存储在本地磁盘 -- （macOS Keychain 或 ~/.claude/.credentials.json），不写入该行。 @@ -273,6 +273,7 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe (1000000240, 'Kimi for Coding', 'kimi-code', 'kimi-for-coding', 'Kimi Code 专用编码模型', 0.2, 32768, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000250, 'GPT-5.4', 'openai-chatgpt', 'gpt-5.4', 'ChatGPT Plus/Pro 会员模型（OAuth 登录）', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000251, 'GPT-5.4 Mini', 'openai-chatgpt', 'gpt-5.4-mini', 'ChatGPT 会员轻量模型', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro 会员旗舰模型', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0), -- GPT-5.5 系列（OpenAI / Azure / OpenRouter） (1000000260, 'GPT-5.5', 'openai', 'gpt-5.5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000261, 'GPT-5.5 Mini', 'openai', 'gpt-5.5-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V89__chatgpt_oauth_model_discovery.sql b/mateclaw-server/src/main/resources/db/migration/h2/V89__chatgpt_oauth_model_discovery.sql new file mode 100644 index 00000000..2b943e28 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V89__chatgpt_oauth_model_discovery.sql @@ -0,0 +1,14 @@ +-- Enable model discovery on the ChatGPT OAuth provider so the catalog can be +-- pulled live from chatgpt.com/backend-api/codex/models, and seed the GPT-5.5 +-- flagship row alongside the existing GPT-5.4 / GPT-5.4 Mini entries. The +-- guard clause + MERGE keep the migration idempotent. + +UPDATE mate_model_provider + SET support_model_discovery = TRUE, + update_time = CURRENT_TIMESTAMP + WHERE provider_id = 'openai-chatgpt' + AND support_model_discovery <> TRUE; + +MERGE INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +KEY (id) +VALUES (1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro flagship model', NULL, 128000, NULL, TRUE, TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V89__chatgpt_oauth_model_discovery.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V89__chatgpt_oauth_model_discovery.sql new file mode 100644 index 00000000..19d745bf --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V89__chatgpt_oauth_model_discovery.sql @@ -0,0 +1,17 @@ +-- Enable model discovery on the ChatGPT OAuth provider so the catalog can be +-- pulled live from chatgpt.com/backend-api/codex/models, and seed the GPT-5.5 +-- flagship row alongside the existing GPT-5.4 / GPT-5.4 Mini entries. The +-- ON DUPLICATE KEY UPDATE clause keeps the migration idempotent. + +UPDATE mate_model_provider + SET support_model_discovery = TRUE, + update_time = CURRENT_TIMESTAMP + WHERE provider_id = 'openai-chatgpt' + AND support_model_discovery <> TRUE; + +INSERT INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +VALUES (1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro flagship model', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + description = VALUES(description), + update_time = NOW(); diff --git a/mateclaw-server/src/main/resources/messages.properties b/mateclaw-server/src/main/resources/messages.properties index f10d47ca..2d012a77 100644 --- a/mateclaw-server/src/main/resources/messages.properties +++ b/mateclaw-server/src/main/resources/messages.properties @@ -231,6 +231,7 @@ err.llm.oauth_no_token=OAuth \u54cd\u5e94\u4e2d\u7f3a\u5c11 access_token err.llm.chatgpt_not_configured=ChatGPT provider \u672a\u914d\u7f6e err.llm.pkce_failed=PKCE \u751f\u6210\u5931\u8d25 err.llm.device_code_start_failed=Device code \u7533\u8bf7\u5931\u8d25 +err.llm.chatgpt_models_fetch_failed=\u62c9\u53d6 ChatGPT \u53ef\u7528\u6a21\u578b\u5931\u8d25 err.llm.chatgpt_stream_failed=ChatGPT \u6d41\u5f0f\u8c03\u7528\u5931\u8d25 err.llm.chatgpt_error=ChatGPT \u8fd4\u56de\u9519\u8bef err.llm.chatgpt_account_missing=chatgpt-account-id \u7f3a\u5931 diff --git a/mateclaw-server/src/main/resources/messages_en.properties b/mateclaw-server/src/main/resources/messages_en.properties index 60de88b0..fa383fe5 100644 --- a/mateclaw-server/src/main/resources/messages_en.properties +++ b/mateclaw-server/src/main/resources/messages_en.properties @@ -243,6 +243,7 @@ err.llm.oauth_no_token=access_token missing in OAuth response err.llm.chatgpt_not_configured=ChatGPT provider not configured, check database initialization err.llm.pkce_failed=PKCE code_challenge generation failed err.llm.device_code_start_failed=Device code request failed +err.llm.chatgpt_models_fetch_failed=Failed to fetch available ChatGPT models err.llm.chatgpt_stream_failed=ChatGPT streaming call failed err.llm.chatgpt_error=ChatGPT returned an error err.llm.chatgpt_account_missing=chatgpt-account-id missing, disconnect and re-login via OAuth