opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(llm): live model discovery for ChatGPT OAuth provider

Browse Source
This commit is contained in:
matevip 2026-05-05 13:06:19 +08:00
parent bf7bc73f5f
commit 704c6317e3
7 changed files with 180 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java
View File

@ -11,6 +11,7 @@ import org.springframework.util.StringUtils;
import org.springframework.web.client.RestClient;
import vip.mate.exception.MateClawException;
import vip.mate.llm.model.*;
import vip.mate.llm.oauth.OpenAIOAuthService;
import java.time.Duration;
import java.util.*;
@ -30,6 +31,36 @@ public class ModelDiscoveryService {
private final ModelProviderService modelProviderService;
private final ModelConfigService modelConfigService;
private final ObjectMapper objectMapper;
private final OpenAIOAuthService openAIOAuthService;
/**
* The Codex models endpoint that the ChatGPT subscription OAuth path exposes.
* Returns a JSON object {@code {"models": [{slug, supported_in_api, visibility,
* priority, ...}]}} once authenticated with a Bearer access token.
*/
static final String CHATGPT_CODEX_MODELS_URL =
"https://chatgpt.com/backend-api/codex/models?client_version=1.0.0";
/**
* Synthetic forward-compat catalog: when a newer Codex slug is not surfaced
* by the live API but a known older sibling is, append the newer slug so
* users can opt into models OpenAI is rolling out without waiting for the
* metadata to flip. Mirrors the upstream Codex CLI behaviour.
*/
private static final List<Map.Entry<String, List<String>>> CHATGPT_FORWARD_COMPAT =
List.of(
Map.entry("gpt-5.5", List.of("gpt-5.4", "gpt-5.4-mini", "gpt-5.3-codex")),
Map.entry("gpt-5.4-mini", List.of("gpt-5.3-codex", "gpt-5.2-codex")),
Map.entry("gpt-5.4", List.of("gpt-5.3-codex", "gpt-5.2-codex")),
Map.entry("gpt-5.3-codex", List.of("gpt-5.2-codex"))
);
private RestClient chatgptCodexClient = RestClient.create();
/** Test seam — let unit tests point this at a {@link org.springframework.test.web.client.MockRestServiceServer}. */
void setChatgptCodexClient(RestClient client) {
this.chatgptCodexClient = client;
}
private static final Duration TIMEOUT = Duration.ofSeconds(10);
@ -364,10 +395,12 @@ public class ModelDiscoveryService {
case DASHSCOPE_NATIVE -> fetchDashScopeModels(provider);
case GEMINI_NATIVE -> fetchGeminiModels(provider);
case ANTHROPIC_MESSAGES -> fetchAnthropicModels(provider);
// Claude Code OAuth provider has a fixed model catalog (Anthropic
// doesn't expose model discovery on Bearer-auth requests). Models
// are seeded via Flyway, not discovered.
case OPENAI_CHATGPT, ANTHROPIC_CLAUDE_CODE ->
// ChatGPT OAuth has its own discovery endpoint at chatgpt.com/backend-api/codex.
case OPENAI_CHATGPT -> fetchChatGPTOAuthModels(provider);
// Claude Code OAuth has a fixed model catalog — Anthropic doesn't
// expose a discovery endpoint to Bearer-auth requests, so models
// are seeded via Flyway.
case ANTHROPIC_CLAUDE_CODE ->
throw new MateClawException("err.llm.oauth_no_discovery",
"OAuth provider 不支持模型发现");
};
@ -453,6 +486,112 @@ public class ModelDiscoveryService {
return parseAnthropicModelsResponse(body);
}
/**
* Fetch the ChatGPT subscription OAuth model catalog. Uses the user's
* already-stored OAuth access token (auto-refreshed if it's near expiry)
* and hits the same Codex endpoint the upstream client uses. Filters out
* models the API marks as not exposed ({@code supported_in_api == false})
* or hidden, sorts by priority, then layers in synthetic forward-compat
* entries (e.g. surface {@code gpt-5.5} when only older siblings are
* returned).
*/
private List<ModelInfoDTO> fetchChatGPTOAuthModels(ModelProviderEntity provider) {
String accessToken;
try {
accessToken = openAIOAuthService.ensureValidAccessToken();
} catch (MateClawException e) {
// Surface the precise i18n key from OpenAIOAuthService (e.g.
// err.llm.oauth_not_connected) so the UI can prompt the user to
// sign in. Wrapping would lose that signal.
throw e;
}
String body;
try {
body = chatgptCodexClient.get()
.uri(CHATGPT_CODEX_MODELS_URL)
.header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken)
.header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
.retrieve()
.body(String.class);
} catch (Exception e) {
log.warn("[ModelDiscovery] ChatGPT OAuth models fetch failed: {}", e.getMessage());
throw new MateClawException("err.llm.chatgpt_models_fetch_failed",
"拉取 ChatGPT 可用模型失败: " + e.getMessage());
}
return addChatGPTForwardCompatModels(parseChatGPTCodexModelsResponse(body));
}
/**
* Parse the {@code {"models": [{slug, supported_in_api, visibility, priority}]}}
* response. Drops entries the API hides from the OAuth catalog and orders
* the rest by ascending {@code priority} (lower = higher precedence in
* the upstream client's UX).
*/
List<ModelInfoDTO> parseChatGPTCodexModelsResponse(String body) {
if (body == null || body.isBlank()) return List.of();
try {
JsonNode root = objectMapper.readTree(body);
JsonNode entries = root.path("models");
if (!entries.isArray()) return List.of();
// Sort by priority ascending while preserving the API-listed slug
List<int[]> indices = new ArrayList<>();
for (int i = 0; i < entries.size(); i++) {
JsonNode item = entries.get(i);
if (!item.isObject()) continue;
String slug = item.path("slug").asText("").trim();
if (slug.isEmpty()) continue;
if (item.path("supported_in_api").asBoolean(true) == false) continue;
String vis = item.path("visibility").asText("").trim().toLowerCase();
if ("hide".equals(vis) || "hidden".equals(vis)) continue;
int priority = item.has("priority") && item.get("priority").isNumber()
? item.get("priority").asInt()
: 10_000;
indices.add(new int[]{priority, i});
}
indices.sort(Comparator.comparingInt((int[] a) -> a[0]).thenComparingInt(a -> a[1]));
Set<String> seen = new LinkedHashSet<>();
List<ModelInfoDTO> out = new ArrayList<>();
for (int[] idx : indices) {
JsonNode item = entries.get(idx[1]);
String slug = item.path("slug").asText("").trim();
if (!seen.add(slug)) continue;
out.add(new ModelInfoDTO(slug, slug));
}
return out;
} catch (Exception e) {
log.warn("[ModelDiscovery] Failed to parse ChatGPT codex models response: {}",
e.getMessage());
return List.of();
}
}
/**
* Append synthetic forward-compat entries for newer slugs that the API
* has not yet surfaced but a known older sibling is present for. Mirrors
* the reference client's behaviour so users can opt into {@code gpt-5.5}
* during a staged rollout.
*/
static List<ModelInfoDTO> addChatGPTForwardCompatModels(List<ModelInfoDTO> input) {
Set<String> seen = new LinkedHashSet<>();
List<ModelInfoDTO> out = new ArrayList<>(input.size() + CHATGPT_FORWARD_COMPAT.size());
for (ModelInfoDTO m : input) {
if (m.getId() != null && seen.add(m.getId())) out.add(m);
}
for (Map.Entry<String, List<String>> e : CHATGPT_FORWARD_COMPAT) {
String synthetic = e.getKey();
if (seen.contains(synthetic)) continue;
if (e.getValue().stream().anyMatch(seen::contains)) {
seen.add(synthetic);
out.add(new ModelInfoDTO(synthetic, synthetic));
}
}
return out;
}
// ==================== 协议分派：单模型测试 ====================
private String sendTestPrompt(ModelProviderEntity provider, ModelProtocol protocol, String modelId) {

3
mateclaw-server/src/main/resources/db/data-en.sql
View File

@ -120,7 +120,7 @@ VALUES ('volcengine-plan', 'Volcano Engine Coding Plan', '', 'OpenAIChatModel',
MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, auth_type, create_time, update_time)
KEY (provider_id)
VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW());
VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, TRUE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW());
-- RFC-062: Anthropic Claude Code OAuth provider. Credentials live on local
-- disk (Keychain / ~/.claude/.credentials.json), not in this row — leave
@ -270,6 +270,7 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe
(1000000240, 'Kimi for Coding', 'kimi-code', 'kimi-for-coding', 'Kimi Code dedicated coding model', 0.2, 32768, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000250, 'GPT-5.4', 'openai-chatgpt', 'gpt-5.4', 'ChatGPT Plus/Pro member model (OAuth login)', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000251, 'GPT-5.4 Mini', 'openai-chatgpt', 'gpt-5.4-mini', 'ChatGPT member lightweight model', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro flagship model', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
-- GPT-5.5 series (direct OpenAI / Azure / OpenRouter / ChatGPT)
(1000000260, 'GPT-5.5', 'openai', 'gpt-5.5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000261, 'GPT-5.5 Mini', 'openai', 'gpt-5.5-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0),

3
mateclaw-server/src/main/resources/db/data-zh.sql
View File

@ -120,7 +120,7 @@ VALUES ('volcengine-plan', 'Volcano Engine Coding Plan (火山方舟代码计划
MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, auth_type, create_time, update_time)
KEY (provider_id)
VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, FALSE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW());
VALUES ('openai-chatgpt', 'OpenAI ChatGPT (OAuth)', '', 'ChatGPTChatModel', '', 'https://chatgpt.com/backend-api', '{}', FALSE, FALSE, TRUE, FALSE, TRUE, FALSE, 'oauth', NOW(), NOW());
-- RFC-062：Anthropic Claude Code OAuth 订阅 provider。凭据存储在本地磁盘
-- （macOS Keychain 或 ~/.claude/.credentials.json），不写入该行。
@ -273,6 +273,7 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe
(1000000240, 'Kimi for Coding', 'kimi-code', 'kimi-for-coding', 'Kimi Code 专用编码模型', 0.2, 32768, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000250, 'GPT-5.4', 'openai-chatgpt', 'gpt-5.4', 'ChatGPT Plus/Pro 会员模型（OAuth 登录）', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000251, 'GPT-5.4 Mini', 'openai-chatgpt', 'gpt-5.4-mini', 'ChatGPT 会员轻量模型', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro 会员旗舰模型', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
-- GPT-5.5 系列（OpenAI / Azure / OpenRouter）
(1000000260, 'GPT-5.5', 'openai', 'gpt-5.5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0),
(1000000261, 'GPT-5.5 Mini', 'openai', 'gpt-5.5-mini', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0),

14
mateclaw-server/src/main/resources/db/migration/h2/V89__chatgpt_oauth_model_discovery.sql Normal file
View File

@ -0,0 +1,14 @@
-- Enable model discovery on the ChatGPT OAuth provider so the catalog can be
-- pulled live from chatgpt.com/backend-api/codex/models, and seed the GPT-5.5
-- flagship row alongside the existing GPT-5.4 / GPT-5.4 Mini entries. The
-- guard clause + MERGE keep the migration idempotent.
UPDATE mate_model_provider
SET support_model_discovery = TRUE,
update_time = CURRENT_TIMESTAMP
WHERE provider_id = 'openai-chatgpt'
AND support_model_discovery <> TRUE;
MERGE INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted)
KEY (id)
VALUES (1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro flagship model', NULL, 128000, NULL, TRUE, TRUE, FALSE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0);

17
mateclaw-server/src/main/resources/db/migration/mysql/V89__chatgpt_oauth_model_discovery.sql Normal file
View File

@ -0,0 +1,17 @@
-- Enable model discovery on the ChatGPT OAuth provider so the catalog can be
-- pulled live from chatgpt.com/backend-api/codex/models, and seed the GPT-5.5
-- flagship row alongside the existing GPT-5.4 / GPT-5.4 Mini entries. The
-- ON DUPLICATE KEY UPDATE clause keeps the migration idempotent.
UPDATE mate_model_provider
SET support_model_discovery = TRUE,
update_time = CURRENT_TIMESTAMP
WHERE provider_id = 'openai-chatgpt'
AND support_model_discovery <> TRUE;
INSERT INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted)
VALUES (1000000252, 'GPT-5.5', 'openai-chatgpt', 'gpt-5.5', 'ChatGPT Plus/Pro flagship model', NULL, 128000, NULL, TRUE, TRUE, FALSE, NOW(), NOW(), 0)
ON DUPLICATE KEY UPDATE
name = VALUES(name),
description = VALUES(description),
update_time = NOW();

1
mateclaw-server/src/main/resources/messages.properties
View File

@ -231,6 +231,7 @@ err.llm.oauth_no_token=OAuth \u54cd\u5e94\u4e2d\u7f3a\u5c11 access_token
err.llm.chatgpt_not_configured=ChatGPT provider \u672a\u914d\u7f6e
err.llm.pkce_failed=PKCE \u751f\u6210\u5931\u8d25
err.llm.device_code_start_failed=Device code \u7533\u8bf7\u5931\u8d25
err.llm.chatgpt_models_fetch_failed=\u62c9\u53d6 ChatGPT \u53ef\u7528\u6a21\u578b\u5931\u8d25
err.llm.chatgpt_stream_failed=ChatGPT \u6d41\u5f0f\u8c03\u7528\u5931\u8d25
err.llm.chatgpt_error=ChatGPT \u8fd4\u56de\u9519\u8bef
err.llm.chatgpt_account_missing=chatgpt-account-id \u7f3a\u5931

1
mateclaw-server/src/main/resources/messages_en.properties
View File

@ -243,6 +243,7 @@ err.llm.oauth_no_token=access_token missing in OAuth response
err.llm.chatgpt_not_configured=ChatGPT provider not configured, check database initialization
err.llm.pkce_failed=PKCE code_challenge generation failed
err.llm.device_code_start_failed=Device code request failed
err.llm.chatgpt_models_fetch_failed=Failed to fetch available ChatGPT models
err.llm.chatgpt_stream_failed=ChatGPT streaming call failed
err.llm.chatgpt_error=ChatGPT returned an error
err.llm.chatgpt_account_missing=chatgpt-account-id missing, disconnect and re-login via OAuth