Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion containers/api-proxy/body-handler.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ function createBodyHandler({ handleRequestError, otel }) {
*/
async function transformRequestBody(body, provider, req, requestId, bodyTransform) {
if (bodyTransform && (req.method === 'POST' || req.method === 'PUT' || req.method === 'PATCH')) {
const transformed = await bodyTransform(body);
const transformed = await bodyTransform(body, req);
if (transformed) body = transformed;
}

Expand Down
2 changes: 1 addition & 1 deletion containers/api-proxy/model-api-mapping.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
"family": "gpt-5.4",
"patterns": ["gpt-5.4*"],
"endpoints": ["chat_completions", "responses"],
"notes": "Supports both endpoints. Includes gpt-5.4-mini, gpt-5.4-nano."
"notes": "Supports both endpoints on OpenAI. Includes gpt-5.4-mini, gpt-5.4-nano. NOTE: gpt-5.4-mini is not accessible via /chat/completions on the Copilot API (responses-only via Copilot); the api-proxy handles this by falling back to the next alias candidate."
},
{
"family": "gpt-5.3",
Expand Down
6 changes: 3 additions & 3 deletions containers/api-proxy/model-body-rewriter.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ const { resolveModel } = require('./model-resolver');
* @param {Record<string, string[]|null>} availableModels - Cached models per provider
* @param {{ enabled?: boolean, strategy?: string }} [modelFallbackConfig]
* @param {{ allowedModels?: string[]|null, disallowedModels?: string[]|null }|null} [modelPolicyConfig]
* @returns {{ body: Buffer, originalModel: string, resolvedModel: string, log: string[], fallback?: object } | null}
* @returns {{ body: Buffer, originalModel: string, resolvedModel: string, candidates: string[], log: string[], fallback?: object } | null}
*/
function rewriteModelInBody(body, provider, aliases, availableModels, modelFallbackConfig, modelPolicyConfig) {
// Only attempt rewrite for non-empty bodies
Expand All @@ -38,7 +38,7 @@ function rewriteModelInBody(body, provider, aliases, availableModels, modelFallb
const resolution = resolveModel(originalModel, aliases, availableModels, provider, [], modelFallbackConfig, modelPolicyConfig);
if (!resolution) return null;

const { resolvedModel, log } = resolution;
const { resolvedModel, candidates, log } = resolution;

// No rewrite needed if the model is already the resolved value
if (resolvedModel === parsed.model) return null;
Expand All @@ -47,7 +47,7 @@ function rewriteModelInBody(body, provider, aliases, availableModels, modelFallb
parsed.model = resolvedModel;
const newBody = Buffer.from(JSON.stringify(parsed), 'utf8');

return { body: newBody, originalModel, resolvedModel, log, fallback: resolution.fallback };
return { body: newBody, originalModel, resolvedModel, candidates, log, fallback: resolution.fallback };
}

module.exports = {
Expand Down
7 changes: 6 additions & 1 deletion containers/api-proxy/model-config.js
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,18 @@ function getEffectiveModelFallbackForReflect(adapters) {
function makeModelBodyTransform(provider, cachedModels, refreshProviderModelsForResolution) {
if (!MODEL_ALIASES) return null;
const providerModelFallback = getModelFallbackForProvider(provider);
return async (body) => {
return async (body, req) => {
let result = rewriteModelInBody(body, provider, MODEL_ALIASES.models, cachedModels, providerModelFallback, MODEL_POLICY_CONFIG);
if (!result || (result.fallback && result.fallback.activated)) {
await refreshProviderModelsForResolution(provider);
result = rewriteModelInBody(body, provider, MODEL_ALIASES.models, cachedModels, providerModelFallback, MODEL_POLICY_CONFIG);
}
if (!result) return null;
// Store ranked candidates on the request object so endpoint-blocked retry
// logic can fall back to the next candidate without re-resolving.
if (req && Array.isArray(result.candidates)) {
req.awfModelCandidates = result.candidates;
}
const originalModel = sanitizeForLog(result.originalModel) || '(none)';
const resolvedModel = sanitizeForLog(result.resolvedModel);
if (providerModelFallback.enabled && result.fallback) {
Expand Down
3 changes: 2 additions & 1 deletion containers/api-proxy/model-resolver.js
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ function _resolveAliasPatterns(aliasKey, aliasDefinition, requestedModel, aliase

return {
resolvedModel: resolved,
candidates: unique,
log,
fallback: fallbackConfig.enabled
? { activated: false, selection_method: 'middle_power_median', reason: 'normal_resolution_succeeded' }
Expand All @@ -262,7 +263,7 @@ function _resolveAliasPatterns(aliasKey, aliasDefinition, requestedModel, aliase
* @param {string[]} [chain=[]] - Accumulates visited alias names for loop detection
* @param {{ enabled?: boolean, strategy?: string }} [modelFallbackConfig]
* @param {{ allowedModels?: string[]|null, disallowedModels?: string[]|null }|null} [modelPolicyConfig]
* @returns {{ resolvedModel: string, log: string[], fallback?: object } | null}
* @returns {{ resolvedModel: string, candidates: string[], log: string[], fallback?: object } | null}
*/
function resolveModel(requestedModel, aliases, availableModels, currentProvider, chain = [], modelFallbackConfig = DEFAULT_MODEL_FALLBACK, modelPolicyConfig = null) {
const log = [];
Expand Down
10 changes: 10 additions & 0 deletions containers/api-proxy/model-resolver.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,16 @@ describe('resolveModel', () => {
expect(result.resolvedModel).toBe('claude-sonnet-4.6'); // 4.6 > 4.5
});

it('should include ranked candidates list for endpoint-blocked fallback', () => {
// When an alias resolves to multiple candidates, all ranked candidates are
// returned so the caller can fall back to the next one if the first fails.
const result = resolveModel('sonnet', aliases, availableModels, 'copilot');
expect(result).not.toBeNull();
expect(Array.isArray(result.candidates)).toBe(true);
expect(result.candidates[0]).toBe('claude-sonnet-4.6'); // highest version first
expect(result.candidates).toContain('claude-sonnet-4.5'); // lower version available as fallback
});

it('should resolve recursive aliases across multiple levels', () => {
// "" → ["sonnet"] → ["copilot/*sonnet*"] → matches copilot models
const result = resolveModel('', aliases, availableModels, 'copilot');
Expand Down
34 changes: 34 additions & 0 deletions containers/api-proxy/upstream-http.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
'use strict';

const { parseBodyAsObject } = require('./body-utils');

/**
* Backoff delays (ms) between successive model-not-supported retries.
* Index 0 → delay before the 1st retry, index 1 → delay before the 2nd retry.
Expand Down Expand Up @@ -54,6 +56,38 @@ function createSendUpstreamRequest({
});
});
},
onModelEndpointBlockedRetry: () => {
// The model resolved from the alias is not accessible via this endpoint
// (e.g. gpt-5.4-mini on Copilot /chat/completions). Try the next
// ranked candidate stored on the request object during body transform.
const candidates = req.awfModelCandidates;
if (!Array.isArray(candidates) || candidates.length < 2) return false;

// Determine which model was sent in the current body.
const parsed = parseBodyAsObject(body);
const currentModel = parsed && parsed.model;
if (!currentModel) return false;

const currentIdx = candidates.indexOf(currentModel);
if (currentIdx < 0 || currentIdx >= candidates.length - 1) return false;

const nextModel = candidates[currentIdx + 1];

// Rewrite the body with the next candidate.
const newParsed = parseBodyAsObject(body);
if (!newParsed) return false;
newParsed.model = nextModel;
const newBody = Buffer.from(JSON.stringify(newParsed), 'utf8');

// Update the candidates list so if the next model also fails we can
// continue falling back (by shifting the current index forward).
sendUpstreamRequest(requestHeaders, {
body: newBody, targetHost, upstreamPath, req, res, provider, requestId, startTime, span, requestBytes,
hasRetried,
modelNotSupportedRetryCount,
});
return true;
},
Comment on lines +66 to +90
});
});

Expand Down
31 changes: 28 additions & 3 deletions containers/api-proxy/upstream-response.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ const MAX_MODEL_NOT_SUPPORTED_RETRIES = 2;
*/
const MODEL_NOT_SUPPORTED_PATTERN = /the requested model is not supported/i;

/**
* Pattern matching the Copilot error for a model that exists in the catalogue
* but is not accessible via the requested endpoint (e.g. /chat/completions).
* This is a permanent per-endpoint restriction, not a transient catalogue issue.
* Examples: "model \"gpt-5.4-mini\" is not accessible via the /chat/completions endpoint"
*/
const MODEL_ENDPOINT_BLOCKED_PATTERN = /not accessible via the .+? endpoint/i;

/**
* Return true when the response body contains a Copilot "model not supported"
* error message.
Expand All @@ -25,6 +33,17 @@ function parseModelNotSupportedFromBody(body) {
return MODEL_NOT_SUPPORTED_PATTERN.test(body.toString('utf8'));
}

/**
* Return true when the response body indicates the model is not accessible
* via the requested endpoint.
*
* @param {Buffer} body
* @returns {boolean}
*/
function parseModelEndpointBlockedFromBody(body) {
return MODEL_ENDPOINT_BLOCKED_PATTERN.test(body.toString('utf8'));
}

function createUpstreamResponseHandlers({
metrics,
logRequest,
Expand Down Expand Up @@ -56,19 +75,22 @@ function createUpstreamResponseHandlers({
body, res, provider, requestId, req, targetHost, startTime, span, requestBytes,
hasRetried, onRetry,
modelNotSupportedRetryCount = 0, onModelNotSupportedRetry,
onModelEndpointBlockedRetry,
}) {
let responseBytes = 0;
const billingInfo = extractBillingHeaders(proxyRes.headers);
const initiatorSent = requestHeaders['x-initiator'] || null;

// Buffer the 400 response body when we may need to inspect it for either:
// (a) a deprecated Anthropic/Copilot beta-header value (first attempt only), or
// (b) a transient Copilot "model not supported" catalogue error (up to MAX retries).
// (a) a deprecated Anthropic/Copilot beta-header value (first attempt only),
// (b) a transient Copilot "model not supported" catalogue error (up to MAX retries), or
// (c) a permanent Copilot "model not accessible via endpoint" error (fallback to next candidate).
const shouldBuffer400 =
proxyRes.statusCode === 400 &&
(
((provider === 'anthropic' || provider === 'copilot') && !hasRetried) ||
(provider === 'copilot' && modelNotSupportedRetryCount < MAX_MODEL_NOT_SUPPORTED_RETRIES)
(provider === 'copilot' && modelNotSupportedRetryCount < MAX_MODEL_NOT_SUPPORTED_RETRIES) ||
(provider === 'copilot' && !!onModelEndpointBlockedRetry)
);

const completionCtx = { startTime, provider, req, requestBytes, targetHost, requestId };
Expand Down Expand Up @@ -96,10 +118,12 @@ function createUpstreamResponseHandlers({
const didRetry = handle400WithRetry(proxyRes, requestHeaders, responseBody, {
provider, requestId, hasRetried, onRetry,
modelNotSupportedRetryCount, maxModelNotSupportedRetries: MAX_MODEL_NOT_SUPPORTED_RETRIES, onModelNotSupportedRetry,
onModelEndpointBlockedRetry,
completionCtx, authErrCtx, initiatorSent, billingInfo, res, span,
parseDeprecatedHeaderFromBody,
learnAndStripDeprecatedHeaderValue,
parseModelNotSupportedFromBody,
parseModelEndpointBlockedFromBody,
logRequest,
sanitizeForLog,
logRequestCompletion,
Expand Down Expand Up @@ -143,6 +167,7 @@ function createUpstreamResponseHandlers({
module.exports = {
createUpstreamResponseHandlers,
parseModelNotSupportedFromBody,
parseModelEndpointBlockedFromBody,
MAX_MODEL_NOT_SUPPORTED_RETRIES,
// Exported for unit-test access only; not part of the public API.
_testing: {
Expand Down
28 changes: 25 additions & 3 deletions containers/api-proxy/upstream-retry.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
function handle400WithRetry(proxyRes, requestHeaders, responseBody, {
provider, requestId, hasRetried, onRetry,
modelNotSupportedRetryCount, maxModelNotSupportedRetries, onModelNotSupportedRetry,
onModelEndpointBlockedRetry,
completionCtx, authErrCtx, initiatorSent, billingInfo, res, span,
parseDeprecatedHeaderFromBody, learnAndStripDeprecatedHeaderValue,
parseModelNotSupportedFromBody, logRequest, sanitizeForLog,
parseModelNotSupportedFromBody, parseModelEndpointBlockedFromBody, logRequest, sanitizeForLog,
logRequestCompletion, logUpstreamAuthError, otel,
}) {
// ── (a) Deprecated beta-header retry (first attempt for anthropic/copilot) ──
Expand All @@ -23,7 +24,28 @@ function handle400WithRetry(proxyRes, requestHeaders, responseBody, {
}
}

// ── (b) Transient model-not-supported retry (copilot only, up to MAX) ──────
// ── (b) Permanent endpoint-blocked fallback (copilot only) ───────────────────
// When Copilot rejects a model because it is not accessible via the requested
// endpoint (e.g. gpt-5.4-mini on /chat/completions), this is a permanent
// per-model restriction — not a transient catalogue issue. Try the next
// ranked candidate from the alias resolution if one is available.
if (
provider === 'copilot' &&
onModelEndpointBlockedRetry &&
parseModelEndpointBlockedFromBody(responseBody)
) {
const { req } = authErrCtx;
logRequest('warn', 'model_endpoint_blocked_fallback', {
request_id: requestId,
provider,
path: sanitizeForLog(req.url),
message: 'Copilot returned 400 endpoint-not-accessible; falling back to next alias candidate',
});
const didRetry = onModelEndpointBlockedRetry();
if (didRetry) return true;
}

// ── (c) Transient model-not-supported retry (copilot only, up to MAX) ──────
if (
provider === 'copilot' &&
modelNotSupportedRetryCount < maxModelNotSupportedRetries &&
Expand All @@ -41,7 +63,7 @@ function handle400WithRetry(proxyRes, requestHeaders, responseBody, {
return true;
}

// ── (c) Model-unavailable diagnostic (non-retryable model-not-supported 400) ───
// ── (d) Model-unavailable diagnostic (non-retryable model-not-supported 400) ───
if (proxyRes.statusCode === 400 && parseModelNotSupportedFromBody(responseBody)) {
const { req } = authErrCtx;
logRequest('error', 'model_unavailable', {
Expand Down
67 changes: 67 additions & 0 deletions containers/api-proxy/upstream-retry.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ describe('upstream-retry', () => {
modelNotSupportedRetryCount: 0,
maxModelNotSupportedRetries: 2,
onModelNotSupportedRetry: jest.fn(),
onModelEndpointBlockedRetry: jest.fn(() => false),
completionCtx: {},
authErrCtx: { req: { url: '/v1/chat/completions' } },
initiatorSent: null,
Expand All @@ -19,6 +20,7 @@ describe('upstream-retry', () => {
parseDeprecatedHeaderFromBody: jest.fn(() => null),
learnAndStripDeprecatedHeaderValue: jest.fn(() => false),
parseModelNotSupportedFromBody: jest.fn(() => false),
parseModelEndpointBlockedFromBody: jest.fn(() => false),
logRequest: jest.fn(),
sanitizeForLog: (value) => value,
logRequestCompletion: jest.fn(),
Expand Down Expand Up @@ -69,4 +71,69 @@ describe('upstream-retry', () => {
}));
expect(opts.otel.endSpan).toHaveBeenCalledWith(opts.span, 400);
});

describe('endpoint-blocked fallback', () => {
const endpointBlockedBody = Buffer.from(
'{"error":{"message":"model \\"gpt-5.4-mini\\" is not accessible via the /chat/completions endpoint"}}'
);

test('calls onModelEndpointBlockedRetry when endpoint-blocked pattern matches', () => {
const opts = createBaseOptions();
opts.parseModelEndpointBlockedFromBody.mockReturnValue(true);
opts.onModelEndpointBlockedRetry.mockReturnValue(true);
const proxyRes = { statusCode: 400, headers: {} };

const didRetry = handle400WithRetry(proxyRes, {}, endpointBlockedBody, opts);

expect(didRetry).toBe(true);
expect(opts.onModelEndpointBlockedRetry).toHaveBeenCalled();
expect(opts.logRequest).toHaveBeenCalledWith(
'warn', 'model_endpoint_blocked_fallback', expect.objectContaining({ provider: 'copilot' })
);
});

test('falls through to forward response when onModelEndpointBlockedRetry returns false (no candidates)', () => {
const opts = createBaseOptions();
opts.parseModelEndpointBlockedFromBody.mockReturnValue(true);
opts.onModelEndpointBlockedRetry.mockReturnValue(false);
const proxyRes = {
statusCode: 400,
headers: { 'content-type': 'application/json' },
};

const didRetry = handle400WithRetry(proxyRes, {}, endpointBlockedBody, opts);

expect(didRetry).toBe(false);
expect(opts.res.writeHead).toHaveBeenCalledWith(400, expect.any(Object));
});

test('does not trigger when provider is not copilot', () => {
const opts = createBaseOptions();
opts.provider = 'openai';
opts.parseModelEndpointBlockedFromBody.mockReturnValue(true);
const proxyRes = {
statusCode: 400,
headers: { 'content-type': 'application/json' },
};

const didRetry = handle400WithRetry(proxyRes, {}, endpointBlockedBody, opts);

expect(didRetry).toBe(false);
expect(opts.onModelEndpointBlockedRetry).not.toHaveBeenCalled();
});

test('does not trigger when onModelEndpointBlockedRetry is not provided', () => {
const opts = createBaseOptions();
delete opts.onModelEndpointBlockedRetry;
opts.parseModelEndpointBlockedFromBody.mockReturnValue(true);
const proxyRes = {
statusCode: 400,
headers: { 'content-type': 'application/json' },
};

const didRetry = handle400WithRetry(proxyRes, {}, endpointBlockedBody, opts);

expect(didRetry).toBe(false);
});
});
});
2 changes: 1 addition & 1 deletion docs/model-api-mapping.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
"family": "gpt-5.4",
"patterns": ["gpt-5.4*"],
"endpoints": ["chat_completions", "responses"],
"notes": "Supports both endpoints. Includes gpt-5.4-mini, gpt-5.4-nano."
"notes": "Supports both endpoints on OpenAI. Includes gpt-5.4-mini, gpt-5.4-nano. NOTE: gpt-5.4-mini is not accessible via /chat/completions on the Copilot API (responses-only via Copilot); the api-proxy handles this by falling back to the next alias candidate."
},
{
"family": "gpt-5.3",
Expand Down
Loading