Custom API Server (Custom Format)
Call your custom torch-serve / internal LLM APIs via LiteLLM
Supported Routes:
-
/v1/chat/completions->litellm.acompletion -
/v1/completions->litellm.atext_completion -
/v1/embeddings->litellm.aembedding -
/v1/images/generations->litellm.aimage_generation -
/v1/images/edits->litellm.aimage_edit -
/v1beta/models/{model}:generateContent->litellm.agenerate_content -
/v1/messages->litellm.acompletion
Quick Start
import litellm
from litellm import CustomLLM, completion, get_llm_provider
class MyCustomLLM(CustomLLM):
def completion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore
my_custom_llm = MyCustomLLM()
litellm.custom_provider_map = [ # KEY STEP - REGISTER HANDLER
{"provider": "my-custom-llm", "custom_handler": my_custom_llm}
]
resp = completion(
model="my-custom-llm/my-fake-model",
messages=[{"role": "user", "content": "Hello world!"}],
)
assert resp.choices[0].message.content == "Hi!"
OpenAI Proxy Usage
- Setup your
custom_handler.pyfile
import litellm
from litellm import CustomLLM, completion, get_llm_provider
class MyCustomLLM(CustomLLM):
def completion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore
async def acompletion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore
my_custom_llm = MyCustomLLM()
- Add to
config.yaml
In the config below, we pass
python_filename: custom_handler.py
custom_handler_instance_name: my_custom_llm. This is defined in Step 1
custom_handler: custom_handler.my_custom_llm
model_list:
- model_name: "test-model"
litellm_params:
model: "openai/text-embedding-ada-002"
- model_name: "my-custom-model"
litellm_params:
model: "my-custom-llm/my-model"
litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
- Test it!
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "my-custom-model",
"messages": [{"role": "user", "content": "Say \"this is a test\" in JSON!"}],
}'
Expected Response
{
"id": "chatcmpl-06f1b9cd-08bc-43f7-9814-a69173921216",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Hi!",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"created": 1721955063,
"model": "gpt-6-luna",
"object": "chat.completion",
"system_fingerprint": null,
"usage": {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30
}
}
Add Streaming Support
Here's a simple example of returning unix epoch seconds for both completion + streaming use-cases.
s/o @Eloy Lafuente for this code example.
import time
from typing import Iterator, AsyncIterator
from litellm.types.utils import GenericStreamingChunk, ModelResponse
from litellm import CustomLLM, completion, acompletion
class UnixTimeLLM(CustomLLM):
def completion(self, *args, **kwargs) -> ModelResponse:
return completion(
model="test/unixtime",
mock_response=str(int(time.time())),
) # type: ignore
async def acompletion(self, *args, **kwargs) -> ModelResponse:
return await acompletion(
model="test/unixtime",
mock_response=str(int(time.time())),
) # type: ignore
def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
generic_streaming_chunk: GenericStreamingChunk = {
"finish_reason": "stop",
"index": 0,
"is_finished": True,
"text": str(int(time.time())),
"tool_use": None,
"usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
}
return generic_streaming_chunk # type: ignore
async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
generic_streaming_chunk: GenericStreamingChunk = {
"finish_reason": "stop",
"index": 0,
"is_finished": True,
"text": str(int(time.time())),
"tool_use": None,
"usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
}
yield generic_streaming_chunk # type: ignore
unixtime = UnixTimeLLM()
Image Generation
- Setup your
custom_handler.pyfile
import litellm
from litellm import CustomLLM
from litellm.types.utils import ImageResponse, ImageObject
class MyCustomLLM(CustomLLM):
async def aimage_generation(self, model: str, prompt: str, model_response: ImageResponse, api_key: Optional[str], api_base: Optional[str], optional_params: dict, logging_obj: Any, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[AsyncHTTPHandler] = None,) -> ImageResponse:
return ImageResponse(
created=int(time.time()),
data=[ImageObject(url="https://example.com/image.png")],
)
my_custom_llm = MyCustomLLM()
- Add to
config.yaml
In the config below, we pass
python_filename: custom_handler.py
custom_handler_instance_name: my_custom_llm. This is defined in Step 1
custom_handler: custom_handler.my_custom_llm
model_list:
- model_name: "test-model"
litellm_params:
model: "openai/text-embedding-ada-002"
- model_name: "my-custom-model"
litellm_params:
model: "my-custom-llm/my-model"
litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
- Test it!
curl -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "my-custom-model",
"prompt": "A cute baby sea otter",
}'
Expected Response
{
"created": 1721955063,
"data": [{"url": "https://example.com/image.png"}],
}
Image Edit
- Setup your
custom_handler.pyfile
import litellm
from litellm import CustomLLM
from litellm.types.utils import ImageResponse, ImageObject
import time
class MyCustomLLM(CustomLLM):
async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
# Your custom image edit logic here
# e.g., call Stability AI, Black Forest Labs, etc.
return ImageResponse(
created=int(time.time()),
data=[ImageObject(url="https://example.com/edited-image.png")],
)
my_custom_llm = MyCustomLLM()
- Add to
config.yaml
In the config below, we pass
python_filename: custom_handler.py
custom_handler_instance_name: my_custom_llm. This is defined in Step 1
custom_handler: custom_handler.my_custom_llm
model_list:
- model_name: "my-custom-image-edit-model"
litellm_params:
model: "my-custom-llm/my-model"
litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
- Test it!
curl -X POST 'http://0.0.0.0:4000/v1/images/edits' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-F 'model=my-custom-image-edit-model' \
-F 'image=@/path/to/image.png' \
-F 'prompt=Make the sky blue'
Expected Response
{
"created": 1721955063,
"data": [{"url": "https://example.com/edited-image.png"}],
}
Video Generation
Video jobs are asynchronous, so a custom video provider is not a CustomLLM handler. It is a BaseVideoConfig subclass that describes the HTTP requests and replies for your backend, and LiteLLM's shared HTTP client sends them. The caller gets a job id back from POST /v1/videos right away and polls GET /v1/videos/{id} until the job is done, so nothing is held open while the video renders.
- Setup your
custom_video.pyfile
The create, status and content routes call the methods at the top. The list, remix and delete methods at the bottom are abstract on BaseVideoConfig too, so they have to exist, and can raise NotImplementedError when your backend has no such call.
import httpx
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.types.videos.main import VideoObject
from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id
class MyVideoConfig(BaseVideoConfig):
def get_supported_openai_params(self, model):
return ["seconds"]
def map_openai_params(self, video_create_optional_params, model, drop_params):
return {k: v for k, v in video_create_optional_params.items() if k == "seconds"}
def validate_environment(self, headers, model, api_key=None, litellm_params=None):
key = (litellm_params.api_key if litellm_params is not None else None) or api_key or ""
return {**headers, "X-Api-Key": key}
def get_complete_url(self, model, api_base, litellm_params):
return f"{api_base.rstrip('/')}/jobs"
def transform_video_create_request(self, model, prompt, api_base, video_create_optional_request_params, litellm_params, headers):
body = {"model": model, "prompt": prompt, "seconds": video_create_optional_request_params.get("seconds")}
return body, [], api_base # (json body, files, url)
def transform_video_create_response(self, model, raw_response, logging_obj, custom_llm_provider=None, request_data=None):
job = raw_response.json()
video_id = encode_video_id_with_provider(job["job_id"], custom_llm_provider, model)
return VideoObject(id=video_id, object="video", status="queued", created_at=0, model=model)
def transform_video_status_retrieve_request(self, video_id, api_base, litellm_params, headers):
return f"{api_base}/{extract_original_video_id(video_id)}", {} # empty body means GET
def transform_video_status_retrieve_response(self, raw_response, logging_obj, custom_llm_provider=None, client=None):
job = raw_response.json()
status = "completed" if job["state"] == "done" else "in_progress"
return VideoObject(id=encode_video_id_with_provider(job["job_id"], custom_llm_provider), object="video", status=status, created_at=0)
def transform_video_content_request(self, video_id, api_base, litellm_params, headers, variant=None):
return f"{api_base}/{extract_original_video_id(video_id)}/file", {}
def transform_video_content_response(self, raw_response, logging_obj, client=None):
return raw_response.content
# The list, remix and delete methods are abstract too. Raise when your backend has no such call.
def transform_video_list_request(self, api_base, litellm_params, headers, after=None, limit=None, order=None, extra_query=None):
return api_base, {}
def transform_video_list_response(self, raw_response, logging_obj, custom_llm_provider=None):
return raw_response.json()
def transform_video_remix_request(self, video_id, prompt, api_base, litellm_params, headers, extra_body=None):
raise NotImplementedError("remix is not supported")
def transform_video_remix_response(self, raw_response, logging_obj, custom_llm_provider=None):
raise NotImplementedError("remix is not supported")
def transform_video_delete_request(self, video_id, api_base, litellm_params, headers):
raise NotImplementedError("delete is not supported")
def transform_video_delete_response(self, raw_response, logging_obj):
raise NotImplementedError("delete is not supported")
my_video_config = MyVideoConfig()
- Add to
config.yaml
The provider name before the / in model is the one you register in custom_provider_map. api_base and api_key on the deployment reach your config through get_complete_url and validate_environment.
model_list:
- model_name: "my-video-model"
litellm_params:
model: "my-video-provider/seedance-1"
api_base: "https://video.internal.example.com"
api_key: os.environ/MY_VIDEO_API_KEY
litellm_settings:
custom_provider_map:
- {"provider": "my-video-provider", "custom_handler": custom_video.my_video_config}
litellm --config /path/to/config.yaml
- Test it!
curl -X POST 'http://0.0.0.0:4000/v1/videos' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{"model": "my-video-model", "prompt": "A cute baby sea otter", "seconds": "8"}'
Expected Response
{"id": "video_bGl0ZWxs...", "object": "video", "status": "queued", "model": "seedance-1", ...}
The id carries your provider name, so the status and content routes find your config without the caller saying which provider it is.
curl 'http://0.0.0.0:4000/v1/videos/video_bGl0ZWxs...' -H "Authorization: Bearer $LITELLM_API_KEY"
curl 'http://0.0.0.0:4000/v1/videos/video_bGl0ZWxs.../content' -H "Authorization: Bearer $LITELLM_API_KEY" --output video.mp4
A provider registered with a CustomLLM handler has no video methods, so a video request for it returns 400 video generation is not supported for <provider>.
Anthropic /v1/messages
- Write the integration for .acompletion
- litellm will transform it to /v1/messages
- Setup your
custom_handler.pyfile
import litellm
from litellm import CustomLLM, completion, get_llm_provider
class MyCustomLLM(CustomLLM):
async def acompletion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore
my_custom_llm = MyCustomLLM()
- Add to
config.yaml
In the config below, we pass
python_filename: custom_handler.py
custom_handler_instance_name: my_custom_llm. This is defined in Step 1
custom_handler: custom_handler.my_custom_llm
model_list:
- model_name: "test-model"
litellm_params:
model: "openai/text-embedding-ada-002"
- model_name: "my-custom-model"
litellm_params:
model: "my-custom-llm/my-model"
litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
- Test it!
curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
-H 'anthropic-version: 2023-06-01' \
-H 'content-type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "my-custom-model",
"max_tokens": 1024,
"messages": [{
"role": "user",
"content": [
{
"type": "text",
"text": "What are the key findings in this document 12?"
}]
}]
}'
Expected Response
{
"id": "chatcmpl-Bm4qEp4h4vCe7Zi4Gud1MAxTWgibO",
"type": "message",
"role": "assistant",
"model": "gpt-6-luna",
"stop_sequence": null,
"usage": {
"input_tokens": 18,
"output_tokens": 44
},
"content": [
{
"type": "text",
"text": "Without the specific document being provided, it is not possible to determine the key findings within it. If you can provide the content or a summary of document 12, I would be happy to help identify the key findings."
}
],
"stop_reason": "end_turn"
}
Google GenAI generateContent
Since v1.111.0
A custom provider serves /v1beta/models/{model}:generateContent in one of two ways.
When the handler overrides agenerate_content, and generate_content for sync SDK calls, LiteLLM passes it the native request. The hook gets contents, the whole generationConfig as config, and tools as the caller sent them, with other request fields like systemInstruction in optional_params. It returns a Gemini GenerateContentResponse, as a dict or a google-genai SDK object, and the caller gets it back as Gemini's camelCase JSON, with spend computed from its usageMetadata. Use this for image models and anything else that needs native settings like imageConfig or responseModalities.
When the handler only has the chat hooks, LiteLLM translates the request to chat completions and the reply back. That path sends text, images, tools, temperature, maxOutputTokens, topP, stopSequences and JSON response schemas. Any other generationConfig key answers 400 unless the deployment sets drop_params: true. The exceptions are responseModalities: ["TEXT"] and candidateCount: 1, which a chat reply already meets.
Streaming (:streamGenerateContent) always goes through the chat hooks, so it needs streaming or astreaming. A handler without them answers 500 before calling anything, and the deployment keeps serving its other requests.
- Setup your
custom_handler.pyfile
import os
import httpx
from litellm import CustomLLM
GEMINI = "https://generativelanguage.googleapis.com/v1beta"
class MyImageLLM(CustomLLM):
async def agenerate_content(self, model, contents, config, tools, optional_params, **kwargs):
async with httpx.AsyncClient(timeout=180) as client:
response = await client.post(
f"{GEMINI}/models/{model}:generateContent",
headers={"x-goog-api-key": os.environ["GEMINI_API_KEY"]},
json={"contents": contents, "generationConfig": config or {}},
)
response.raise_for_status()
return response.json()
my_image_llm = MyImageLLM()
- Add to
config.yaml
Set prices on the deployment so spend is tracked, since LiteLLM has no prices for a custom provider's models.
model_list:
- model_name: my-image-model
litellm_params:
model: my-image-llm/gemini-3.1-flash-image
input_cost_per_token: 0.0000005
output_cost_per_token: 0.00003
litellm_settings:
custom_provider_map:
- {"provider": "my-image-llm", "custom_handler": custom_handler.my_image_llm}
litellm --config /path/to/config.yaml
- Test it!
curl -X POST 'http://0.0.0.0:4000/v1beta/models/my-image-model:generateContent' \
-H 'content-type: application/json' \
-H "x-goog-api-key: $LITELLM_API_KEY" \
-d '{
"contents": [{"role": "user", "parts": [{"text": "A red apple on a white table"}]}],
"generationConfig": {"responseModalities": ["IMAGE"], "imageConfig": {"aspectRatio": "1:1"}}
}'
Expected Response, with the image bytes shortened
{
"candidates": [
{
"content": {
"parts": [
{
"inlineData": {
"mimeType": "image/jpeg",
"data": "/9j/4AAQSkZJRgAB..."
},
"thoughtSignature": "ErjWMgq01jIBaRR9..."
}
],
"role": "model"
},
"finishReason": "STOP",
"index": 0
}
],
"usageMetadata": {
"promptTokenCount": 8,
"candidatesTokenCount": 1536,
"totalTokenCount": 1544,
"candidatesTokensDetails": [{"modality": "IMAGE", "tokenCount": 1120}]
},
"modelVersion": "gemini-3.1-flash-image"
}
Additional Parameters
Additional parameters are passed inside optional_params key in the completion or image_generation function.
Here's how to set this:
- SDK
- Proxy
import litellm
from litellm import CustomLLM, completion, get_llm_provider
class MyCustomLLM(CustomLLM):
def completion(self, *args, **kwargs) -> litellm.ModelResponse:
assert kwargs["optional_params"] == {"my_custom_param": "my-custom-param"} # CHECK HERE
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore
my_custom_llm = MyCustomLLM()
litellm.custom_provider_map = [ # KEY STEP - REGISTER HANDLER
{"provider": "my-custom-llm", "custom_handler": my_custom_llm}
]
resp = completion(model="my-custom-llm/my-model", my_custom_param="my-custom-param")
- Setup your
custom_handler.pyfile
import litellm
from litellm import CustomLLM
from litellm.types.utils import ImageResponse, ImageObject
class MyCustomLLM(CustomLLM):
async def aimage_generation(self, model: str, prompt: str, model_response: ImageResponse, api_key: Optional[str], api_base: Optional[str], optional_params: dict, logging_obj: Any, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[AsyncHTTPHandler] = None,) -> ImageResponse:
assert optional_params == {"my_custom_param": "my-custom-param"} # CHECK HERE
return ImageResponse(
created=int(time.time()),
data=[ImageObject(url="https://example.com/image.png")],
)
my_custom_llm = MyCustomLLM()
- Add to
config.yaml
In the config below, we pass
python_filename: custom_handler.py
custom_handler_instance_name: my_custom_llm. This is defined in Step 1
custom_handler: custom_handler.my_custom_llm
model_list:
- model_name: "test-model"
litellm_params:
model: "openai/text-embedding-ada-002"
- model_name: "my-custom-model"
litellm_params:
model: "my-custom-llm/my-model"
my_custom_param: "my-custom-param" # CUSTOM PARAM
litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
- Test it!
curl -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "my-custom-model",
"prompt": "A cute baby sea otter",
}'
Custom Handler Spec
from litellm.types.utils import GenericStreamingChunk, ModelResponse, ImageResponse
from typing import Iterator, AsyncIterator, Any, Mapping, Optional, Union
from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.router import GenericLiteLLMParams
from litellm.llms.base import BaseLLM
class CustomLLMError(Exception): # use this for all your exceptions
def __init__(
self,
status_code,
message,
):
self.status_code = status_code
self.message = message
super().__init__(
self.message
) # Call the base class constructor with the parameters it needs
class CustomLLM(BaseLLM):
def __init__(self) -> None:
super().__init__()
def completion(self, *args, **kwargs) -> ModelResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def acompletion(self, *args, **kwargs) -> ModelResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def image_generation(
self,
model: str,
prompt: str,
api_key: Optional[str],
api_base: Optional[str],
model_response: ImageResponse,
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[HTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def aimage_generation(
self,
model: str,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def image_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[HTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
def generate_content(
self,
model: str,
contents: object,
config: Optional[Mapping[str, object]],
tools: Optional[object],
litellm_params: GenericLiteLLMParams,
logging_obj: Any,
optional_params: Mapping[str, object],
api_key: Optional[str] = None,
api_base: Optional[str] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[GenerateContentResponse, Mapping[str, object]]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")
async def agenerate_content(
self,
model: str,
contents: object,
config: Optional[Mapping[str, object]],
tools: Optional[object],
litellm_params: GenericLiteLLMParams,
logging_obj: Any,
optional_params: Mapping[str, object],
api_key: Optional[str] = None,
api_base: Optional[str] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[GenerateContentResponse, Mapping[str, object]]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")