Skip to main content

Custom API Server (Custom Format)

Call your custom torch-serve / internal LLM APIs via LiteLLM

info
  • For calling an openai-compatible endpoint, go here
  • For modifying incoming/outgoing calls on proxy, go here

Supported Routes:

  • /v1/chat/completions -> litellm.acompletion

  • /v1/completions -> litellm.atext_completion

  • /v1/embeddings -> litellm.aembedding

  • /v1/images/generations -> litellm.aimage_generation

  • /v1/images/edits -> litellm.aimage_edit

  • /v1beta/models/{model}:generateContent -> litellm.agenerate_content

  • /v1/messages -> litellm.acompletion

Quick Start​

import litellm
from litellm import CustomLLM, completion, get_llm_provider


class MyCustomLLM(CustomLLM):
def completion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore

my_custom_llm = MyCustomLLM()

litellm.custom_provider_map = [ # KEY STEP - REGISTER HANDLER
{"provider": "my-custom-llm", "custom_handler": my_custom_llm}
]

resp = completion(
model="my-custom-llm/my-fake-model",
messages=[{"role": "user", "content": "Hello world!"}],
)

assert resp.choices[0].message.content == "Hi!"

OpenAI Proxy Usage​

  1. Setup your custom_handler.py file
import litellm
from litellm import CustomLLM, completion, get_llm_provider


class MyCustomLLM(CustomLLM):
def completion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore

async def acompletion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore


my_custom_llm = MyCustomLLM()
  1. Add to config.yaml

In the config below, we pass

python_filename: custom_handler.py custom_handler_instance_name: my_custom_llm. This is defined in Step 1

custom_handler: custom_handler.my_custom_llm

model_list:
- model_name: "test-model"
litellm_params:
model: "openai/text-embedding-ada-002"
- model_name: "my-custom-model"
litellm_params:
model: "my-custom-llm/my-model"

litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
  1. Test it!
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "my-custom-model",
"messages": [{"role": "user", "content": "Say \"this is a test\" in JSON!"}],
}'

Expected Response

{
"id": "chatcmpl-06f1b9cd-08bc-43f7-9814-a69173921216",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"message": {
"content": "Hi!",
"role": "assistant",
"tool_calls": null,
"function_call": null
}
}
],
"created": 1721955063,
"model": "gpt-6-luna",
"object": "chat.completion",
"system_fingerprint": null,
"usage": {
"prompt_tokens": 10,
"completion_tokens": 20,
"total_tokens": 30
}
}

Add Streaming Support​

Here's a simple example of returning unix epoch seconds for both completion + streaming use-cases.

s/o @Eloy Lafuente for this code example.

import time
from typing import Iterator, AsyncIterator
from litellm.types.utils import GenericStreamingChunk, ModelResponse
from litellm import CustomLLM, completion, acompletion

class UnixTimeLLM(CustomLLM):
def completion(self, *args, **kwargs) -> ModelResponse:
return completion(
model="test/unixtime",
mock_response=str(int(time.time())),
) # type: ignore

async def acompletion(self, *args, **kwargs) -> ModelResponse:
return await acompletion(
model="test/unixtime",
mock_response=str(int(time.time())),
) # type: ignore

def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
generic_streaming_chunk: GenericStreamingChunk = {
"finish_reason": "stop",
"index": 0,
"is_finished": True,
"text": str(int(time.time())),
"tool_use": None,
"usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
}
return generic_streaming_chunk # type: ignore

async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
generic_streaming_chunk: GenericStreamingChunk = {
"finish_reason": "stop",
"index": 0,
"is_finished": True,
"text": str(int(time.time())),
"tool_use": None,
"usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
}
yield generic_streaming_chunk # type: ignore

unixtime = UnixTimeLLM()

Image Generation​

  1. Setup your custom_handler.py file
import litellm
from litellm import CustomLLM
from litellm.types.utils import ImageResponse, ImageObject


class MyCustomLLM(CustomLLM):
async def aimage_generation(self, model: str, prompt: str, model_response: ImageResponse, api_key: Optional[str], api_base: Optional[str], optional_params: dict, logging_obj: Any, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[AsyncHTTPHandler] = None,) -> ImageResponse:
return ImageResponse(
created=int(time.time()),
data=[ImageObject(url="https://example.com/image.png")],
)

my_custom_llm = MyCustomLLM()
  1. Add to config.yaml

In the config below, we pass

python_filename: custom_handler.py custom_handler_instance_name: my_custom_llm. This is defined in Step 1

custom_handler: custom_handler.my_custom_llm

model_list:
- model_name: "test-model"
litellm_params:
model: "openai/text-embedding-ada-002"
- model_name: "my-custom-model"
litellm_params:
model: "my-custom-llm/my-model"

litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
  1. Test it!
curl -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "my-custom-model",
"prompt": "A cute baby sea otter",
}'

Expected Response

{
"created": 1721955063,
"data": [{"url": "https://example.com/image.png"}],
}

Image Edit​

  1. Setup your custom_handler.py file
import litellm
from litellm import CustomLLM
from litellm.types.utils import ImageResponse, ImageObject
import time

class MyCustomLLM(CustomLLM):
async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
# Your custom image edit logic here
# e.g., call Stability AI, Black Forest Labs, etc.
return ImageResponse(
created=int(time.time()),
data=[ImageObject(url="https://example.com/edited-image.png")],
)

my_custom_llm = MyCustomLLM()
  1. Add to config.yaml

In the config below, we pass

python_filename: custom_handler.py custom_handler_instance_name: my_custom_llm. This is defined in Step 1

custom_handler: custom_handler.my_custom_llm

model_list:
- model_name: "my-custom-image-edit-model"
litellm_params:
model: "my-custom-llm/my-model"

litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
  1. Test it!
curl -X POST 'http://0.0.0.0:4000/v1/images/edits' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-F 'model=my-custom-image-edit-model' \
-F 'image=@/path/to/image.png' \
-F 'prompt=Make the sky blue'

Expected Response

{
"created": 1721955063,
"data": [{"url": "https://example.com/edited-image.png"}],
}

Video Generation​

Video jobs are asynchronous, so a custom video provider is not a CustomLLM handler. It is a BaseVideoConfig subclass that describes the HTTP requests and replies for your backend, and LiteLLM's shared HTTP client sends them. The caller gets a job id back from POST /v1/videos right away and polls GET /v1/videos/{id} until the job is done, so nothing is held open while the video renders.

  1. Setup your custom_video.py file

The create, status and content routes call the methods at the top. The list, remix and delete methods at the bottom are abstract on BaseVideoConfig too, so they have to exist, and can raise NotImplementedError when your backend has no such call.

import httpx
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.types.videos.main import VideoObject
from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id


class MyVideoConfig(BaseVideoConfig):
def get_supported_openai_params(self, model):
return ["seconds"]

def map_openai_params(self, video_create_optional_params, model, drop_params):
return {k: v for k, v in video_create_optional_params.items() if k == "seconds"}

def validate_environment(self, headers, model, api_key=None, litellm_params=None):
key = (litellm_params.api_key if litellm_params is not None else None) or api_key or ""
return {**headers, "X-Api-Key": key}

def get_complete_url(self, model, api_base, litellm_params):
return f"{api_base.rstrip('/')}/jobs"

def transform_video_create_request(self, model, prompt, api_base, video_create_optional_request_params, litellm_params, headers):
body = {"model": model, "prompt": prompt, "seconds": video_create_optional_request_params.get("seconds")}
return body, [], api_base # (json body, files, url)

def transform_video_create_response(self, model, raw_response, logging_obj, custom_llm_provider=None, request_data=None):
job = raw_response.json()
video_id = encode_video_id_with_provider(job["job_id"], custom_llm_provider, model)
return VideoObject(id=video_id, object="video", status="queued", created_at=0, model=model)

def transform_video_status_retrieve_request(self, video_id, api_base, litellm_params, headers):
return f"{api_base}/{extract_original_video_id(video_id)}", {} # empty body means GET

def transform_video_status_retrieve_response(self, raw_response, logging_obj, custom_llm_provider=None, client=None):
job = raw_response.json()
status = "completed" if job["state"] == "done" else "in_progress"
return VideoObject(id=encode_video_id_with_provider(job["job_id"], custom_llm_provider), object="video", status=status, created_at=0)

def transform_video_content_request(self, video_id, api_base, litellm_params, headers, variant=None):
return f"{api_base}/{extract_original_video_id(video_id)}/file", {}

def transform_video_content_response(self, raw_response, logging_obj, client=None):
return raw_response.content

# The list, remix and delete methods are abstract too. Raise when your backend has no such call.
def transform_video_list_request(self, api_base, litellm_params, headers, after=None, limit=None, order=None, extra_query=None):
return api_base, {}

def transform_video_list_response(self, raw_response, logging_obj, custom_llm_provider=None):
return raw_response.json()

def transform_video_remix_request(self, video_id, prompt, api_base, litellm_params, headers, extra_body=None):
raise NotImplementedError("remix is not supported")

def transform_video_remix_response(self, raw_response, logging_obj, custom_llm_provider=None):
raise NotImplementedError("remix is not supported")

def transform_video_delete_request(self, video_id, api_base, litellm_params, headers):
raise NotImplementedError("delete is not supported")

def transform_video_delete_response(self, raw_response, logging_obj):
raise NotImplementedError("delete is not supported")


my_video_config = MyVideoConfig()
  1. Add to config.yaml

The provider name before the / in model is the one you register in custom_provider_map. api_base and api_key on the deployment reach your config through get_complete_url and validate_environment.

model_list:
- model_name: "my-video-model"
litellm_params:
model: "my-video-provider/seedance-1"
api_base: "https://video.internal.example.com"
api_key: os.environ/MY_VIDEO_API_KEY

litellm_settings:
custom_provider_map:
- {"provider": "my-video-provider", "custom_handler": custom_video.my_video_config}
litellm --config /path/to/config.yaml
  1. Test it!
curl -X POST 'http://0.0.0.0:4000/v1/videos' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{"model": "my-video-model", "prompt": "A cute baby sea otter", "seconds": "8"}'

Expected Response

{"id": "video_bGl0ZWxs...", "object": "video", "status": "queued", "model": "seedance-1", ...}

The id carries your provider name, so the status and content routes find your config without the caller saying which provider it is.

curl 'http://0.0.0.0:4000/v1/videos/video_bGl0ZWxs...' -H "Authorization: Bearer $LITELLM_API_KEY"
curl 'http://0.0.0.0:4000/v1/videos/video_bGl0ZWxs.../content' -H "Authorization: Bearer $LITELLM_API_KEY" --output video.mp4

A provider registered with a CustomLLM handler has no video methods, so a video request for it returns 400 video generation is not supported for <provider>.

Anthropic /v1/messages​

  • Write the integration for .acompletion
  • litellm will transform it to /v1/messages
  1. Setup your custom_handler.py file
import litellm
from litellm import CustomLLM, completion, get_llm_provider


class MyCustomLLM(CustomLLM):
async def acompletion(self, *args, **kwargs) -> litellm.ModelResponse:
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore


my_custom_llm = MyCustomLLM()
  1. Add to config.yaml

In the config below, we pass

python_filename: custom_handler.py custom_handler_instance_name: my_custom_llm. This is defined in Step 1

custom_handler: custom_handler.my_custom_llm

model_list:
- model_name: "test-model"
litellm_params:
model: "openai/text-embedding-ada-002"
- model_name: "my-custom-model"
litellm_params:
model: "my-custom-llm/my-model"

litellm_settings:
custom_provider_map:
- {"provider": "my-custom-llm", "custom_handler": custom_handler.my_custom_llm}
litellm --config /path/to/config.yaml
  1. Test it!
curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
-H 'anthropic-version: 2023-06-01' \
-H 'content-type: application/json' \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "my-custom-model",
"max_tokens": 1024,
"messages": [{
"role": "user",
"content": [
{
"type": "text",
"text": "What are the key findings in this document 12?"
}]
}]
}'

Expected Response

{
"id": "chatcmpl-Bm4qEp4h4vCe7Zi4Gud1MAxTWgibO",
"type": "message",
"role": "assistant",
"model": "gpt-6-luna",
"stop_sequence": null,
"usage": {
"input_tokens": 18,
"output_tokens": 44
},
"content": [
{
"type": "text",
"text": "Without the specific document being provided, it is not possible to determine the key findings within it. If you can provide the content or a summary of document 12, I would be happy to help identify the key findings."
}
],
"stop_reason": "end_turn"
}

Google GenAI generateContent​

Since v1.111.0

A custom provider serves /v1beta/models/{model}:generateContent in one of two ways.

When the handler overrides agenerate_content, and generate_content for sync SDK calls, LiteLLM passes it the native request. The hook gets contents, the whole generationConfig as config, and tools as the caller sent them, with other request fields like systemInstruction in optional_params. It returns a Gemini GenerateContentResponse, as a dict or a google-genai SDK object, and the caller gets it back as Gemini's camelCase JSON, with spend computed from its usageMetadata. Use this for image models and anything else that needs native settings like imageConfig or responseModalities.

When the handler only has the chat hooks, LiteLLM translates the request to chat completions and the reply back. That path sends text, images, tools, temperature, maxOutputTokens, topP, stopSequences and JSON response schemas. Any other generationConfig key answers 400 unless the deployment sets drop_params: true. The exceptions are responseModalities: ["TEXT"] and candidateCount: 1, which a chat reply already meets.

Streaming (:streamGenerateContent) always goes through the chat hooks, so it needs streaming or astreaming. A handler without them answers 500 before calling anything, and the deployment keeps serving its other requests.

  1. Setup your custom_handler.py file
import os

import httpx
from litellm import CustomLLM

GEMINI = "https://generativelanguage.googleapis.com/v1beta"


class MyImageLLM(CustomLLM):
async def agenerate_content(self, model, contents, config, tools, optional_params, **kwargs):
async with httpx.AsyncClient(timeout=180) as client:
response = await client.post(
f"{GEMINI}/models/{model}:generateContent",
headers={"x-goog-api-key": os.environ["GEMINI_API_KEY"]},
json={"contents": contents, "generationConfig": config or {}},
)
response.raise_for_status()
return response.json()


my_image_llm = MyImageLLM()
  1. Add to config.yaml

Set prices on the deployment so spend is tracked, since LiteLLM has no prices for a custom provider's models.

model_list:
- model_name: my-image-model
litellm_params:
model: my-image-llm/gemini-3.1-flash-image
input_cost_per_token: 0.0000005
output_cost_per_token: 0.00003

litellm_settings:
custom_provider_map:
- {"provider": "my-image-llm", "custom_handler": custom_handler.my_image_llm}
litellm --config /path/to/config.yaml
  1. Test it!
curl -X POST 'http://0.0.0.0:4000/v1beta/models/my-image-model:generateContent' \
-H 'content-type: application/json' \
-H "x-goog-api-key: $LITELLM_API_KEY" \
-d '{
"contents": [{"role": "user", "parts": [{"text": "A red apple on a white table"}]}],
"generationConfig": {"responseModalities": ["IMAGE"], "imageConfig": {"aspectRatio": "1:1"}}
}'

Expected Response, with the image bytes shortened

{
"candidates": [
{
"content": {
"parts": [
{
"inlineData": {
"mimeType": "image/jpeg",
"data": "/9j/4AAQSkZJRgAB..."
},
"thoughtSignature": "ErjWMgq01jIBaRR9..."
}
],
"role": "model"
},
"finishReason": "STOP",
"index": 0
}
],
"usageMetadata": {
"promptTokenCount": 8,
"candidatesTokenCount": 1536,
"totalTokenCount": 1544,
"candidatesTokensDetails": [{"modality": "IMAGE", "tokenCount": 1120}]
},
"modelVersion": "gemini-3.1-flash-image"
}

Additional Parameters​

Additional parameters are passed inside optional_params key in the completion or image_generation function.

Here's how to set this:

import litellm
from litellm import CustomLLM, completion, get_llm_provider


class MyCustomLLM(CustomLLM):
def completion(self, *args, **kwargs) -> litellm.ModelResponse:
assert kwargs["optional_params"] == {"my_custom_param": "my-custom-param"} # CHECK HERE
return litellm.completion(
model="gpt-6-luna",
messages=[{"role": "user", "content": "Hello world"}],
mock_response="Hi!",
) # type: ignore

my_custom_llm = MyCustomLLM()

litellm.custom_provider_map = [ # KEY STEP - REGISTER HANDLER
{"provider": "my-custom-llm", "custom_handler": my_custom_llm}
]

resp = completion(model="my-custom-llm/my-model", my_custom_param="my-custom-param")

Custom Handler Spec​

from litellm.types.utils import GenericStreamingChunk, ModelResponse, ImageResponse
from typing import Iterator, AsyncIterator, Any, Mapping, Optional, Union

from litellm.types.google_genai.main import GenerateContentResponse
from litellm.types.router import GenericLiteLLMParams
from litellm.llms.base import BaseLLM

class CustomLLMError(Exception): # use this for all your exceptions
def __init__(
self,
status_code,
message,
):
self.status_code = status_code
self.message = message
super().__init__(
self.message
) # Call the base class constructor with the parameters it needs

class CustomLLM(BaseLLM):
def __init__(self) -> None:
super().__init__()

def completion(self, *args, **kwargs) -> ModelResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

async def acompletion(self, *args, **kwargs) -> ModelResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

def image_generation(
self,
model: str,
prompt: str,
api_key: Optional[str],
api_base: Optional[str],
model_response: ImageResponse,
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[HTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

async def aimage_generation(
self,
model: str,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

def image_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[HTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

async def aimage_edit(
self,
model: str,
image: Any,
prompt: str,
model_response: ImageResponse,
api_key: Optional[str],
api_base: Optional[str],
optional_params: dict,
logging_obj: Any,
timeout: Optional[Union[float, httpx.Timeout]] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> ImageResponse:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

def generate_content(
self,
model: str,
contents: object,
config: Optional[Mapping[str, object]],
tools: Optional[object],
litellm_params: GenericLiteLLMParams,
logging_obj: Any,
optional_params: Mapping[str, object],
api_key: Optional[str] = None,
api_base: Optional[str] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[GenerateContentResponse, Mapping[str, object]]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")

async def agenerate_content(
self,
model: str,
contents: object,
config: Optional[Mapping[str, object]],
tools: Optional[object],
litellm_params: GenericLiteLLMParams,
logging_obj: Any,
optional_params: Mapping[str, object],
api_key: Optional[str] = None,
api_base: Optional[str] = None,
timeout: Optional[Union[float, httpx.Timeout]] = None,
) -> Union[GenerateContentResponse, Mapping[str, object]]:
raise CustomLLMError(status_code=500, message="Not implemented yet!")