LiteLLM for AI Engineering: OpenAI Format, Load Balancing & Budgeting

import asyncio
from litellm import acompletion, completion_cost

async def generate_with_fallback(prompt: str):
  # Define primary model and automatic fallbacks if primary rate limits or fails
  model_list = ["claude-3-5-sonnet-20241022", "gpt-4o", "bedrock/anthropic.claude-v2"]
  
  response = None
  for model in model_list:
      try:
          print(f"Attempting completion via model: {model}...")
          response = await acompletion(
              model=model,
              messages=[{"role": "user", "content": prompt}],
              temperature=0.2,
              fallbacks=[]
          )
          # Calculate cost of completion in USD using LiteLLM cost calculator
          cost = completion_cost(completion_response=response)
          print(f"Success via {model}! Total Token Cost: {cost}")
          break
      except Exception as e:
          print(f"Model {model} failed with error: {str(e)}. Retrying next model...")

  if response:
      return response.choices[0].message.content
  raise RuntimeError("All configured model providers failed.")

if __name__ == "__main__":
  result = asyncio.run(generate_with_fallback("Explain LiteLLM architecture in 2 sentences."))
  print("Response Output:", result)