LangSmith for Enterprise AI: Architecture & Integration

import os
from langsmith import Client, evaluate

# Configure environment variables for automatic LangSmith tracing
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_API_KEY"] = "ls_prod_key_884920"
os.environ["LANGCHAIN_PROJECT"] = "enterprise_financial_agent"

client = Client()

# Target prediction function to evaluate
def target_agent_pipeline(inputs: dict) -> dict:
  from openai import OpenAI
  openai_client = OpenAI()
  
  response = openai_client.chat.completions.create(
      model="gpt-4o",
      messages=[{"role": "user", "content": inputs["question"]}],
      temperature=0.0
  )
  return {"answer": response.choices[0].message.content}

# Custom evaluator function
def exact_match_evaluator(run, example) -> dict:
  prediction = run.outputs.get("answer", "")
  expected = example.outputs.get("answer", "")
  score = 1.0 if expected.lower() in prediction.lower() else 0.0
  return {"key": "correctness", "score": score}

def run_production_experiment():
  dataset_name = "financial_faq_benchmark"
  
  # Run evaluation experiment against registered dataset
  results = evaluate(
      target_agent_pipeline,
      data=dataset_name,
      evaluators=[exact_match_evaluator],
      experiment_prefix="gpt4o_v2_prompt"
  )
  print("Experiment successfully published to LangSmith Dashboard!")

if __name__ == "__main__":
  run_production_experiment()