Langfuseのデータセットとは？効率的なモデル評価の方法を徹底解説

from langfuse import Langfuse
from langfuse.openai import openai
from dotenv import load_dotenv
load_dotenv() 

langfuse = Langfuse()

# データセットの定義
langfuse.create_dataset(
    name="test-data-set",
    # optional description
    description="My first dataset",
    # メタデータの設定
    metadata={
        "author": "Alice",
        "date": "2022-01-01",
        "type": "benchmark"
    }
)

from langfuse import Langfuse
from langfuse.openai import openai
from dotenv import load_dotenv
load_dotenv() 

langfuse = Langfuse()

langfuse.create_dataset_item(
    dataset_name="test-data-set",
    # any python object or value, optional
    input={
        "text": "hello world"
    },
    # any python object or value, optional
    expected_output={
        "text": "hello world"
    },
    # metadata, optional
    metadata={
        "model": "llama3",
    }
)

from datetime import datetime
 
def run_my_custom_llm_app(input, system_prompt):
  messages = [
      {"role":"system", "content": system_prompt},
      {"role":"user", "content": input["country"]}
  ]
 
  generationStartTime = datetime.now()
 
  openai_completion = openai.chat.completions.create(
      model="gpt-3.5-turbo",
      messages=messages
  ).choices[0].message.content
 
  langfuse_generation = langfuse.generation(
    name="guess-countries",
    input=messages,
    output=openai_completion,
    model="gpt-3.5-turbo",
    start_time=generationStartTime,
    end_time=datetime.now()
  )
 
  return openai_completion, langfuse_generation

# データセットの新規作成
langfuse.create_dataset(
    name="capital_cities",
)

# アイテムの定義
local_items = [
    {"input": {"country": "Italy"}, "expected_output": "Rome"},
    {"input": {"country": "Spain"}, "expected_output": "Madrid"},
    {"input": {"country": "Brazil"}, "expected_output": "Brasília"},
    {"input": {"country": "Japan"}, "expected_output": "Tokyo"},
    {"input": {"country": "India"}, "expected_output": "New Delhi"},
    {"input": {"country": "Canada"}, "expected_output": "Ottawa"},
    {"input": {"country": "South Korea"}, "expected_output": "Seoul"},
    {"input": {"country": "Argentina"}, "expected_output": "Buenos Aires"},
    {"input": {"country": "South Africa"}, "expected_output": "Pretoria"},
    {"input": {"country": "Egypt"}, "expected_output": "Cairo"},
]

# データセットにアイテムを追加
for item in local_items:
  langfuse.create_dataset_item(
      dataset_name="capital_cities",
      # any python object or value
      input=item["input"],
      # any python object or value, optional
      expected_output=item["expected_output"]
)

from langchain.evaluation import load_evaluator

#レーベンシュタイン距離を用いた評価方法を定義
def evaluation(output, expected_output):
  evaluator = load_evaluator("string_distance")
  score = evaluator.evaluate_strings(
    prediction=output,
    reference=expected_output,
  )
  return score['score']

def run_experiment(experiment_name, system_prompt):

  # データセットの取得
  dataset = langfuse.get_dataset("capital_cities")
 
  for item in dataset.items:

    # アプリケーションによる予測の生成
    completion, langfuse_generation = run_my_custom_llm_app(item.input, system_prompt)
 
    item.link(langfuse_generation, experiment_name) # pass the observation/generation object or the id
 
    langfuse_generation.score(
      name="string_distance",
      value=evaluation(completion, item.expected_output)
    )

run_experiment(
    "famous_city",
    "The user will input countries, respond with the most famous city in this country"
)
run_experiment(
    "directly_ask",
    "What is the capital of the following country?"
)
run_experiment(
    "asking_specifically",
    "The user will input countries, respond with only the name of the capital"
)
run_experiment(
    "asking_specifically_2nd_try",
    "The user will input countries, respond with only the name of the capital. State only the name of the city."
)