Remote Dataset Runs

from langfuse import get_client
 
dataset = get_client().get_dataset("<dataset_name>")

import { LangfuseClient } from "@langfuse/client";
 
const langfuse = new LangfuseClient();
 
const dataset = await langfuse.dataset.get("<dataset_name>");

from langfuse import get_client, observe
from langfuse.openai import OpenAI
 
@observe
def my_llm_function(question: str):
    response = OpenAI().chat.completions.create(
        model="gpt-4o", messages=[{"role": "user", "content": question}]
    )
    output = response.choices[0].message.content
 
    # Update trace input / output
    get_client().update_current_trace(input=question, output=output)
 
    return output

import { OpenAI } from "openai"
 
import { LangfuseClient } from "@langfuse/client";
import { startActiveObservation } from "@langfuse/tracing";
import { observeOpenAI } from "@langfuse/openai";
 
const myLLMApplication = async (input: string) => {
  return startActiveObservation("my-llm-application", async (span) => {
    const output = await observeOpenAI(new OpenAI()).chat.completions.create({
      model: "gpt-4o",
      messages: [{ role: "user", content: input }],
    });
 
    span.update({ input, output: output.choices[0].message.content });
 
    // return reference to span and output
    // will be simplified in a future version of the SDK
    return [span, output] as const;
  }
};

from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
 
def my_llm_function(question):
  llm = ChatOpenAI(model_name="gpt-4o")
  prompt = ChatPromptTemplate.from_template("Answer the question: {question}")
  chain = prompt | llm
 
  response = chain.invoke(
      {"question": question},
      config={"callbacks": [langfuse_handler]})
 
  return response

import { CallbackHandler } from "@langfuse/langchain";
 
const myLLMApplication = async (input: string) => {
  return startActiveObservation('my_llm_application', async (span) => {
    // ... your Langchain code ...
    const langfuseHandler = new CallbackHandler();
    const output = await chain.invoke({ input }, { callbacks: [langfuseHandler] });
 
    span.update({ input, output });
 
    // return reference to span and output
    // will be simplified in a future version of the SDK
    return [span, output] as const;
  }
}

const runMyLLMApplication = async (input: string, traceId: string) => {
  return startActiveObservation("my_llm_application", async (span) => {
    const output = await generateText({
      model: openai("gpt-4o"),
      maxTokens: 50,
      prompt: input,
      experimental_telemetry: {
        isEnabled: true,
        functionId: "vercel-ai-sdk-example-trace",
      },
    });
 
    span.update({ input, output: output.text });
 
    // return reference to span and output
    // will be simplified in a future version of the SDK
    return [span, output] as const;
  }
};

from langfuse import get_client
from .app import my_llm_application
 
# Load the dataset
dataset = get_client().get_dataset("<dataset_name>")
 
# Loop over the dataset items
for item in dataset.items:
    # Use the item.run() context manager for automatic trace linking
    with item.run(
        run_name="<run_name>",
        run_description="My first run",
        run_metadata={"model": "llama3"},
    ) as root_span:
        # Execute your LLM-app against the dataset item input
        output = my_llm_application.run(item.input)
 
        # Optionally: Add scores computed in your experiment runner, e.g. json equality check
        root_span.score_trace(
            name="<example_eval>",
            value=my_eval_fn(item.input, output, item.expected_output),
            comment="This is a comment",  # optional, useful to add reasoning
        )
 
# Flush the langfuse client to ensure all data is sent to the server at the end of the experiment run
get_client().flush()

import { LangfuseClient } from "@langfuse/client";
 
const langfuse = new LangfuseClient();
 
for (const item of dataset.items) {
  // execute application function and get langfuseObject (trace/span/generation/event)
  // output also returned as it is used to evaluate the run
  // you can also link using ids, see sdk reference for details
  const [span, output] = await myLlmApplication.run(item.input);
 
  // link the execution trace to the dataset item and give it a run_name
  await item.link(span, "<run_name>", {
    description: "My first run", // optional run description
    metadata: { model: "llama3" }, // optional run metadata
  });
 
  // Optionally: Add scores
  langfuse.score.trace(span, {
    name: "<score_name>",
    value: myEvalFunction(item.input, output, item.expectedOutput),
    comment: "This is a comment", // optional, useful to add reasoning
  });
}
 
// Flush the langfuse client to ensure all score data is sent to the server at the end of the experiment run
await langfuse.flush();

from langfuse import get_client
 
# Load the dataset
dataset = get_client().get_dataset("<dataset_name>")
 
# Loop over the dataset items
for item in dataset.items:
    # Langchain callback handler that automatically links the execution trace to the dataset item
    handler = item.get_langchain_handler(run_name="<run_name>")
 
    # Execute application and pass custom handler
    my_langchain_chain.run(item.input, callbacks=[handler])
 
    # Optionally: Add scores computed in your experiment runner, e.g. json equality check
    langfuse.score(trace_id=handler.get_trace_id(), name="my_score", value=1)
 
# Flush the langfuse client to ensure all data is sent to the server at the end of the experiment run
langfuse.flush()

import { LangfuseClient } from "@langfuse/client";
import { CallbackHandler } from "@langfuse/langchain";
...
 
const langfuse = new LangfuseClient()
const runName = "my-dataset-run";
for (const item of dataset.items) {
  const [span, output] = await startActiveObservation('my_llm_application', async (span) => {
    // ... your Langchain code ...
    const langfuseHandler = new CallbackHandler();
    const output = await chain.invoke({ input: item.input }, { callbacks: [langfuseHandler] });
 
    span.update({ input: item.input, output });
 
    return [span, output] as const;
  });
 
  await item.link(span, runName)
 
  // Optionally: Add scores
  langfuse.score.trace(span, {
    name: "test-score",
    value: 0.5,
  });
}
 
await langfuse.flush();

import { LangfuseClient } from "@langfuse/client";
 
const langfuse = new LangfuseClient();
 
// iterate over the dataset items
for (const item of dataset.items) {
  // run application on the dataset item input
  const [span, output] = await runMyLLMApplication(item.input, trace.id);
 
  // link the execution trace to the dataset item and give it a run_name
  await item.link(span, "<run_name>", {
    description: "My first run", // optional run description
    metadata: { model: "gpt-4o" }, // optional run metadata
  });
 
  // Optionally: Add scores
  langfuse.score.trace(span, {
    name: "<score_name>",
    value: myEvalFunction(item.input, output, item.expectedOutput),
    comment: "This is a comment", // optional, useful to add reasoning
  });
}
 
// Flush the langfuse client to ensure all score data is sent to the server at the end of the experiment run
await langfuse.flush();