Migrating Data Between Langfuse Projects (Python SDK v4)
This cookbook is also available in Japanese.
This notebook migrates data from one Langfuse project to another using the Langfuse Python SDK v4.
Migrating observation and score data into a Langfuse project creates new ingestion events in the destination project. This migration therefore incurs Langfuse ingestion charges based on the observations and scores transferred.
Common use cases:
- Migrating between cloud regions (US ↔ EU ↔ JP ↔ HIPAA)
- Migrating from a self-hosted Langfuse to Langfuse Cloud
What gets migrated
- Score Configs
- Custom Model Definitions
- Prompts (all versions)
- Evaluators (latest LLM-as-a-Judge and code evaluator definitions, via the public API)
- Observations (traces + nested observations, via the OTLP endpoint so original
start_time/end_timeare preserved) - Scores
- Datasets (definitions + items, linked to migrated traces/observations)
What is not migrated
This notebook does not copy media files, custom dashboards, users / RBAC / SSO, or project/organization settings. Recreate these separately. Historical dataset run items are also left behind; see section 7.
Evaluator version history and evaluation rules are not copied by this notebook. See section 4 for how to recreate rules with the migrated evaluator IDs.
0. Setup
This notebook needs Python SDK 4.8.1 or newer — earlier 4.x releases do not include the v3 Scores API client.
%pip install --quiet "langfuse>=4.8.1,<5" "opentelemetry-sdk>=1.25.0" "opentelemetry-exporter-otlp-proto-http>=1.25.0"import os
# --- SOURCE project credentials ---
os.environ.setdefault("LANGFUSE_SOURCE_PUBLIC_KEY", "pk-lf-..");
os.environ.setdefault("LANGFUSE_SOURCE_SECRET_KEY", "sk-lf-...");
os.environ.setdefault("LANGFUSE_SOURCE_HOST", "https://cloud.langfuse.com"); # e.g. EU cloud
# --- DESTINATION project credentials ---
os.environ.setdefault("LANGFUSE_DEST_PUBLIC_KEY", "pk-lf-..");
os.environ.setdefault("LANGFUSE_DEST_SECRET_KEY", "sk-lf-...");
os.environ.setdefault("LANGFUSE_DEST_HOST", "https://hipaa.cloud.langfuse.com"); # e.g. HIPAA cloud
# --- Optional time filter for the observations migration (ISO 8601) ---
# Leave empty to migrate all traces.
os.environ.setdefault("LANGFUSE_MIGRATE_FROM_TIMESTAMP", "2026-03-01T00:00:00Z"); # e.g. "2025-01-01T00:00:00Z"
os.environ.setdefault("LANGFUSE_MIGRATE_TO_TIMESTAMP", "2026-04-20T00:00:00Z"); # e.g. "2025-02-01T00:00:00Z"
import base64
import datetime as dt
import json
import re
import time
from hashlib import sha256
from typing import Any, Dict, List, Optional
from langfuse import Langfuse
SOURCE_CFG = {
"public_key": os.environ["LANGFUSE_SOURCE_PUBLIC_KEY"],
"secret_key": os.environ["LANGFUSE_SOURCE_SECRET_KEY"],
"base_url": os.environ.get("LANGFUSE_SOURCE_HOST", "https://cloud.langfuse.com").rstrip("/"),
}
DEST_CFG = {
"public_key": os.environ["LANGFUSE_DEST_PUBLIC_KEY"],
"secret_key": os.environ["LANGFUSE_DEST_SECRET_KEY"],
"base_url": os.environ.get("LANGFUSE_DEST_HOST", "https://cloud.langfuse.com").rstrip("/"),
}
# Sanity: source and destination must use different public keys. The Langfuse v4
# resource manager is a singleton keyed by public_key, so reusing a key would
# make the second client silently share the first client's base_url/secret_key.
assert SOURCE_CFG["public_key"] != DEST_CFG["public_key"], (
"Source and destination must use different public keys. The Langfuse v4 "
"resource manager is a singleton keyed by public_key; using the same key "
"for both projects will cause the second client to reuse the first client's "
"host and secret key."
)
# Defensively clear any leaked Langfuse env vars that would otherwise be picked
# up by the SDK constructor. In particular, `LANGFUSE_BASE_URL` takes priority
# over the `host=` argument; `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` are
# fallbacks. We always want the explicit per-project values above to win.
for _leak in (
"LANGFUSE_BASE_URL",
"LANGFUSE_HOST",
"LANGFUSE_PUBLIC_KEY",
"LANGFUSE_SECRET_KEY",
"LANGFUSE_TRACING_ENVIRONMENT",
"LANGFUSE_RELEASE",
):
os.environ.pop(_leak, None)
# Two independent Langfuse clients. `tracing_enabled=False` disables the
# background OTel exporter on each client so they only act as REST + helper
# wrappers. The observations section sets up its own dedicated OTLP pipeline.
# Using `base_url=` (not `host=`) because `base_url` has the highest precedence
# in the Langfuse constructor and cannot be overridden by env vars.
# `timeout=60` overrides the SDK default of 5s — trace.get() on large traces
# with many nested observations routinely needs more than 5s to respond.
src = Langfuse(**SOURCE_CFG, tracing_enabled=False, timeout=60)
dst = Langfuse(**DEST_CFG, tracing_enabled=False, timeout=60)
assert src.auth_check(), "Source credentials invalid"
assert dst.auth_check(), "Destination credentials invalid"
print(f"Source: {SOURCE_CFG['base_url']}")
print(f"Destination: {DEST_CFG['base_url']}")
# Shared utilities used across all sections.
_HEX32 = re.compile(r"^[0-9a-f]{32}$")
_HEX16 = re.compile(r"^[0-9a-f]{16}$")
def dumps(obj: Any) -> str:
"""UTF-8-safe JSON serialization (keeps Japanese / non-ASCII readable)."""
return json.dumps(obj, ensure_ascii=False, default=str)
def to_otel_trace_id(source_trace_id: str) -> str:
"""Return a valid 32-hex-char Langfuse/OTel trace ID for a source ID.
- If the source ID is already 32 lowercase hex chars, return it as-is so
the original ID is preserved in the destination project.
- Otherwise, deterministically derive one from the source ID (sha256) so
re-runs always produce the same destination ID.
"""
sid = source_trace_id.lower()
if _HEX32.match(sid) and int(sid, 16):
return sid
return sha256(source_trace_id.encode("utf-8")).hexdigest()[:32]
def to_otel_span_id(source_obs_id: str) -> str:
"""Return a valid 16-hex-char OTel span ID for a source observation ID."""
sid = source_obs_id.lower()
if _HEX16.match(sid) and int(sid, 16):
return sid
return sha256(source_obs_id.encode("utf-8")).hexdigest()[:16]
import httpx
_RETRIABLE_CODES = {429, 500, 502, 503, 504}
# Transient network errors we want to retry on (no status_code attribute).
_RETRIABLE_EXCEPTIONS = (
httpx.TimeoutException, # includes ReadTimeout / ConnectTimeout / WriteTimeout / PoolTimeout
httpx.RemoteProtocolError, # server closed connection mid-response
httpx.ConnectError, # DNS/TCP level failures
httpx.NetworkError, # umbrella catch for socket-level errors
TimeoutError, # stdlib asyncio / socket timeouts
ConnectionError, # stdlib connection resets
)
def with_retries(fn, *, max_retries: int = 5, base_sleep: float = 0.5):
"""Exponential-backoff retry on rate limits / 5xx / transient network errors.
Retries on (a) Fern or httpx HTTP errors with status codes
in {429, 500, 502, 503, 504}, or (b) the transient-network exception
classes listed in `_RETRIABLE_EXCEPTIONS`. Everything else is raised
immediately.
"""
for attempt in range(max_retries):
try:
return fn()
except Exception as exc:
status = getattr(exc, "status_code", None) or getattr(
getattr(exc, "response", None), "status_code", None
)
retriable = status in _RETRIABLE_CODES or isinstance(exc, _RETRIABLE_EXCEPTIONS)
if not retriable or attempt == max_retries - 1:
raise
sleep_for = base_sleep * (2 ** attempt)
label = status if status is not None else type(exc).__name__
print(f" retriable {label} — sleeping {sleep_for:.1f}s")
time.sleep(sleep_for)
def parse_ts(s: Optional[str]) -> Optional[dt.datetime]:
if not s:
return None
if s.endswith("Z"):
s = s[:-1] + "+00:00"
d = dt.datetime.fromisoformat(s)
return d.replace(tzinfo=dt.timezone.utc) if d.tzinfo is None else d
FROM_TS = parse_ts(os.environ.get("LANGFUSE_MIGRATE_FROM_TIMESTAMP") or None)
TO_TS = parse_ts(os.environ.get("LANGFUSE_MIGRATE_TO_TIMESTAMP") or None)
print(f"Time filter: from={FROM_TS} to={TO_TS}")
1. Score Configs
Score configs must be migrated first so that scores created in the destination can reference them by ID. We keep a config_id_map to remap references in later sections.
from langfuse.api import ConfigCategory
config_id_map: Dict[str, str] = {}
existing_by_name = {}
page = 1
while True:
resp = with_retries(lambda: dst.api.score_configs.get(page=page, limit=100))
existing_by_name.update({c.name: c for c in resp.data})
if page >= resp.meta.total_pages:
break
page += 1
page = 1
migrated = skipped = failed = 0
while True:
resp = with_retries(lambda: src.api.score_configs.get(page=page, limit=100))
if not resp.data:
break
for cfg in resp.data:
if cfg.name in existing_by_name:
config_id_map[cfg.id] = existing_by_name[cfg.name].id
skipped += 1
continue
try:
# The Langfuse API only accepts `categories` for CATEGORICAL configs
# (BOOLEAN categories are auto-generated server-side) and only
# accepts `min_value` / `max_value` for NUMERIC configs. Build the
# kwargs conditionally so we don't send fields the server rejects.
create_kwargs: Dict[str, Any] = {
"name": cfg.name,
"data_type": cfg.data_type,
"description": cfg.description,
}
if cfg.data_type == "CATEGORICAL" and cfg.categories:
create_kwargs["categories"] = [
ConfigCategory(label=c.label, value=c.value) for c in cfg.categories
]
if cfg.data_type == "NUMERIC":
if cfg.min_value is not None:
create_kwargs["min_value"] = cfg.min_value
if cfg.max_value is not None:
create_kwargs["max_value"] = cfg.max_value
created = with_retries(lambda: dst.api.score_configs.create(**create_kwargs))
config_id_map[cfg.id] = created.id
existing_by_name[cfg.name] = created
migrated += 1
except Exception as exc:
print(f" failed to migrate score config '{cfg.name}': {exc}")
failed += 1
if page >= getattr(resp.meta, "total_pages", page):
break
page += 1
print(f"Score configs — migrated: {migrated}, skipped (already present): {skipped}, failed: {failed}")
2. Custom Model Definitions
Only user-defined models are migrated. Langfuse-managed models are skipped — they exist on every project by default.
from langfuse.api import PricingTierInput
page = 1
migrated = skipped = failed = 0
while True:
resp = with_retries(lambda: src.api.models.list(page=page, limit=100))
if not resp.data:
break
for m in resp.data:
if getattr(m, "is_langfuse_managed", False):
skipped += 1
continue
try:
# The server rejects requests that mix flat prices with
# pricing_tiers. If the source uses pricing_tiers (e.g. token-
# tiered models like gpt-image-1), forward them and omit the
# flat prices; otherwise send only the flat prices.
create_kwargs: Dict[str, Any] = {
"model_name": m.model_name,
"match_pattern": m.match_pattern,
"start_date": m.start_date,
"unit": m.unit,
"tokenizer_id": m.tokenizer_id,
"tokenizer_config": m.tokenizer_config,
}
if getattr(m, "pricing_tiers", None):
create_kwargs["pricing_tiers"] = [
PricingTierInput(
name=t.name, is_default=t.is_default, priority=t.priority,
conditions=t.conditions, prices=t.prices,
) for t in m.pricing_tiers
]
else:
if m.input_price is not None:
create_kwargs["input_price"] = m.input_price
if m.output_price is not None:
create_kwargs["output_price"] = m.output_price
if m.total_price is not None:
create_kwargs["total_price"] = m.total_price
with_retries(lambda: dst.api.models.create(**create_kwargs))
migrated += 1
except Exception as exc:
if getattr(exc, "status_code", None) == 409 or "already exists" in str(exc).lower():
skipped += 1
else:
print(f" failed to migrate model '{m.model_name}': {exc}")
failed += 1
if page >= getattr(resp.meta, "total_pages", page):
break
page += 1
print(f"Custom models — migrated: {migrated}, skipped: {skipped}, failed: {failed}")
3. Prompts
Every version of every prompt is migrated. The destination assigns fresh version numbers; the original source version is recorded in commit_message so it stays traceable. Tags and labels are preserved as-is. The code records destination versions in prompt_version_map so migrated observations reference the correct prompt version.
from urllib.parse import quote
from langfuse.api import (
CreateChatPromptRequest,
CreateChatPromptType,
CreateTextPromptRequest,
CreateTextPromptType,
Prompt_Chat,
Prompt_Text,
)
prompt_version_map = {}
page = 1
prompt_versions = [] # (name, version) tuples
while True:
resp = with_retries(lambda: src.api.prompts.list(page=page, limit=100))
if not resp.data:
break
for meta in resp.data:
for v in meta.versions:
prompt_versions.append((meta.name, v))
if page >= getattr(resp.meta, "total_pages", page):
break
page += 1
prompt_versions.sort(key=lambda x: (x[0], x[1]))
print(f"Found {len(prompt_versions)} prompt versions on source")
migrated = failed = 0
for name, version in prompt_versions:
try:
# Folder prompts (names containing '/') must have their path segments
# URL-encoded on the GET endpoint — the Fern client does not do this
# automatically. The request body on create() keeps the raw name.
encoded_name = quote(name, safe="")
p = with_retries(lambda: src.api.prompts.get(encoded_name, version=version))
commit_msg = f"Migrated from source. Original version: {version}."
if isinstance(p, Prompt_Chat):
req = CreateChatPromptRequest(
name=p.name,
type=CreateChatPromptType.CHAT,
prompt=p.prompt,
labels=p.labels or [],
tags=p.tags or [],
config=p.config,
commit_message=commit_msg,
)
elif isinstance(p, Prompt_Text):
req = CreateTextPromptRequest(
name=p.name,
type=CreateTextPromptType.TEXT,
prompt=p.prompt,
labels=p.labels or [],
tags=p.tags or [],
config=p.config,
commit_message=commit_msg,
)
else:
print(f" unknown prompt type for {name} v{version}: {type(p).__name__}")
failed += 1
continue
created = with_retries(lambda: dst.api.prompts.create(request=req))
prompt_version_map[(name, version)] = created.version
migrated += 1
except Exception as exc:
print(f" failed {name} v{version}: {exc}")
failed += 1
print(f"Prompts — migrated: {migrated}, failed: {failed}")
4. Evaluators
The stable Evaluators and Evaluation Rules APIs used here are available on Langfuse Cloud and on self-hosted Langfuse v4.23.0 or newer. Both source and destination must support these endpoints.
We use GET /api/public/v2/evaluators with cursor pagination and POST /api/public/v2/evaluators to copy the latest definition of each LLM-as-a-Judge or code evaluator. These requests use HTTP Basic authentication with the same project keys configured in section 0. See the Evaluators API reference.
The destination assigns new evaluator IDs and starts each evaluator at version 1. The code keeps an evaluator_id_map for rule assignments. It copies prompts, output definitions, complete default variable mappings, explicit model choices, and code definitions; historical versions, original authors, timestamps, and paused states are not preserved.
Before running this cell, configure the destination's LLM connections. An evaluator with modelConfig: null uses the destination's default evaluation model. For self-hosted code evaluators, also configure the code evaluator executor. Check each copied evaluator's status in the destination before using it.
Successful creations are skipped if you rerun the cell in the same kernel. Save evaluator_id_map before restarting the notebook: names are not unique identifiers, and rerunning without the map creates duplicates. POST requests are deliberately not retried automatically; after an ambiguous timeout, check the destination before retrying.
# Reuse successful source-to-destination mappings within this notebook session.
evaluator_id_map = globals().get("evaluator_id_map", {})
def evaluation_request(client, method, path, **kwargs):
def send():
response = client.request(method, path, **kwargs)
response.raise_for_status()
return response.json()
return with_retries(send) if method == "GET" else send()
def evaluator_create_body(evaluator):
body = {
"name": evaluator["name"],
"description": evaluator.get("description"),
"type": evaluator["type"],
}
if evaluator["type"] == "llm_as_judge":
mappings = evaluator.get("variableMapping")
if mappings is not None and any(not m.get("source") for m in mappings):
raise ValueError("Complete the source evaluator's default variable mapping first")
body.update(
prompt=evaluator["prompt"],
outputDefinition=evaluator["outputDefinition"],
variableMapping=mappings,
modelConfig=evaluator.get("modelConfig"),
)
elif evaluator["type"] == "code":
body.update(
sourceCode=evaluator["sourceCode"],
sourceCodeLanguage=evaluator["sourceCodeLanguage"],
)
else:
raise ValueError(f"Unsupported evaluator type: {evaluator['type']}")
return body
migrated = skipped = failed = 0
with httpx.Client(
base_url=SOURCE_CFG["base_url"],
auth=(SOURCE_CFG["public_key"], SOURCE_CFG["secret_key"]),
timeout=60,
) as source_api, httpx.Client(
base_url=DEST_CFG["base_url"],
auth=(DEST_CFG["public_key"], DEST_CFG["secret_key"]),
timeout=60,
) as destination_api:
cursor = None
while True:
params = {"limit": 100}
if cursor:
params["cursor"] = cursor
page = evaluation_request(
source_api, "GET", "/api/public/v2/evaluators", params=params
)
for evaluator in page["data"]:
if evaluator["id"] in evaluator_id_map:
skipped += 1
continue
try:
# Only writable definition fields are sent, not IDs or rule assignments.
created = evaluation_request(
destination_api, "POST", "/api/public/v2/evaluators",
json=evaluator_create_body(evaluator),
)
evaluator_id_map[evaluator["id"]] = created["id"]
migrated += 1
print(f" {evaluator['name']}: {created['id']} (status={created['status']})")
except Exception as exc:
print(f" failed evaluator {evaluator['id']} ({evaluator['name']}): {exc}")
failed += 1
cursor = page.get("meta", {}).get("cursor")
if not cursor:
break
print(f"Evaluators — migrated: {migrated}, skipped: {skipped}, failed: {failed}")
print(dumps(evaluator_id_map))
Recreate evaluation rules
Evaluators define how to score data; evaluation rules define which incoming observations to evaluate. Copying an evaluator does not copy or enable its rules.
To migrate rules through the Evaluation Rules API:
- Read the source rules with cursor-paginated
GET /api/public/v2/evaluation-rules. - Build a create body with
name,sampling,filter, andevaluatorAssignments. Replace every assignment'sevaluatorIdwithevaluator_id_map[source_id]and preserve itsvariableMappingoverride. Remap project-specific filter values, such asdatasetId, to destination IDs; dataset names from section 7 can help match datasets. If an evaluator failed to migrate, resolve that failure before copying its rules. - Create each rule with
POST /api/public/v2/evaluation-rules, settingenabled: false. Review its filters, mappings, evaluator status, and sampling in the destination, then enable it withPATCH /api/public/v2/evaluation-rules/{evaluationRuleId}and{"enabled": true}.
Legacy trace and dataset rules may be returned by the read endpoint, but cannot be recreated unchanged. Convert them to observation rules following the evaluator upgrade guide. New rules evaluate incoming observations; copying them does not rerun evaluation on the historical data copied by this notebook.
5. Observations (via OTLP ingestion endpoint)
This is the core of the migration. We push each source trace (and its nested observations) into the destination's OTLP ingestion endpoint at /api/public/otel/v1/traces. The OTLP endpoint is the only way to set the authoritative start_time / end_time of an observation at ingest — the legacy REST ingestion rewrites those fields.
Self-hosted deployments: the v2 Observations API requires Langfuse v4, which is generally available for self-hosted deployments. If your source deployment still runs Langfuse v3, the v2 endpoints are unavailable there: enumerate traces with
src.api.trace.list(page=..., from_timestamp=FROM_TS, to_timestamp=TO_TS)and fetch each trace's observations withsrc.api.legacy.observations_v1.get_many(trace_id=...). Both are deprecated on v4, but they are the only option against a v3 source.
Which traces are migrated: the listing below pages through logical root observations, and deduplicates trace IDs when a trace has multiple app roots. A trace whose root observation never reached Langfuse — for example when only child spans were exported — has no logical root and is skipped. Compare the migrated trace count against the source project's trace count; if they differ, page through observations without the root filter and group the rows by
trace_idto pick up the remainder.
from contextvars import ContextVar
from opentelemetry import trace as otel_trace
from opentelemetry.context import Context
from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags
# Reuse migrated/restored prompt versions; keep source versions if prompts were skipped.
prompt_version_map = globals().get("prompt_version_map", {})
# Force the IDs actually exported to match the score/dataset reference maps.
_migration_trace_id = ContextVar("migration_trace_id", default=None)
_migration_span_id = ContextVar("migration_span_id", default=None)
class MigrationIdGenerator(RandomIdGenerator):
def generate_trace_id(self):
return _migration_trace_id.get() or super().generate_trace_id()
def generate_span_id(self):
return _migration_span_id.get() or super().generate_span_id()
class MigrationSpanExporter(OTLPSpanExporter):
failed_span_ids = []
def export(self, spans):
result = SpanExportResult.FAILURE
try:
result = super().export(spans)
return result
finally:
if result != SpanExportResult.SUCCESS:
self.failed_span_ids.extend(format(s.context.span_id, "016x") for s in spans)
# --- Build a dedicated OTLP pipeline that points at the destination project ---
_auth = base64.b64encode(
f"{DEST_CFG['public_key']}:{DEST_CFG['secret_key']}".encode("utf-8")
).decode("ascii")
otlp_provider = TracerProvider(
resource=Resource.create({"service.name": "langfuse-migration"}),
id_generator=MigrationIdGenerator(),
)
otlp_provider.add_span_processor(
SimpleSpanProcessor(
MigrationSpanExporter(
endpoint=f"{DEST_CFG['base_url']}/api/public/otel/v1/traces",
headers={
"Authorization": f"Basic {_auth}",
"x-langfuse-ingestion-version": "4",
},
)
)
)
migration_tracer = otlp_provider.get_tracer("langfuse-migration")
def _to_ns(d: Optional[dt.datetime]) -> Optional[int]:
if d is None:
return None
if d.tzinfo is None:
d = d.replace(tzinfo=dt.timezone.utc)
return int(d.timestamp() * 1_000_000_000)
def _set_scalar(span, key: str, value: Any) -> None:
"""Set an OTel attribute on `span` stringifying non-primitive values via UTF-8 JSON."""
if value is None:
return
if isinstance(value, (str, bool, int, float)):
span.set_attribute(key, value)
return
span.set_attribute(key, dumps(value))
def _as_attr_string(value: Any) -> Optional[str]:
"""Coerce an input/output/metadata value to a JSON attribute string.
The Observations API v2 returns input/output as raw JSON strings,
while older trace responses may return parsed dicts/lists. Pass strings
through unchanged so we don't end up double-encoded; JSON-encode
everything else with ensure_ascii=False (Japanese-safe).
"""
if value is None:
return None
if isinstance(value, str):
return value
return dumps(value)
def _set_trace_attrs(span, trace_obj) -> None:
"""Set the context associated with this observation."""
trace_name = getattr(trace_obj, "trace_name", None) or trace_obj.name
_set_scalar(span, "langfuse.trace.name", trace_name)
_set_scalar(span, "user.id", trace_obj.user_id)
_set_scalar(span, "session.id", trace_obj.session_id)
_set_scalar(span, "langfuse.release", getattr(trace_obj, "release", None))
_set_scalar(span, "langfuse.version", trace_obj.version)
_set_scalar(span, "langfuse.environment", trace_obj.environment or "default")
if trace_obj.public is not None:
span.set_attribute("langfuse.trace.public", bool(trace_obj.public))
if getattr(trace_obj, "tags", None):
# Langfuse accepts both a string[] attribute and a JSON string.
span.set_attribute("langfuse.trace.tags", list(trace_obj.tags))
_OBS_TYPE_MAP = {
kind: kind.lower() for kind in (
"SPAN", "GENERATION", "EVENT", "AGENT", "TOOL", "CHAIN", "RETRIEVER",
"EVALUATOR", "EMBEDDING", "GUARDRAIL",
)
}
def _set_observation_attrs(span, obs) -> None:
obs_type = _OBS_TYPE_MAP.get(obs.type, "span")
span.set_attribute("langfuse.observation.type", obs_type)
_set_scalar(span, "langfuse.observation.level", obs.level)
_set_scalar(span, "langfuse.observation.status_message", obs.status_message)
_set_scalar(span, "langfuse.version", obs.version)
_set_scalar(span, "langfuse.environment", obs.environment or "default")
input_str = _as_attr_string(obs.input)
if input_str is not None:
span.set_attribute("langfuse.observation.input", input_str)
output_str = _as_attr_string(obs.output)
if output_str is not None:
span.set_attribute("langfuse.observation.output", output_str)
md = obs.metadata
if isinstance(md, dict):
for k, v in md.items():
_set_scalar(span, f"langfuse.observation.metadata.{k}", v)
elif md is not None:
_set_scalar(span, "langfuse.observation.metadata._raw", md)
# Generation-only attributes. ObservationV2 exposes the model name as
# `model`; retain a fallback for older SDK response types.
if obs_type in ("generation", "embedding"):
model_name = getattr(obs, "model", None) or getattr(obs, "provided_model_name", None)
_set_scalar(span, "langfuse.observation.model.name", model_name)
if obs.model_parameters:
span.set_attribute("langfuse.observation.model.parameters", dumps(obs.model_parameters))
if obs.usage_details:
span.set_attribute("langfuse.observation.usage_details", dumps(obs.usage_details))
if obs.cost_details:
span.set_attribute("langfuse.observation.cost_details", dumps(obs.cost_details))
if obs.completion_start_time is not None:
d = obs.completion_start_time
if d.tzinfo is None:
d = d.replace(tzinfo=dt.timezone.utc)
span.set_attribute(
"langfuse.observation.completion_start_time",
d.isoformat().replace("+00:00", "Z"),
)
if getattr(obs, "prompt_name", None):
_set_scalar(span, "langfuse.observation.prompt.name", obs.prompt_name)
if getattr(obs, "prompt_version", None) is not None:
destination_version = prompt_version_map.get(
(obs.prompt_name, obs.prompt_version), obs.prompt_version
)
span.set_attribute("langfuse.observation.prompt.version", int(destination_version))
def _parent_ctx(trace_id_hex: str, parent_span_id_hex: Optional[str]):
if not parent_span_id_hex:
return Context()
parent_sc = SpanContext(
trace_id=int(trace_id_hex, 16),
span_id=int(parent_span_id_hex, 16),
is_remote=True,
trace_flags=TraceFlags(0x01),
)
return otel_trace.set_span_in_context(NonRecordingSpan(parent_sc))
trace_id_map: Dict[str, str] = {}
obs_id_map: Dict[str, str] = {}
# Field groups we need from the v2 Observations API. `core` is always included.
# `basic` gives name/level/status/version/environment; `time` adds
# completionStartTime; `io` adds input/output; `metadata` adds metadata;
# `model` adds model/modelParameters; `usage` adds
# usageDetails/costDetails; `prompt` adds promptName/promptVersion.
OBSERVATIONS_FIELDS = "core,basic,time,io,metadata,model,usage,prompt,trace_context"
def _fetch_observations_v2(source_trace_id: str) -> List[Any]:
"""Fetch all observations for a trace via the v2 API with cursor pagination.
Requires the v2 Observations API (Langfuse Cloud, or self-hosted Langfuse v4+).
On a Langfuse v3 source, swap the call for the deprecated v1 read
`src.api.legacy.observations_v1.get_many(trace_id=source_trace_id, page=..., limit=100)`,
which pages by number instead of by cursor.
"""
results: List[Any] = []
cursor: Optional[str] = None
while True:
resp = with_retries(lambda: src.api.observations.get_many(
trace_id=source_trace_id,
fields=OBSERVATIONS_FIELDS,
limit=1000,
cursor=cursor,
))
metadata_keys = sorted({key for o in resp.data for key in (o.metadata or {})})
if metadata_keys:
# The default API response truncates metadata values at 200 characters.
resp = with_retries(lambda: src.api.observations.get_many(
trace_id=source_trace_id, fields=OBSERVATIONS_FIELDS,
limit=1000, cursor=cursor, expand_metadata=",".join(metadata_keys),
))
if resp.data:
results.extend(resp.data)
cursor = getattr(resp.meta, "cursor", None)
if not cursor:
break
return results
def migrate_one_trace(trace_obj) -> int:
"""Copy the source observations using deterministic trace and span IDs."""
source_trace_id = getattr(trace_obj, "trace_id", None) or trace_obj.id
observations = _fetch_observations_v2(source_trace_id)
new_trace_hex = to_otel_trace_id(source_trace_id)
# Precompute every ID so equal timestamps or child-first API results cannot
# break parent links. A parent's span need not be exported before its child.
local_ids = {o.id: to_otel_span_id(o.id) for o in observations}
for obs in observations:
parent_hex = to_otel_span_id(obs.parent_observation_id) if obs.parent_observation_id else None
trace_token = _migration_trace_id.set(int(new_trace_hex, 16))
span_token = _migration_span_id.set(int(local_ids[obs.id], 16))
try:
span = migration_tracer.start_span(
name=obs.name or obs.type.lower(),
start_time=_to_ns(obs.start_time),
context=_parent_ctx(new_trace_hex, parent_hex),
)
finally:
_migration_span_id.reset(span_token)
_migration_trace_id.reset(trace_token)
try:
_set_trace_attrs(span, obs)
_set_observation_attrs(span, obs)
if getattr(obs, "is_root_observation", False):
span.set_attribute("langfuse.internal.is_app_root", True)
finally:
span.end(end_time=_to_ns(obs.end_time) or _to_ns(obs.start_time))
trace_id_map[source_trace_id] = new_trace_hex
obs_id_map.update(local_ids)
return len(observations)
# --- Drive the migration ---
cursor: Optional[str] = None
limit = 50
total = 0
total_obs = 0
failures = 0
progress_every = 10 # print a progress line every N traces
t_start = time.time()
while True:
# Use the first-class logical-root query parameter. Pass it via request_options
# because SDK 4.8.1 does not expose the is_root_observation keyword yet.
listing = with_retries(lambda: src.api.observations.get_many(
fields="core,basic,io,metadata,trace_context",
limit=limit,
cursor=cursor,
from_start_time=FROM_TS,
to_start_time=TO_TS,
request_options={"additional_query_parameters": {"isRootObservation": True}},
))
cursor = getattr(listing.meta, "cursor", None)
for t in listing.data:
source_trace_id = getattr(t, "trace_id", None) or t.id
if source_trace_id in trace_id_map:
continue
try:
obs_count = migrate_one_trace(t)
total += 1
total_obs += obs_count
if total % progress_every == 0:
elapsed = time.time() - t_start
rate = total / elapsed if elapsed > 0 else 0
print(
f" [{total} traces] observations={total_obs} "
f"failed={failures} rate={rate:.1f} traces/s"
)
except Exception as exc:
source_trace_id = getattr(t, "trace_id", t.id)
print(f" failed trace {source_trace_id}: {exc}")
failures += 1
print(
f" batch complete — cumulative: traces={total}, observations={total_obs}, failed={failures}"
)
if not cursor:
break
otlp_provider.force_flush()
assert not MigrationSpanExporter.failed_span_ids, (
f"OTLP export failed for {len(MigrationSpanExporter.failed_span_ids)} observation(s). "
"Resolve the ingestion error and rerun the observations cell before copying scores or datasets."
)
elapsed = time.time() - t_start
print(
f"\nObservations — migrated: {total} traces / {total_obs} observations, "
f"failed: {failures}, elapsed: {elapsed:.1f}s"
)
print(f" trace_id_map entries: {len(trace_id_map)}")
print(f" obs_id_map entries: {len(obs_id_map)}")
6. Scores
Scores are fetched from the source via the v3 Scores API (api.scores_v3.get_many_v3) and re-created on the destination through the POST /api/public/scores endpoint (api.legacy.score_v1.create). Original id is preserved so re-runs are idempotent. config_id is remapped through config_id_map from Section 1; trace / observation references are remapped through trace_id_map / obs_id_map from Section 5.
Compared to the older v2 endpoint, the v3 API differs in three ways that matter here:
- Cursor pagination — pass the opaque
meta.cursorback until it is empty, instead of counting pages. - Single
subject— each score references its target through onesubjectobject (kind+id) instead of flattrace_id/observation_id/session_id/dataset_run_idfields. Dataset runs are called experiments in v3. Rare legacy scores carrying two subject links (e.g. both a trace and a session) surface only one of them in v3 — the previous flow failed to re-create those scores entirely, so nothing regresses. - Typed
value—valuefollows the score'sdata_type(number for NUMERIC, boolean for BOOLEAN, string for CATEGORICAL and other types) instead of always being numeric alongside a separatestring_value.
A few behavioral notes:
- v2 silently dropped scores whose trace no longer exists, while v3 returns them. If slightly more scores migrate than the v2 flow used to transfer, these previously hidden scores are now carried over — an improvement, not a loss.
- Scores attached to dataset runs (
kind="experiment") are skipped by this section: historical experiment runs are not copied, so the source run ID would be a dangling reference. Re-run experiments on the destination to produce new results. - Score creation is asynchronous: a success response means the score was accepted for ingestion, not that it is already durably written. Verify the destination project once ingestion has caught up.
- The create endpoint does not accept a timestamp, so migrated scores carry the migration time as their timestamp; source score timestamps cannot be preserved through the public API.
Self-hosted deployments: the v3 Scores API is available on Langfuse Cloud and on self-hosted instances from v3.180.0. On older self-hosted instances, use the v2 pattern instead: page-based
src.api.scores.get_many(page=page, limit=100, ...)with flattrace_id/observation_id/session_idfields andstring_valuefor CATEGORICAL scores. The v2 read endpoints are removed on deployments fully migrated to Langfuse v4.
trace_id_map = globals().get("trace_id_map", {})
obs_id_map = globals().get("obs_id_map", {})
config_id_map = globals().get("config_id_map", {})
# Field groups for the v3 Scores API. `core` (id/name/value/dataType/...) is
# always included; `details` adds comment/metadata/configId; `subject` adds
# the subject reference; `annotation` adds authorUserId/queueId.
SCORES_FIELDS = "details,subject,annotation"
cursor: Optional[str] = None
migrated = failed = skipped = 0
while True:
resp = with_retries(lambda: src.api.scores_v3.get_many_v3(
limit=100,
cursor=cursor,
fields=SCORES_FIELDS,
from_timestamp=FROM_TS,
to_timestamp=TO_TS,
))
for s in resp.data:
# A null value must be caught before any conversion — the BOOLEAN
# branch below would otherwise turn a missing value into false.
if s.value is None:
skipped += 1
continue
# Map the v3 subject back to the flat reference fields the create
# endpoint expects: exactly one of traceId / sessionId, and
# observationId additionally requires traceId.
trace_id = observation_id = session_id = None
kind = s.subject.kind if s.subject else None
if kind == "trace":
trace_id = s.subject.id
elif kind == "observation":
if not s.subject.trace_id:
print(f" skipping score {s.id} ({s.name}): observation subject without a trace")
skipped += 1
continue
observation_id = s.subject.id
trace_id = s.subject.trace_id
elif kind == "session":
session_id = s.subject.id
elif kind == "experiment": # v3 name for dataset runs
# The subject id is the source project's run id; the destination
# assigns new run ids, so this reference cannot be carried over.
print(f" skipping score {s.id} ({s.name}): historical experiment runs are not copied; rerun the experiment on the destination")
skipped += 1
continue
else:
print(f" skipping score {s.id} ({s.name}): unsupported subject kind {kind!r}")
skipped += 1
continue
# Remap references
new_trace_id = trace_id_map.get(trace_id, trace_id) if trace_id else None
new_obs_id = obs_id_map.get(observation_id, observation_id) if observation_id else None
new_cfg_id = config_id_map.get(s.config_id, s.config_id) if s.config_id else None
# v3 serves a typed `value`; convert it to what the write path expects.
if s.data_type == "NUMERIC":
value = s.value
elif s.data_type == "BOOLEAN":
value = 1 if s.value else 0 # write path expects numeric 0/1
elif s.data_type == "CATEGORICAL":
value = s.value # already the category string
else:
# Other data types (e.g. TEXT) carry string values; pass them through.
print(f" passing through {s.data_type} score {s.id} ({s.name}) as-is")
value = s.value
try:
# POST /api/public/scores. The endpoint accepts no timestamp, so
# the migrated score is stamped with the migration time. Ingestion
# is asynchronous: success means queued, not yet visible.
with_retries(lambda: dst.api.legacy.score_v1.create(
id=s.id, # preserves ID for idempotent re-runs
name=s.name,
value=value,
data_type=s.data_type,
trace_id=new_trace_id,
observation_id=new_obs_id,
session_id=session_id,
comment=s.comment,
config_id=new_cfg_id,
metadata=s.metadata,
environment=s.environment,
))
migrated += 1
except Exception as exc:
print(f" failed score {s.id} ({s.name}): {exc}")
failed += 1
cursor = getattr(resp.meta, "cursor", None)
if not cursor:
break
print(f"Scores — migrated: {migrated}, skipped: {skipped}, failed: {failed}")7. Datasets and items
We migrate datasets themselves via the api.datasets.create endpoint (including input and expected-output schemas) and items via create_dataset_item (preserving source item IDs).
Historical dataset run items are not copied. The legacy POST /api/public/dataset-run-items endpoint is deprecated, and both of its replacements create new experiment runs rather than backfilling past ones. To reproduce a run on the destination, either re-run the experiment with the Experiments SDK, or emit the item traces yourself with experiment attributes over OTLP.
# Fallback to empty maps when this cell is run standalone (see Scores section).
trace_id_map = globals().get("trace_id_map", {})
obs_id_map = globals().get("obs_id_map", {})
from urllib.parse import quote as _urlquote
def _enc(name: str) -> str:
return _urlquote(name, safe="")
ds_migrated = ds_skipped = ds_failed = 0
items_migrated = items_failed = 0
t_start = time.time()
ds_seen = 0 # count of source datasets processed (new + existing)
page_ds = 1
while True:
dsresp = with_retries(lambda: src.api.datasets.list(page=page_ds, limit=100))
if not dsresp.data:
break
total_ds = getattr(dsresp.meta, "total_items", None)
total_pages = getattr(dsresp.meta, "total_pages", None)
if page_ds == 1 and total_ds is not None:
print(f"Starting dataset migration: {total_ds} dataset(s) across {total_pages} page(s)")
for ds in dsresp.data:
ds_seen += 1
progress = f"{ds_seen}/{total_ds}" if total_ds is not None else str(ds_seen)
print(f" [{progress}] '{ds.name}'")
# Create dataset if not present. Path parameters must be URL-encoded
# for folder datasets (names containing "/").
try:
dst.api.datasets.get(dataset_name=_enc(ds.name))
ds_skipped += 1
print(f" dataset already exists in destination — skipping create")
except Exception as exc:
if getattr(exc, "status_code", None) != 404:
print(f" failed dataset lookup: {exc}")
ds_failed += 1
continue
try:
with_retries(lambda: dst.api.datasets.create(
name=ds.name, description=ds.description, metadata=ds.metadata,
input_schema=ds.input_schema, expected_output_schema=ds.expected_output_schema,
))
ds_migrated += 1
print(f" created dataset in destination")
except Exception as exc2:
print(f" failed to create dataset: {exc2}")
ds_failed += 1
continue
# Items (preserve source IDs for repeatable migrations)
ds_items_before = items_migrated
ds_items_failed_before = items_failed
page_item = 1
while True:
iresp = with_retries(lambda: src.api.dataset_items.list(
dataset_name=ds.name, page=page_item, limit=100
))
if not iresp.data:
break
for it in iresp.data:
try:
with_retries(lambda: dst.create_dataset_item(
dataset_name=ds.name,
id=it.id,
input=it.input,
expected_output=it.expected_output,
metadata=it.metadata,
source_trace_id=trace_id_map.get(it.source_trace_id, it.source_trace_id) if it.source_trace_id else None,
source_observation_id=obs_id_map.get(it.source_observation_id, it.source_observation_id) if it.source_observation_id else None,
status=it.status,
))
items_migrated += 1
except Exception as exc:
print(f" failed item {it.id}: {exc}")
items_failed += 1
if page_item >= getattr(iresp.meta, "total_pages", page_item):
break
page_item += 1
ds_items = items_migrated - ds_items_before
ds_items_fail = items_failed - ds_items_failed_before
print(f" items: {ds_items} migrated, {ds_items_fail} failed")
if page_ds >= getattr(dsresp.meta, "total_pages", page_ds):
break
page_ds += 1
elapsed = time.time() - t_start
print(
f"\nDatasets — new: {ds_migrated}, existing: {ds_skipped}, failed: {ds_failed} | "
f"items: {items_migrated} migrated / {items_failed} failed | "
f"elapsed: {elapsed:.1f}s"
)
print("Historical experiment runs are not copied; rerun experiments on the destination.")
Last updated on