Langfuse launches AI Gateway Learn more
GuidesData Migration Script
This is a Jupyter notebook

Migrating Data Between Langfuse Projects (Python SDK v4)

This cookbook is also available in Japanese.

This notebook migrates data from one Langfuse project to another using the Langfuse Python SDK v4.

Migrating observation and score data into a Langfuse project creates new ingestion events in the destination project. This migration therefore incurs Langfuse ingestion charges based on the observations and scores transferred.

Common use cases:

  • Migrating between cloud regions (US ↔ EU ↔ JP ↔ HIPAA)
  • Migrating from a self-hosted Langfuse to Langfuse Cloud

What gets migrated

  1. Score Configs
  2. Custom Model Definitions
  3. Prompts (all versions)
  4. Evaluators (latest LLM-as-a-Judge and code evaluator definitions, via the public API)
  5. Observations (traces + nested observations, via the OTLP endpoint so original start_time / end_time are preserved)
  6. Scores
  7. Datasets (definitions + items, linked to migrated traces/observations)

What is not migrated

This notebook does not copy media files, custom dashboards, users / RBAC / SSO, or project/organization settings. Recreate these separately. Historical dataset run items are also left behind; see section 7.

Evaluator version history and evaluation rules are not copied by this notebook. See section 4 for how to recreate rules with the migrated evaluator IDs.

0. Setup

This notebook needs Python SDK 4.8.1 or newer — earlier 4.x releases do not include the v3 Scores API client.

%pip install --quiet "langfuse>=4.8.1,<5" "opentelemetry-sdk>=1.25.0" "opentelemetry-exporter-otlp-proto-http>=1.25.0"
import os

# --- SOURCE project credentials ---
os.environ.setdefault("LANGFUSE_SOURCE_PUBLIC_KEY", "pk-lf-..");
os.environ.setdefault("LANGFUSE_SOURCE_SECRET_KEY", "sk-lf-...");
os.environ.setdefault("LANGFUSE_SOURCE_HOST", "https://cloud.langfuse.com");  # e.g. EU cloud

# --- DESTINATION project credentials ---
os.environ.setdefault("LANGFUSE_DEST_PUBLIC_KEY", "pk-lf-..");
os.environ.setdefault("LANGFUSE_DEST_SECRET_KEY", "sk-lf-...");
os.environ.setdefault("LANGFUSE_DEST_HOST", "https://hipaa.cloud.langfuse.com");  # e.g. HIPAA cloud

# --- Optional time filter for the observations migration (ISO 8601) ---
# Leave empty to migrate all traces.
os.environ.setdefault("LANGFUSE_MIGRATE_FROM_TIMESTAMP", "2026-03-01T00:00:00Z");  # e.g. "2025-01-01T00:00:00Z"
os.environ.setdefault("LANGFUSE_MIGRATE_TO_TIMESTAMP", "2026-04-20T00:00:00Z");    # e.g. "2025-02-01T00:00:00Z"
import base64
import datetime as dt
import json
import re
import time
from hashlib import sha256
from typing import Any, Dict, List, Optional

from langfuse import Langfuse

SOURCE_CFG = {
    "public_key": os.environ["LANGFUSE_SOURCE_PUBLIC_KEY"],
    "secret_key": os.environ["LANGFUSE_SOURCE_SECRET_KEY"],
    "base_url": os.environ.get("LANGFUSE_SOURCE_HOST", "https://cloud.langfuse.com").rstrip("/"),
}
DEST_CFG = {
    "public_key": os.environ["LANGFUSE_DEST_PUBLIC_KEY"],
    "secret_key": os.environ["LANGFUSE_DEST_SECRET_KEY"],
    "base_url": os.environ.get("LANGFUSE_DEST_HOST", "https://cloud.langfuse.com").rstrip("/"),
}

# Sanity: source and destination must use different public keys. The Langfuse v4
# resource manager is a singleton keyed by public_key, so reusing a key would
# make the second client silently share the first client's base_url/secret_key.
assert SOURCE_CFG["public_key"] != DEST_CFG["public_key"], (
    "Source and destination must use different public keys. The Langfuse v4 "
    "resource manager is a singleton keyed by public_key; using the same key "
    "for both projects will cause the second client to reuse the first client's "
    "host and secret key."
)

# Defensively clear any leaked Langfuse env vars that would otherwise be picked
# up by the SDK constructor. In particular, `LANGFUSE_BASE_URL` takes priority
# over the `host=` argument; `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` are
# fallbacks. We always want the explicit per-project values above to win.
for _leak in (
    "LANGFUSE_BASE_URL",
    "LANGFUSE_HOST",
    "LANGFUSE_PUBLIC_KEY",
    "LANGFUSE_SECRET_KEY",
    "LANGFUSE_TRACING_ENVIRONMENT",
    "LANGFUSE_RELEASE",
):
    os.environ.pop(_leak, None)

# Two independent Langfuse clients. `tracing_enabled=False` disables the
# background OTel exporter on each client so they only act as REST + helper
# wrappers. The observations section sets up its own dedicated OTLP pipeline.
# Using `base_url=` (not `host=`) because `base_url` has the highest precedence
# in the Langfuse constructor and cannot be overridden by env vars.
# `timeout=60` overrides the SDK default of 5s — trace.get() on large traces
# with many nested observations routinely needs more than 5s to respond.
src = Langfuse(**SOURCE_CFG, tracing_enabled=False, timeout=60)
dst = Langfuse(**DEST_CFG, tracing_enabled=False, timeout=60)

assert src.auth_check(), "Source credentials invalid"
assert dst.auth_check(), "Destination credentials invalid"
print(f"Source:      {SOURCE_CFG['base_url']}")
print(f"Destination: {DEST_CFG['base_url']}")
# Shared utilities used across all sections.

_HEX32 = re.compile(r"^[0-9a-f]{32}$")
_HEX16 = re.compile(r"^[0-9a-f]{16}$")


def dumps(obj: Any) -> str:
    """UTF-8-safe JSON serialization (keeps Japanese / non-ASCII readable)."""
    return json.dumps(obj, ensure_ascii=False, default=str)


def to_otel_trace_id(source_trace_id: str) -> str:
    """Return a valid 32-hex-char Langfuse/OTel trace ID for a source ID.

    - If the source ID is already 32 lowercase hex chars, return it as-is so
      the original ID is preserved in the destination project.
    - Otherwise, deterministically derive one from the source ID (sha256) so
      re-runs always produce the same destination ID.
    """
    sid = source_trace_id.lower()
    if _HEX32.match(sid) and int(sid, 16):
        return sid
    return sha256(source_trace_id.encode("utf-8")).hexdigest()[:32]


def to_otel_span_id(source_obs_id: str) -> str:
    """Return a valid 16-hex-char OTel span ID for a source observation ID."""
    sid = source_obs_id.lower()
    if _HEX16.match(sid) and int(sid, 16):
        return sid
    return sha256(source_obs_id.encode("utf-8")).hexdigest()[:16]


import httpx

_RETRIABLE_CODES = {429, 500, 502, 503, 504}
# Transient network errors we want to retry on (no status_code attribute).
_RETRIABLE_EXCEPTIONS = (
    httpx.TimeoutException,          # includes ReadTimeout / ConnectTimeout / WriteTimeout / PoolTimeout
    httpx.RemoteProtocolError,       # server closed connection mid-response
    httpx.ConnectError,              # DNS/TCP level failures
    httpx.NetworkError,              # umbrella catch for socket-level errors
    TimeoutError,                    # stdlib asyncio / socket timeouts
    ConnectionError,                 # stdlib connection resets
)


def with_retries(fn, *, max_retries: int = 5, base_sleep: float = 0.5):
    """Exponential-backoff retry on rate limits / 5xx / transient network errors.

    Retries on (a) Fern or httpx HTTP errors with status codes
    in {429, 500, 502, 503, 504}, or (b) the transient-network exception
    classes listed in `_RETRIABLE_EXCEPTIONS`. Everything else is raised
    immediately.
    """
    for attempt in range(max_retries):
        try:
            return fn()
        except Exception as exc:
            status = getattr(exc, "status_code", None) or getattr(
                getattr(exc, "response", None), "status_code", None
            )
            retriable = status in _RETRIABLE_CODES or isinstance(exc, _RETRIABLE_EXCEPTIONS)
            if not retriable or attempt == max_retries - 1:
                raise
            sleep_for = base_sleep * (2 ** attempt)
            label = status if status is not None else type(exc).__name__
            print(f"    retriable {label} — sleeping {sleep_for:.1f}s")
            time.sleep(sleep_for)


def parse_ts(s: Optional[str]) -> Optional[dt.datetime]:
    if not s:
        return None
    if s.endswith("Z"):
        s = s[:-1] + "+00:00"
    d = dt.datetime.fromisoformat(s)
    return d.replace(tzinfo=dt.timezone.utc) if d.tzinfo is None else d


FROM_TS = parse_ts(os.environ.get("LANGFUSE_MIGRATE_FROM_TIMESTAMP") or None)
TO_TS = parse_ts(os.environ.get("LANGFUSE_MIGRATE_TO_TIMESTAMP") or None)
print(f"Time filter: from={FROM_TS}  to={TO_TS}")

1. Score Configs

Score configs must be migrated first so that scores created in the destination can reference them by ID. We keep a config_id_map to remap references in later sections.

from langfuse.api import ConfigCategory

config_id_map: Dict[str, str] = {}
existing_by_name = {}
page = 1
while True:
    resp = with_retries(lambda: dst.api.score_configs.get(page=page, limit=100))
    existing_by_name.update({c.name: c for c in resp.data})
    if page >= resp.meta.total_pages:
        break
    page += 1

page = 1
migrated = skipped = failed = 0
while True:
    resp = with_retries(lambda: src.api.score_configs.get(page=page, limit=100))
    if not resp.data:
        break
    for cfg in resp.data:
        if cfg.name in existing_by_name:
            config_id_map[cfg.id] = existing_by_name[cfg.name].id
            skipped += 1
            continue
        try:
            # The Langfuse API only accepts `categories` for CATEGORICAL configs
            # (BOOLEAN categories are auto-generated server-side) and only
            # accepts `min_value` / `max_value` for NUMERIC configs. Build the
            # kwargs conditionally so we don't send fields the server rejects.
            create_kwargs: Dict[str, Any] = {
                "name": cfg.name,
                "data_type": cfg.data_type,
                "description": cfg.description,
            }
            if cfg.data_type == "CATEGORICAL" and cfg.categories:
                create_kwargs["categories"] = [
                    ConfigCategory(label=c.label, value=c.value) for c in cfg.categories
                ]
            if cfg.data_type == "NUMERIC":
                if cfg.min_value is not None:
                    create_kwargs["min_value"] = cfg.min_value
                if cfg.max_value is not None:
                    create_kwargs["max_value"] = cfg.max_value
            created = with_retries(lambda: dst.api.score_configs.create(**create_kwargs))
            config_id_map[cfg.id] = created.id
            existing_by_name[cfg.name] = created
            migrated += 1
        except Exception as exc:
            print(f"  failed to migrate score config '{cfg.name}': {exc}")
            failed += 1
    if page >= getattr(resp.meta, "total_pages", page):
        break
    page += 1

print(f"Score configs — migrated: {migrated}, skipped (already present): {skipped}, failed: {failed}")

2. Custom Model Definitions

Only user-defined models are migrated. Langfuse-managed models are skipped — they exist on every project by default.

from langfuse.api import PricingTierInput

page = 1
migrated = skipped = failed = 0
while True:
    resp = with_retries(lambda: src.api.models.list(page=page, limit=100))
    if not resp.data:
        break
    for m in resp.data:
        if getattr(m, "is_langfuse_managed", False):
            skipped += 1
            continue
        try:
            # The server rejects requests that mix flat prices with
            # pricing_tiers. If the source uses pricing_tiers (e.g. token-
            # tiered models like gpt-image-1), forward them and omit the
            # flat prices; otherwise send only the flat prices.
            create_kwargs: Dict[str, Any] = {
                "model_name": m.model_name,
                "match_pattern": m.match_pattern,
                "start_date": m.start_date,
                "unit": m.unit,
                "tokenizer_id": m.tokenizer_id,
                "tokenizer_config": m.tokenizer_config,
            }
            if getattr(m, "pricing_tiers", None):
                create_kwargs["pricing_tiers"] = [
                    PricingTierInput(
                        name=t.name, is_default=t.is_default, priority=t.priority,
                        conditions=t.conditions, prices=t.prices,
                    ) for t in m.pricing_tiers
                ]
            else:
                if m.input_price is not None:
                    create_kwargs["input_price"] = m.input_price
                if m.output_price is not None:
                    create_kwargs["output_price"] = m.output_price
                if m.total_price is not None:
                    create_kwargs["total_price"] = m.total_price
            with_retries(lambda: dst.api.models.create(**create_kwargs))
            migrated += 1
        except Exception as exc:
            if getattr(exc, "status_code", None) == 409 or "already exists" in str(exc).lower():
                skipped += 1
            else:
                print(f"  failed to migrate model '{m.model_name}': {exc}")
                failed += 1
    if page >= getattr(resp.meta, "total_pages", page):
        break
    page += 1

print(f"Custom models — migrated: {migrated}, skipped: {skipped}, failed: {failed}")

3. Prompts

Every version of every prompt is migrated. The destination assigns fresh version numbers; the original source version is recorded in commit_message so it stays traceable. Tags and labels are preserved as-is. The code records destination versions in prompt_version_map so migrated observations reference the correct prompt version.

from urllib.parse import quote

from langfuse.api import (
    CreateChatPromptRequest,
    CreateChatPromptType,
    CreateTextPromptRequest,
    CreateTextPromptType,
    Prompt_Chat,
    Prompt_Text,
)

prompt_version_map = {}

page = 1
prompt_versions = []  # (name, version) tuples
while True:
    resp = with_retries(lambda: src.api.prompts.list(page=page, limit=100))
    if not resp.data:
        break
    for meta in resp.data:
        for v in meta.versions:
            prompt_versions.append((meta.name, v))
    if page >= getattr(resp.meta, "total_pages", page):
        break
    page += 1

prompt_versions.sort(key=lambda x: (x[0], x[1]))
print(f"Found {len(prompt_versions)} prompt versions on source")

migrated = failed = 0
for name, version in prompt_versions:
    try:
        # Folder prompts (names containing '/') must have their path segments
        # URL-encoded on the GET endpoint — the Fern client does not do this
        # automatically. The request body on create() keeps the raw name.
        encoded_name = quote(name, safe="")
        p = with_retries(lambda: src.api.prompts.get(encoded_name, version=version))
        commit_msg = f"Migrated from source. Original version: {version}."
        if isinstance(p, Prompt_Chat):
            req = CreateChatPromptRequest(
                name=p.name,
                type=CreateChatPromptType.CHAT,
                prompt=p.prompt,
                labels=p.labels or [],
                tags=p.tags or [],
                config=p.config,
                commit_message=commit_msg,
            )
        elif isinstance(p, Prompt_Text):
            req = CreateTextPromptRequest(
                name=p.name,
                type=CreateTextPromptType.TEXT,
                prompt=p.prompt,
                labels=p.labels or [],
                tags=p.tags or [],
                config=p.config,
                commit_message=commit_msg,
            )
        else:
            print(f"  unknown prompt type for {name} v{version}: {type(p).__name__}")
            failed += 1
            continue
        created = with_retries(lambda: dst.api.prompts.create(request=req))
        prompt_version_map[(name, version)] = created.version
        migrated += 1
    except Exception as exc:
        print(f"  failed {name} v{version}: {exc}")
        failed += 1

print(f"Prompts — migrated: {migrated}, failed: {failed}")

4. Evaluators

The stable Evaluators and Evaluation Rules APIs used here are available on Langfuse Cloud and on self-hosted Langfuse v4.23.0 or newer. Both source and destination must support these endpoints.

We use GET /api/public/v2/evaluators with cursor pagination and POST /api/public/v2/evaluators to copy the latest definition of each LLM-as-a-Judge or code evaluator. These requests use HTTP Basic authentication with the same project keys configured in section 0. See the Evaluators API reference.

The destination assigns new evaluator IDs and starts each evaluator at version 1. The code keeps an evaluator_id_map for rule assignments. It copies prompts, output definitions, complete default variable mappings, explicit model choices, and code definitions; historical versions, original authors, timestamps, and paused states are not preserved.

Before running this cell, configure the destination's LLM connections. An evaluator with modelConfig: null uses the destination's default evaluation model. For self-hosted code evaluators, also configure the code evaluator executor. Check each copied evaluator's status in the destination before using it.

Successful creations are skipped if you rerun the cell in the same kernel. Save evaluator_id_map before restarting the notebook: names are not unique identifiers, and rerunning without the map creates duplicates. POST requests are deliberately not retried automatically; after an ambiguous timeout, check the destination before retrying.

# Reuse successful source-to-destination mappings within this notebook session.
evaluator_id_map = globals().get("evaluator_id_map", {})


def evaluation_request(client, method, path, **kwargs):
    def send():
        response = client.request(method, path, **kwargs)
        response.raise_for_status()
        return response.json()

    return with_retries(send) if method == "GET" else send()


def evaluator_create_body(evaluator):
    body = {
        "name": evaluator["name"],
        "description": evaluator.get("description"),
        "type": evaluator["type"],
    }
    if evaluator["type"] == "llm_as_judge":
        mappings = evaluator.get("variableMapping")
        if mappings is not None and any(not m.get("source") for m in mappings):
            raise ValueError("Complete the source evaluator's default variable mapping first")
        body.update(
            prompt=evaluator["prompt"],
            outputDefinition=evaluator["outputDefinition"],
            variableMapping=mappings,
            modelConfig=evaluator.get("modelConfig"),
        )
    elif evaluator["type"] == "code":
        body.update(
            sourceCode=evaluator["sourceCode"],
            sourceCodeLanguage=evaluator["sourceCodeLanguage"],
        )
    else:
        raise ValueError(f"Unsupported evaluator type: {evaluator['type']}")
    return body


migrated = skipped = failed = 0
with httpx.Client(
    base_url=SOURCE_CFG["base_url"],
    auth=(SOURCE_CFG["public_key"], SOURCE_CFG["secret_key"]),
    timeout=60,
) as source_api, httpx.Client(
    base_url=DEST_CFG["base_url"],
    auth=(DEST_CFG["public_key"], DEST_CFG["secret_key"]),
    timeout=60,
) as destination_api:
    cursor = None
    while True:
        params = {"limit": 100}
        if cursor:
            params["cursor"] = cursor
        page = evaluation_request(
            source_api, "GET", "/api/public/v2/evaluators", params=params
        )
        for evaluator in page["data"]:
            if evaluator["id"] in evaluator_id_map:
                skipped += 1
                continue
            try:
                # Only writable definition fields are sent, not IDs or rule assignments.
                created = evaluation_request(
                    destination_api, "POST", "/api/public/v2/evaluators",
                    json=evaluator_create_body(evaluator),
                )
                evaluator_id_map[evaluator["id"]] = created["id"]
                migrated += 1
                print(f"  {evaluator['name']}: {created['id']} (status={created['status']})")
            except Exception as exc:
                print(f"  failed evaluator {evaluator['id']} ({evaluator['name']}): {exc}")
                failed += 1
        cursor = page.get("meta", {}).get("cursor")
        if not cursor:
            break

print(f"Evaluators — migrated: {migrated}, skipped: {skipped}, failed: {failed}")
print(dumps(evaluator_id_map))

Recreate evaluation rules

Evaluators define how to score data; evaluation rules define which incoming observations to evaluate. Copying an evaluator does not copy or enable its rules.

To migrate rules through the Evaluation Rules API:

  1. Read the source rules with cursor-paginated GET /api/public/v2/evaluation-rules.
  2. Build a create body with name, sampling, filter, and evaluatorAssignments. Replace every assignment's evaluatorId with evaluator_id_map[source_id] and preserve its variableMapping override. Remap project-specific filter values, such as datasetId, to destination IDs; dataset names from section 7 can help match datasets. If an evaluator failed to migrate, resolve that failure before copying its rules.
  3. Create each rule with POST /api/public/v2/evaluation-rules, setting enabled: false. Review its filters, mappings, evaluator status, and sampling in the destination, then enable it with PATCH /api/public/v2/evaluation-rules/{evaluationRuleId} and {"enabled": true}.

Legacy trace and dataset rules may be returned by the read endpoint, but cannot be recreated unchanged. Convert them to observation rules following the evaluator upgrade guide. New rules evaluate incoming observations; copying them does not rerun evaluation on the historical data copied by this notebook.

5. Observations (via OTLP ingestion endpoint)

This is the core of the migration. We push each source trace (and its nested observations) into the destination's OTLP ingestion endpoint at /api/public/otel/v1/traces. The OTLP endpoint is the only way to set the authoritative start_time / end_time of an observation at ingest — the legacy REST ingestion rewrites those fields.

Self-hosted deployments: the v2 Observations API requires Langfuse v4, which is generally available for self-hosted deployments. If your source deployment still runs Langfuse v3, the v2 endpoints are unavailable there: enumerate traces with src.api.trace.list(page=..., from_timestamp=FROM_TS, to_timestamp=TO_TS) and fetch each trace's observations with src.api.legacy.observations_v1.get_many(trace_id=...). Both are deprecated on v4, but they are the only option against a v3 source.

Which traces are migrated: the listing below pages through logical root observations, and deduplicates trace IDs when a trace has multiple app roots. A trace whose root observation never reached Langfuse — for example when only child spans were exported — has no logical root and is skipped. Compare the migrated trace count against the source project's trace count; if they differ, page through observations without the root filter and group the rows by trace_id to pick up the remainder.

from contextvars import ContextVar

from opentelemetry import trace as otel_trace
from opentelemetry.context import Context
from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags

# Reuse migrated/restored prompt versions; keep source versions if prompts were skipped.
prompt_version_map = globals().get("prompt_version_map", {})

# Force the IDs actually exported to match the score/dataset reference maps.
_migration_trace_id = ContextVar("migration_trace_id", default=None)
_migration_span_id = ContextVar("migration_span_id", default=None)


class MigrationIdGenerator(RandomIdGenerator):
    def generate_trace_id(self):
        return _migration_trace_id.get() or super().generate_trace_id()

    def generate_span_id(self):
        return _migration_span_id.get() or super().generate_span_id()


class MigrationSpanExporter(OTLPSpanExporter):
    failed_span_ids = []

    def export(self, spans):
        result = SpanExportResult.FAILURE
        try:
            result = super().export(spans)
            return result
        finally:
            if result != SpanExportResult.SUCCESS:
                self.failed_span_ids.extend(format(s.context.span_id, "016x") for s in spans)


# --- Build a dedicated OTLP pipeline that points at the destination project ---
_auth = base64.b64encode(
    f"{DEST_CFG['public_key']}:{DEST_CFG['secret_key']}".encode("utf-8")
).decode("ascii")

otlp_provider = TracerProvider(
    resource=Resource.create({"service.name": "langfuse-migration"}),
    id_generator=MigrationIdGenerator(),
)
otlp_provider.add_span_processor(
    SimpleSpanProcessor(
        MigrationSpanExporter(
            endpoint=f"{DEST_CFG['base_url']}/api/public/otel/v1/traces",
            headers={
                "Authorization": f"Basic {_auth}",
                "x-langfuse-ingestion-version": "4",
            },
        )
    )
)
migration_tracer = otlp_provider.get_tracer("langfuse-migration")


def _to_ns(d: Optional[dt.datetime]) -> Optional[int]:
    if d is None:
        return None
    if d.tzinfo is None:
        d = d.replace(tzinfo=dt.timezone.utc)
    return int(d.timestamp() * 1_000_000_000)


def _set_scalar(span, key: str, value: Any) -> None:
    """Set an OTel attribute on `span` stringifying non-primitive values via UTF-8 JSON."""
    if value is None:
        return
    if isinstance(value, (str, bool, int, float)):
        span.set_attribute(key, value)
        return
    span.set_attribute(key, dumps(value))


def _as_attr_string(value: Any) -> Optional[str]:
    """Coerce an input/output/metadata value to a JSON attribute string.

    The Observations API v2 returns input/output as raw JSON strings,
    while older trace responses may return parsed dicts/lists. Pass strings
    through unchanged so we don't end up double-encoded; JSON-encode
    everything else with ensure_ascii=False (Japanese-safe).
    """
    if value is None:
        return None
    if isinstance(value, str):
        return value
    return dumps(value)


def _set_trace_attrs(span, trace_obj) -> None:
    """Set the context associated with this observation."""
    trace_name = getattr(trace_obj, "trace_name", None) or trace_obj.name
    _set_scalar(span, "langfuse.trace.name", trace_name)
    _set_scalar(span, "user.id", trace_obj.user_id)
    _set_scalar(span, "session.id", trace_obj.session_id)
    _set_scalar(span, "langfuse.release", getattr(trace_obj, "release", None))
    _set_scalar(span, "langfuse.version", trace_obj.version)
    _set_scalar(span, "langfuse.environment", trace_obj.environment or "default")
    if trace_obj.public is not None:
        span.set_attribute("langfuse.trace.public", bool(trace_obj.public))
    if getattr(trace_obj, "tags", None):
        # Langfuse accepts both a string[] attribute and a JSON string.
        span.set_attribute("langfuse.trace.tags", list(trace_obj.tags))


_OBS_TYPE_MAP = {
    kind: kind.lower() for kind in (
        "SPAN", "GENERATION", "EVENT", "AGENT", "TOOL", "CHAIN", "RETRIEVER",
        "EVALUATOR", "EMBEDDING", "GUARDRAIL",
    )
}


def _set_observation_attrs(span, obs) -> None:
    obs_type = _OBS_TYPE_MAP.get(obs.type, "span")
    span.set_attribute("langfuse.observation.type", obs_type)
    _set_scalar(span, "langfuse.observation.level", obs.level)
    _set_scalar(span, "langfuse.observation.status_message", obs.status_message)
    _set_scalar(span, "langfuse.version", obs.version)
    _set_scalar(span, "langfuse.environment", obs.environment or "default")
    input_str = _as_attr_string(obs.input)
    if input_str is not None:
        span.set_attribute("langfuse.observation.input", input_str)
    output_str = _as_attr_string(obs.output)
    if output_str is not None:
        span.set_attribute("langfuse.observation.output", output_str)
    md = obs.metadata
    if isinstance(md, dict):
        for k, v in md.items():
            _set_scalar(span, f"langfuse.observation.metadata.{k}", v)
    elif md is not None:
        _set_scalar(span, "langfuse.observation.metadata._raw", md)
    # Generation-only attributes. ObservationV2 exposes the model name as
    # `model`; retain a fallback for older SDK response types.
    if obs_type in ("generation", "embedding"):
        model_name = getattr(obs, "model", None) or getattr(obs, "provided_model_name", None)
        _set_scalar(span, "langfuse.observation.model.name", model_name)
        if obs.model_parameters:
            span.set_attribute("langfuse.observation.model.parameters", dumps(obs.model_parameters))
        if obs.usage_details:
            span.set_attribute("langfuse.observation.usage_details", dumps(obs.usage_details))
        if obs.cost_details:
            span.set_attribute("langfuse.observation.cost_details", dumps(obs.cost_details))
        if obs.completion_start_time is not None:
            d = obs.completion_start_time
            if d.tzinfo is None:
                d = d.replace(tzinfo=dt.timezone.utc)
            span.set_attribute(
                "langfuse.observation.completion_start_time",
                d.isoformat().replace("+00:00", "Z"),
            )
        if getattr(obs, "prompt_name", None):
            _set_scalar(span, "langfuse.observation.prompt.name", obs.prompt_name)
        if getattr(obs, "prompt_version", None) is not None:
            destination_version = prompt_version_map.get(
                (obs.prompt_name, obs.prompt_version), obs.prompt_version
            )
            span.set_attribute("langfuse.observation.prompt.version", int(destination_version))


def _parent_ctx(trace_id_hex: str, parent_span_id_hex: Optional[str]):
    if not parent_span_id_hex:
        return Context()
    parent_sc = SpanContext(
        trace_id=int(trace_id_hex, 16),
        span_id=int(parent_span_id_hex, 16),
        is_remote=True,
        trace_flags=TraceFlags(0x01),
    )
    return otel_trace.set_span_in_context(NonRecordingSpan(parent_sc))


trace_id_map: Dict[str, str] = {}
obs_id_map: Dict[str, str] = {}

# Field groups we need from the v2 Observations API. `core` is always included.
# `basic` gives name/level/status/version/environment; `time` adds
# completionStartTime; `io` adds input/output; `metadata` adds metadata;
# `model` adds model/modelParameters; `usage` adds
# usageDetails/costDetails; `prompt` adds promptName/promptVersion.
OBSERVATIONS_FIELDS = "core,basic,time,io,metadata,model,usage,prompt,trace_context"


def _fetch_observations_v2(source_trace_id: str) -> List[Any]:
    """Fetch all observations for a trace via the v2 API with cursor pagination.

    Requires the v2 Observations API (Langfuse Cloud, or self-hosted Langfuse v4+).
    On a Langfuse v3 source, swap the call for the deprecated v1 read
    `src.api.legacy.observations_v1.get_many(trace_id=source_trace_id, page=..., limit=100)`,
    which pages by number instead of by cursor.
    """
    results: List[Any] = []
    cursor: Optional[str] = None
    while True:
        resp = with_retries(lambda: src.api.observations.get_many(
            trace_id=source_trace_id,
            fields=OBSERVATIONS_FIELDS,
            limit=1000,
            cursor=cursor,
        ))
        metadata_keys = sorted({key for o in resp.data for key in (o.metadata or {})})
        if metadata_keys:
            # The default API response truncates metadata values at 200 characters.
            resp = with_retries(lambda: src.api.observations.get_many(
                trace_id=source_trace_id, fields=OBSERVATIONS_FIELDS,
                limit=1000, cursor=cursor, expand_metadata=",".join(metadata_keys),
            ))
        if resp.data:
            results.extend(resp.data)
        cursor = getattr(resp.meta, "cursor", None)
        if not cursor:
            break
    return results


def migrate_one_trace(trace_obj) -> int:
    """Copy the source observations using deterministic trace and span IDs."""
    source_trace_id = getattr(trace_obj, "trace_id", None) or trace_obj.id
    observations = _fetch_observations_v2(source_trace_id)
    new_trace_hex = to_otel_trace_id(source_trace_id)
    # Precompute every ID so equal timestamps or child-first API results cannot
    # break parent links. A parent's span need not be exported before its child.
    local_ids = {o.id: to_otel_span_id(o.id) for o in observations}
    for obs in observations:
        parent_hex = to_otel_span_id(obs.parent_observation_id) if obs.parent_observation_id else None
        trace_token = _migration_trace_id.set(int(new_trace_hex, 16))
        span_token = _migration_span_id.set(int(local_ids[obs.id], 16))
        try:
            span = migration_tracer.start_span(
                name=obs.name or obs.type.lower(),
                start_time=_to_ns(obs.start_time),
                context=_parent_ctx(new_trace_hex, parent_hex),
            )
        finally:
            _migration_span_id.reset(span_token)
            _migration_trace_id.reset(trace_token)
        try:
            _set_trace_attrs(span, obs)
            _set_observation_attrs(span, obs)
            if getattr(obs, "is_root_observation", False):
                span.set_attribute("langfuse.internal.is_app_root", True)
        finally:
            span.end(end_time=_to_ns(obs.end_time) or _to_ns(obs.start_time))
    trace_id_map[source_trace_id] = new_trace_hex
    obs_id_map.update(local_ids)
    return len(observations)

# --- Drive the migration ---
cursor: Optional[str] = None
limit = 50
total = 0
total_obs = 0
failures = 0
progress_every = 10  # print a progress line every N traces
t_start = time.time()
while True:
    # Use the first-class logical-root query parameter. Pass it via request_options
    # because SDK 4.8.1 does not expose the is_root_observation keyword yet.
    listing = with_retries(lambda: src.api.observations.get_many(
        fields="core,basic,io,metadata,trace_context",
        limit=limit,
        cursor=cursor,
        from_start_time=FROM_TS,
        to_start_time=TO_TS,
        request_options={"additional_query_parameters": {"isRootObservation": True}},
    ))
    cursor = getattr(listing.meta, "cursor", None)
    for t in listing.data:
        source_trace_id = getattr(t, "trace_id", None) or t.id
        if source_trace_id in trace_id_map:
            continue
        try:
            obs_count = migrate_one_trace(t)
            total += 1
            total_obs += obs_count
            if total % progress_every == 0:
                elapsed = time.time() - t_start
                rate = total / elapsed if elapsed > 0 else 0
                print(
                    f"  [{total} traces] observations={total_obs} "
                    f"failed={failures} rate={rate:.1f} traces/s"
                )
        except Exception as exc:
            source_trace_id = getattr(t, "trace_id", t.id)
            print(f"  failed trace {source_trace_id}: {exc}")
            failures += 1

    print(
        f"  batch complete — cumulative: traces={total}, observations={total_obs}, failed={failures}"
    )
    if not cursor:
        break

otlp_provider.force_flush()
assert not MigrationSpanExporter.failed_span_ids, (
    f"OTLP export failed for {len(MigrationSpanExporter.failed_span_ids)} observation(s). "
    "Resolve the ingestion error and rerun the observations cell before copying scores or datasets."
)
elapsed = time.time() - t_start
print(
    f"\nObservations — migrated: {total} traces / {total_obs} observations, "
    f"failed: {failures}, elapsed: {elapsed:.1f}s"
)
print(f"  trace_id_map entries: {len(trace_id_map)}")
print(f"  obs_id_map entries:   {len(obs_id_map)}")

6. Scores

Scores are fetched from the source via the v3 Scores API (api.scores_v3.get_many_v3) and re-created on the destination through the POST /api/public/scores endpoint (api.legacy.score_v1.create). Original id is preserved so re-runs are idempotent. config_id is remapped through config_id_map from Section 1; trace / observation references are remapped through trace_id_map / obs_id_map from Section 5.

Compared to the older v2 endpoint, the v3 API differs in three ways that matter here:

  • Cursor pagination — pass the opaque meta.cursor back until it is empty, instead of counting pages.
  • Single subject — each score references its target through one subject object (kind + id) instead of flat trace_id / observation_id / session_id / dataset_run_id fields. Dataset runs are called experiments in v3. Rare legacy scores carrying two subject links (e.g. both a trace and a session) surface only one of them in v3 — the previous flow failed to re-create those scores entirely, so nothing regresses.
  • Typed value — value follows the score's data_type (number for NUMERIC, boolean for BOOLEAN, string for CATEGORICAL and other types) instead of always being numeric alongside a separate string_value.

A few behavioral notes:

  • v2 silently dropped scores whose trace no longer exists, while v3 returns them. If slightly more scores migrate than the v2 flow used to transfer, these previously hidden scores are now carried over — an improvement, not a loss.
  • Scores attached to dataset runs (kind="experiment") are skipped by this section: historical experiment runs are not copied, so the source run ID would be a dangling reference. Re-run experiments on the destination to produce new results.
  • Score creation is asynchronous: a success response means the score was accepted for ingestion, not that it is already durably written. Verify the destination project once ingestion has caught up.
  • The create endpoint does not accept a timestamp, so migrated scores carry the migration time as their timestamp; source score timestamps cannot be preserved through the public API.

Self-hosted deployments: the v3 Scores API is available on Langfuse Cloud and on self-hosted instances from v3.180.0. On older self-hosted instances, use the v2 pattern instead: page-based src.api.scores.get_many(page=page, limit=100, ...) with flat trace_id / observation_id / session_id fields and string_value for CATEGORICAL scores. The v2 read endpoints are removed on deployments fully migrated to Langfuse v4.

trace_id_map = globals().get("trace_id_map", {})
obs_id_map = globals().get("obs_id_map", {})
config_id_map = globals().get("config_id_map", {})

# Field groups for the v3 Scores API. `core` (id/name/value/dataType/...) is
# always included; `details` adds comment/metadata/configId; `subject` adds
# the subject reference; `annotation` adds authorUserId/queueId.
SCORES_FIELDS = "details,subject,annotation"

cursor: Optional[str] = None
migrated = failed = skipped = 0
while True:
    resp = with_retries(lambda: src.api.scores_v3.get_many_v3(
        limit=100,
        cursor=cursor,
        fields=SCORES_FIELDS,
        from_timestamp=FROM_TS,
        to_timestamp=TO_TS,
    ))
    for s in resp.data:
        # A null value must be caught before any conversion — the BOOLEAN
        # branch below would otherwise turn a missing value into false.
        if s.value is None:
            skipped += 1
            continue

        # Map the v3 subject back to the flat reference fields the create
        # endpoint expects: exactly one of traceId / sessionId, and
        # observationId additionally requires traceId.
        trace_id = observation_id = session_id = None
        kind = s.subject.kind if s.subject else None
        if kind == "trace":
            trace_id = s.subject.id
        elif kind == "observation":
            if not s.subject.trace_id:
                print(f"  skipping score {s.id} ({s.name}): observation subject without a trace")
                skipped += 1
                continue
            observation_id = s.subject.id
            trace_id = s.subject.trace_id
        elif kind == "session":
            session_id = s.subject.id
        elif kind == "experiment":  # v3 name for dataset runs
            # The subject id is the source project's run id; the destination
            # assigns new run ids, so this reference cannot be carried over.
            print(f"  skipping score {s.id} ({s.name}): historical experiment runs are not copied; rerun the experiment on the destination")
            skipped += 1
            continue
        else:
            print(f"  skipping score {s.id} ({s.name}): unsupported subject kind {kind!r}")
            skipped += 1
            continue

        # Remap references
        new_trace_id = trace_id_map.get(trace_id, trace_id) if trace_id else None
        new_obs_id = obs_id_map.get(observation_id, observation_id) if observation_id else None
        new_cfg_id = config_id_map.get(s.config_id, s.config_id) if s.config_id else None

        # v3 serves a typed `value`; convert it to what the write path expects.
        if s.data_type == "NUMERIC":
            value = s.value
        elif s.data_type == "BOOLEAN":
            value = 1 if s.value else 0  # write path expects numeric 0/1
        elif s.data_type == "CATEGORICAL":
            value = s.value  # already the category string
        else:
            # Other data types (e.g. TEXT) carry string values; pass them through.
            print(f"  passing through {s.data_type} score {s.id} ({s.name}) as-is")
            value = s.value

        try:
            # POST /api/public/scores. The endpoint accepts no timestamp, so
            # the migrated score is stamped with the migration time. Ingestion
            # is asynchronous: success means queued, not yet visible.
            with_retries(lambda: dst.api.legacy.score_v1.create(
                id=s.id,  # preserves ID for idempotent re-runs
                name=s.name,
                value=value,
                data_type=s.data_type,
                trace_id=new_trace_id,
                observation_id=new_obs_id,
                session_id=session_id,
                comment=s.comment,
                config_id=new_cfg_id,
                metadata=s.metadata,
                environment=s.environment,
            ))
            migrated += 1
        except Exception as exc:
            print(f"  failed score {s.id} ({s.name}): {exc}")
            failed += 1
    cursor = getattr(resp.meta, "cursor", None)
    if not cursor:
        break

print(f"Scores — migrated: {migrated}, skipped: {skipped}, failed: {failed}")

7. Datasets and items

We migrate datasets themselves via the api.datasets.create endpoint (including input and expected-output schemas) and items via create_dataset_item (preserving source item IDs).

Historical dataset run items are not copied. The legacy POST /api/public/dataset-run-items endpoint is deprecated, and both of its replacements create new experiment runs rather than backfilling past ones. To reproduce a run on the destination, either re-run the experiment with the Experiments SDK, or emit the item traces yourself with experiment attributes over OTLP.

# Fallback to empty maps when this cell is run standalone (see Scores section).
trace_id_map = globals().get("trace_id_map", {})
obs_id_map = globals().get("obs_id_map", {})

from urllib.parse import quote as _urlquote


def _enc(name: str) -> str:
    return _urlquote(name, safe="")


ds_migrated = ds_skipped = ds_failed = 0
items_migrated = items_failed = 0

t_start = time.time()
ds_seen = 0  # count of source datasets processed (new + existing)

page_ds = 1
while True:
    dsresp = with_retries(lambda: src.api.datasets.list(page=page_ds, limit=100))
    if not dsresp.data:
        break

    total_ds = getattr(dsresp.meta, "total_items", None)
    total_pages = getattr(dsresp.meta, "total_pages", None)
    if page_ds == 1 and total_ds is not None:
        print(f"Starting dataset migration: {total_ds} dataset(s) across {total_pages} page(s)")

    for ds in dsresp.data:
        ds_seen += 1
        progress = f"{ds_seen}/{total_ds}" if total_ds is not None else str(ds_seen)
        print(f"  [{progress}] '{ds.name}'")

        # Create dataset if not present. Path parameters must be URL-encoded
        # for folder datasets (names containing "/").
        try:
            dst.api.datasets.get(dataset_name=_enc(ds.name))
            ds_skipped += 1
            print(f"    dataset already exists in destination — skipping create")
        except Exception as exc:
            if getattr(exc, "status_code", None) != 404:
                print(f"    failed dataset lookup: {exc}")
                ds_failed += 1
                continue
            try:
                with_retries(lambda: dst.api.datasets.create(
                    name=ds.name, description=ds.description, metadata=ds.metadata,
                    input_schema=ds.input_schema, expected_output_schema=ds.expected_output_schema,
                ))
                ds_migrated += 1
                print(f"    created dataset in destination")
            except Exception as exc2:
                print(f"    failed to create dataset: {exc2}")
                ds_failed += 1
                continue

        # Items (preserve source IDs for repeatable migrations)
        ds_items_before = items_migrated
        ds_items_failed_before = items_failed
        page_item = 1
        while True:
            iresp = with_retries(lambda: src.api.dataset_items.list(
                dataset_name=ds.name, page=page_item, limit=100
            ))
            if not iresp.data:
                break
            for it in iresp.data:
                try:
                    with_retries(lambda: dst.create_dataset_item(
                        dataset_name=ds.name,
                        id=it.id,
                        input=it.input,
                        expected_output=it.expected_output,
                        metadata=it.metadata,
                        source_trace_id=trace_id_map.get(it.source_trace_id, it.source_trace_id) if it.source_trace_id else None,
                        source_observation_id=obs_id_map.get(it.source_observation_id, it.source_observation_id) if it.source_observation_id else None,
                        status=it.status,
                    ))
                    items_migrated += 1
                except Exception as exc:
                    print(f"      failed item {it.id}: {exc}")
                    items_failed += 1
            if page_item >= getattr(iresp.meta, "total_pages", page_item):
                break
            page_item += 1
        ds_items = items_migrated - ds_items_before
        ds_items_fail = items_failed - ds_items_failed_before
        print(f"    items: {ds_items} migrated, {ds_items_fail} failed")

    if page_ds >= getattr(dsresp.meta, "total_pages", page_ds):
        break
    page_ds += 1

elapsed = time.time() - t_start
print(
    f"\nDatasets — new: {ds_migrated}, existing: {ds_skipped}, failed: {ds_failed}  |  "
    f"items: {items_migrated} migrated / {items_failed} failed  |  "
    f"elapsed: {elapsed:.1f}s"
)

print("Historical experiment runs are not copied; rerun experiments on the destination.")

Was this page helpful?

Last updated on