Example: Multi-modality and attachments

%pip install langfuse langchain langchain_openai

import os
from urllib.request import urlretrieve
from urllib.error import URLError
 
REPO_URL = "https://github.com/langfuse/langfuse-python"
download_path = "static"
os.makedirs(download_path, exist_ok=True)
 
test_files = ["puton.jpg", "joke_prompt.wav", "bitcoin.pdf"]
raw_url = f"{REPO_URL}/raw/main/{download_path}"
 
for file in test_files:
   try:
       urlretrieve(f"{raw_url}/{file}", f"{download_path}/{file}")
       print(f"Successfully downloaded: {file}")
   except URLError as e:
       print(f"Failed to download {file}: {e}")
   except OSError as e:
       print(f"Failed to save {file}: {e}")

Successfully downloaded: puton.jpg
Successfully downloaded: joke_prompt.wav
Successfully downloaded: bitcoin.pdf

import os
 
# Get keys for your project from the project settings page: https://cloud.langfuse.com
os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..." 
os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..." 
os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # 🇪🇺 EU region
# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # 🇺🇸 US region
 
# Your openai key
os.environ["OPENAI_API_KEY"] = "sk-proj-..."

from langfuse.openai import openai
from langfuse import get_client
import base64
 
client = openai.OpenAI()
 
def encode_file(image_path):
    with open(image_path, "rb") as file:
        return base64.b64encode(file.read()).decode("utf-8")

from langfuse import get_client
 
content_path = "static/puton.jpg"
content_type = "image/jpeg"
 
base64_image = encode_file(content_path)
 
response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What’s in this image?"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:{content_type};base64,{base64_image}"
                    },
                },
            ],
        }
    ],
    max_tokens=300,
)
 
print(response.__dict__)
 
# Flush the trace
langfuse = get_client()
langfuse.flush()

{'id': 'chatcmpl-Bhf794La4LhadJktsGaroFwbg2BIL', 'choices': [Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content="The image features a dog sitting on a person's lap with its front paws resting on their knee. The dog has a curly coat with black and white fur and appears to be happy, with its tongue out. In the background, there are people standing, likely engaged in conversation. The setting seems to be a cozy indoor space with wooden flooring and a colorful rug.", refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=None))], 'created': 1749745847, 'model': 'gpt-4o-mini-2024-07-18', 'object': 'chat.completion', 'service_tier': 'default', 'system_fingerprint': 'fp_62a23a81ef', 'usage': CompletionUsage(completion_tokens=72, prompt_tokens=25514, total_tokens=25586, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0)), '_request_id': 'req_da2df5cf5f1964746a107af72fb2daee'}

from langfuse import get_client
 
content_path = "static/joke_prompt.wav"
 
base64_string = encode_file(content_path)
 
response = client.chat.completions.create(
    model="gpt-4o-audio-preview",
    modalities=["text", "audio"],
    audio={"voice": "alloy", "format": "wav"},
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "Do what this recording says."},
                {
                    "type": "input_audio",
                    "input_audio": {"data": base64_string, "format": "wav"},
                },
            ],
        },
    ],
)
 
print(response.__dict__)
 
# Flush the trace
langfuse = get_client()
langfuse.flush()

{'id': 'chatcmpl-Bhf92tYBL9Swp2MwBkA7bCQPVe9Vh', 'choices': [Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=None, refusal=None, role='assistant', annotations=[], audio=ChatCompletionAudio(id='audio_684b01341fd081918a825276eb36472b', data=<langfuse.media.LangfuseMedia object at 0x10d82a9c0>, expires_at=1749749572, transcript='Why did the Berlin Bear get lost in the city? Because he couldn\'t decide whether to take the U-Bahn, the S-Bahn, or just "bear"ly walk anywhere!'), function_call=None, tool_calls=None))], 'created': 1749745964, 'model': 'gpt-4o-audio-preview-2024-12-17', 'object': 'chat.completion', 'service_tier': 'default', 'system_fingerprint': 'fp_bf8dbd2ceb', 'usage': CompletionUsage(completion_tokens=245, prompt_tokens=66, total_tokens=311, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=0, audio_tokens=194, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=51), prompt_tokens_details=PromptTokensDetails(audio_tokens=49, cached_tokens=0, text_tokens=17, image_tokens=0)), '_request_id': 'req_ead21d343638b42eefb42e80e1621c63'}

from langfuse import observe, get_client
from langfuse.media import LangfuseMedia
 
with open("static/bitcoin.pdf", "rb") as pdf_file:
        pdf_bytes = pdf_file.read()
 
wrapped_obj = LangfuseMedia(
    obj=pdf_bytes, content_bytes=pdf_bytes, content_type="application/pdf"
)
 
@observe()
def main():
    langfuse.update_current_trace(
        metadata={
            "context": wrapped_obj
        },
    )
 
    return # Limitation: LangfuseMedia object does not work in decorated function IO
 
main()
 
# Flush the trace
langfuse = get_client()
langfuse.flush()

from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage
from langfuse.langchain import CallbackHandler
from langfuse import get_client
 
# Initialize Langfuse CallbackHandler for Langchain (tracing)
handler = CallbackHandler()
 
model = ChatOpenAI(model="gpt-4o-mini")
image_data = encode_file("static/puton.jpg")
 
message = HumanMessage(
    content=[
        {"type": "text", "text": "What's in this image?"},
        {
            "type": "image_url",
            "image_url": {"url": f"data:image/jpeg;base64,{image_data}"},
        },
    ],
)
 
response = model.invoke([message], config={"callbacks": [handler]})
 
print(response.content)
 
# Flush the trace
langfuse = get_client()
langfuse.flush()

The image features a dog with curly fur, sitting with its front paws resting on a person's knee. The dog appears to be friendly and is sticking out its tongue. In the background, there are a few people and some indoor furniture, suggesting a home environment. The floor has a colorful rug, and there are items like a basket and a leash visible.

import os
import requests
import base64
import hashlib
import uuid
 
base_URL = os.getenv("LANGFUSE_HOST")
public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
secret_key = os.getenv("LANGFUSE_SECRET_KEY")
 
file_path = "static/puton.jpg"
 
with open(file_path, "rb") as f:
    content_bytes = f.read()
 
content_type = "image/jpeg"
content_sha256 = base64.b64encode(hashlib.sha256(content_bytes).digest()).decode()
trace_id = str(uuid.uuid4())
content_length = len(content_bytes)
field = "input"  # or "output" or "metadata"
 
create_upload_url_body = {
    "traceId": trace_id,
    "contentType": content_type,
    "contentLength": content_length,
    "sha256Hash": content_sha256,
    "field": field,
}
 
create_upload_url_body

{'traceId': '6f330ea4-0d96-4dfe-b4b4-d63daef4b240',
 'contentType': 'image/jpeg',
 'contentLength': 650780,
 'sha256Hash': 'i5BuV2qX9nPaAAPf7c0gCYPLPU2GS3VUFKctrbzTKu4=',
 'field': 'input'}

upload_url_request = requests.post(
    f"{base_URL}/api/public/media",
    auth=(public_key or "", secret_key or ""),
    headers={"Content-Type": "application/json"},
    json=create_upload_url_body,
)
 
upload_url_response = upload_url_request.json()
upload_url_response

{'mediaId': 'a78bf29d-e1ac-496e-8bb3-94cda265a2d5', 'uploadUrl': None}

# If there is no uploadUrl, file was already uploaded
if (
    upload_url_response["mediaId"] is not None
    and upload_url_response["uploadUrl"] is not None
):
    upload_response = requests.put(
        upload_url_response["uploadUrl"],
        headers={
            "Content-Type": content_type,
            "x-amz-checksum-sha256": content_sha256,
        },
        data=content_bytes,
    )
 
    print("File uploaded")

from datetime import datetime, timezone
 
if upload_response is not None:
    requests.patch(
        f"{base_URL}/api/public/media/{upload_url_response['mediaId']}",
        auth=(public_key or "", secret_key or ""),
        headers={"Content-Type": "application/json"},
        json={
            "uploadedAt": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), # ISO 8601
            "uploadHttpStatus": upload_response.status_code,
            "uploadHttpError": upload_response.text if upload_response.status_code != 200 else None,
        },
    )
 
    print("Upload status updated")

media_request = requests.get(
    f"{base_URL}/api/public/media/{upload_url_response['mediaId']}",
    auth=(public_key or "", secret_key or "")
)
 
media_response = media_request.json()
media_response
 

{'mediaId': 'a78bf29d-e1ac-496e-8bb3-94cda265a2d5',
 'contentType': 'image/jpeg',
 'contentLength': 650780,
 'url': 'https://langfuse-prod-eu-media.s3.eu-west-1.amazonaws.com/cloramnkj0002jz088vzn1ja4/a78bf29d-e1ac-496e-8bb3-94cda265a2d5.jpeg?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=ASIAXEFUNOYRIGEVFBHC%2F20250612%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20250612T163759Z&X-Amz-Expires=3600&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEBgaCWV1LXdlc3QtMSJIMEYCIQD7Bye8IP4T7lt9UOH1a8wi8U3aQQPBulSl0Crh2LJW8AIhALyDgSbqWFUYR5RDB7B4rzcNipoGo%2BnZYftAjBnKmJyxKv4DCPH%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEQABoMNDkwMDA0NjQxMzE0IgxjkSsssdQEnoxRrsQq0gMxV0ZdTUay4A1eOUes90KuMGQSn69pLzvcJYrlSGpXkQ53xt0bxYPq9Gnq1KMuAHIev9EZNaypWRfnGiPq%2BDaD11K0f8U%2BybkSidYIpUczc1jpexwGXCQtT9XrIVn%2BsnN3spstFPaBsLavfQcXZMHq3yWjbolV8fEDBfhTYuBcFHtA4ELvGGSCmgWJY0UXY3078NTGzRXu3xzDMQrlClkbjuxOC75hGEIG9vmnUI%2BcG1L5Azl%2Bg47x5RpV5Nq8v0ilvYp%2B%2FkAC25OFVnMPMfaP6a2afY8UNdJGEqtFTlJVKmWT0nVsgAz6zAKw8aX4%2FGF8%2FjhSRRqPs%2BpdWwvtbM68deHXbNNudhg2joyUwgg1lZ90T%2BWHIRgH2KniyuOCwxhVuIanxrb1CA7cgptP%2BLHbYlszFBOF96DiRewJPreimyCaOX04A14puVfneD73cD16HKG3SQUPksujL9ySw4M3d54hLiSEqYhOQEC0ZDbfe121cR6yaGpqBtiE0bduKWfr33gBdzuBJQAA8MfPQw00J90CCvowu0B103mD9HVVrBL%2B%2BNby%2FYEikMBBUgbSmrLJNUdWQuxewinHy7qgdKvnB3MM0b%2FoqVgaJ0fw%2Fyd9NsZRMPDyq8IGOqQBH7QoPjK0NHpgkP7RKAFwoUzsYnjM1LRWzZDRxUy7YYGPyeesdc%2F8jy9cdErvfe%2BNiaGnixd707uXxfbRnYWEPuwV2PvimO%2FnlKxsIRmW27mmYeo2FTo4QC%2BBa%2F1zNfCf6G%2FgDKkw8hF2YsNoACWHTBNeUcc2PZ%2FHYTq6eyQkBAj9FHCnmEmSiDk6NVKanuE2EOGpoxEAWuDxHqRB73LatwU9OOQ%3D&X-Amz-Signature=8c1d8c4a721f2eb882405cdd47e2514b55eac1efd634791b8ea963a07a1bca01&X-Amz-SignedHeaders=host&x-id=GetObject',
 'urlExpiry': '2025-06-12T17:37:59.356Z',
 'uploadedAt': '2024-11-14T10:44:32.535Z'}