litellm/litellm/litellm_core_utils/default_encoding.py

import os

import litellm

try:
    # New and recommended way to access resources
    from importlib import resources

    filename = str(resources.files(litellm).joinpath("litellm_core_utils/tokenizers"))
except (ImportError, AttributeError):
    # Old way to access resources, which setuptools deprecated some time ago
    import pkg_resources  # type: ignore

    filename = pkg_resources.resource_filename(
        __name__, "litellm_core_utils/tokenizers"
    )

# Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
# unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
# This keeps tiktoken fully offline-capable by default (see #1071).
custom_cache_dir = os.getenv("CUSTOM_TIKTOKEN_CACHE_DIR")
if custom_cache_dir:
    # If the user opts into a custom cache dir, ensure it exists.
    os.makedirs(custom_cache_dir, exist_ok=True)
    cache_dir = custom_cache_dir
else:
    cache_dir = filename

os.environ["TIKTOKEN_CACHE_DIR"] = (
    cache_dir  # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
)

import tiktoken
import time
import random

# Retry logic to handle race conditions when multiple processes try to create
# the tiktoken cache file simultaneously (common in parallel test execution on Windows)
_max_retries = 5
_retry_delay = 0.1  # Start with 100ms

for attempt in range(_max_retries):
    try:
        encoding = tiktoken.get_encoding("cl100k_base")
        break
    except (FileExistsError, OSError):
        if attempt == _max_retries - 1:
            # Last attempt, re-raise the exception
            raise
        # Exponential backoff with jitter to reduce collision probability
        delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
        time.sleep(delay)