基础模型:

• ds4sd/SmolDocling-256M-preview 语言:
• en 库名称: transformers 许可证: cdla-permissive-2.0 管道标签: image-text-to-text 标签:
• mlx

SmolDocling-256M-preview-mlx-bf16

此模型使用mlx-vlm版本0.1.18从ds4sd/SmolDocling-256M-preview转换为MLX格式。
包含适用于Docling-Snap的配置文件。
更多模型详情请参考原始模型卡片。

使用MLX运行

pip install -U mlx-vlm pillow docling-core

# /// 脚本
# 需要python版本 = ">=3.12"
# 依赖 = [
#     "docling-core",
#     "mlx-vlm",
#     "pillow",
# ]
# ///
from io import BytesIO
from pathlib import Path
from urllib.parse import urlparse

import requests
from PIL import Image
from docling_core.types.doc import ImageRefMode
from docling_core.types.doc.document import DocTagsDocument, DoclingDocument
from mlx_vlm import load, generate
from mlx_vlm.prompt_utils import apply_chat_template
from mlx_vlm.utils import load_config, stream_generate

## 设置
SHOW_IN_BROWSER = True  # 将输出导出为HTML并在浏览器中打开

## 加载模型
model_path = "ds4sd/SmolDocling-256M-preview-mlx-bf16"
model, processor = load(model_path)
config = load_config(model_path)

## 准备输入
prompt = "将此页面转换为docling格式。"

# image = "https://ibm.biz/docling-page-with-list"
image = "https://ibm.biz/docling-page-with-table"

# 加载图像资源
if urlparse(image).scheme != "":  # 如果是URL
    response = requests.get(image, stream=True, timeout=10)
    response.raise_for_status()
    pil_image = Image.open(BytesIO(response.content))
else:
    pil_image = Image.open(image)

# 应用聊天模板
formatted_prompt = apply_chat_template(processor, config, prompt, num_images=1)

## 生成输出
print("文档标签: \n\n")

output = ""
for token in stream_generate(
    model, processor, formatted_prompt, [image], max_tokens=4096, verbose=False
):
    output += token.text
    print(token.text, end="")
    if "</doctag>" in token.text:
        break

print("\n\n")

# 填充文档
doctags_doc = DocTagsDocument.from_doctags_and_image_pairs([output], [pil_image])
# 创建docling文档
doc = DoclingDocument(name="示例文档")
doc.load_from_doctags(doctags_doc)

## 导出为任意格式
# Markdown
print("Markdown格式: \n\n")
print(doc.export_to_markdown())

# HTML
if SHOW_IN_BROWSER:
    import webbrowser

    out_path = Path("./output.html")
    doc.save_as_html(out_path, image_mode=ImageRefMode.EMBEDDED)
    webbrowser.open(f"file:///{str(out_path.resolve())}")