24 lines
816 B
Python
24 lines
816 B
Python
from langchain.text_splitter import CharacterTextSplitter
|
|
from langchain_community.document_loaders import TextLoader
|
|
|
|
# 1. 文档加载
|
|
loader = TextLoader("../../data/C2/txt/蜂医.txt", encoding="utf-8")
|
|
docs = loader.load()
|
|
|
|
# 2. 初始化固定大小分块器
|
|
text_splitter = CharacterTextSplitter(
|
|
chunk_size=200, # 每个块的大小
|
|
chunk_overlap=10 # 块之间的重叠大小
|
|
)
|
|
|
|
# 3. 执行分块
|
|
chunks = text_splitter.split_documents(docs)
|
|
|
|
# 4. 打印结果
|
|
print(f"文本被切分为 {len(chunks)} 个块。\n")
|
|
print("--- 前5个块内容示例 ---")
|
|
for i, chunk in enumerate(chunks[:5]):
|
|
print("=" * 60)
|
|
# chunk 是一个 Document 对象，需要访问它的 .page_content 属性来获取文本
|
|
print(f'块 {i+1} (长度: {len(chunk.page_content)}): "{chunk.page_content}"')
|