Initial commit

code/C2/01_unstructured_example.py

@@ -0,0 +1,25 @@
+from unstructured.partition.auto import partition
+
+# PDF文件路径
+pdf_path = "../../data/C2/pdf/rag.pdf"
+
+# 使用Unstructured加载并解析PDF文档
+elements = partition(
+    filename=pdf_path,
+    content_type="application/pdf"
+)
+
+# 打印解析结果
+print(f"解析完成: {len(elements)} 个元素, {sum(len(str(e)) for e in elements)} 字符")
+
+# 统计元素类型
+from collections import Counter
+types = Counter(e.category for e in elements)
+print(f"元素类型: {dict(types)}")
+
+# 显示所有元素
+print("\n所有元素:")
+for i, element in enumerate(elements, 1):
+    print(f"Element {i} ({element.category}):")
+    print(element)
+    print("=" * 60)

code/C2/02_character_splitter.py

@@ -0,0 +1,23 @@
+from langchain.text_splitter import CharacterTextSplitter
+from langchain_community.document_loaders import TextLoader
+
+# 1. 文档加载
+loader = TextLoader("../../data/C2/txt/蜂医.txt", encoding="utf-8")
+docs = loader.load()
+
+# 2. 初始化固定大小分块器
+text_splitter = CharacterTextSplitter(
+    chunk_size=200,    # 每个块的大小
+    chunk_overlap=10   # 块之间的重叠大小
+)
+
+# 3. 执行分块
+chunks = text_splitter.split_documents(docs)
+
+# 4. 打印结果
+print(f"文本被切分为 {len(chunks)} 个块。\n")
+print("--- 前5个块内容示例 ---")
+for i, chunk in enumerate(chunks[:5]):
+    print("=" * 60)
+    # chunk 是一个 Document 对象，需要访问它的 .page_content 属性来获取文本
+    print(f'块 {i+1} (长度: {len(chunk.page_content)}): "{chunk.page_content}"')

code/C2/03_recursive_character_splitter.py

@@ -0,0 +1,20 @@
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import TextLoader
+
+loader = TextLoader("../../data/C2/txt/蜂医.txt", encoding="utf-8")
+docs = loader.load()
+
+text_splitter = RecursiveCharacterTextSplitter(
+    # 针对中英文混合文本，定义一个更全面的分隔符列表
+    separators=["\n\n", "\n", "。", "，", " ", ""], # 按顺序尝试分割
+    chunk_size=200,
+    chunk_overlap=10
+)
+
+chunks = text_splitter.split_documents(docs)
+
+print(f"文本被切分为 {len(chunks)} 个块。\n")
+print("--- 前5个块内容示例 ---")
+for i, chunk in enumerate(chunks[:5]):
+    print("=" * 60)
+    print(f'块 {i+1} (长度: {len(chunk.page_content)}): "{chunk.page_content}"')

code/C2/04_semantic_chunker.py

@@ -0,0 +1,26 @@
+from langchain_experimental.text_splitter import SemanticChunker
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.document_loaders import TextLoader
+
+embeddings = HuggingFaceEmbeddings(
+    model_name="BAAI/bge-small-zh-v1.5",
+    model_kwargs={'device': 'cpu'},
+    encode_kwargs={'normalize_embeddings': True}
+)
+
+# 初始化 SemanticChunker
+text_splitter = SemanticChunker(
+    embeddings,
+    breakpoint_threshold_type="percentile" # 也可以是 "standard_deviation", "interquartile", "gradient"
+)
+
+loader = TextLoader("../../data/C2/txt/蜂医.txt", encoding="utf-8")
+documents = loader.load()
+
+docs = text_splitter.split_documents(documents)
+
+print(f"文本被切分为 {len(docs)} 个块。\n")
+print("--- 前2个块内容示例 ---")
+for i, chunk in enumerate(docs[:2]):
+    print("=" * 60)
+    print(f'块 {i+1} (长度: {len(chunk.page_content)}):\n"{chunk.page_content}"')