Initial commit

code/C2/04_semantic_chunker.py

@@ -0,0 +1,26 @@
+from langchain_experimental.text_splitter import SemanticChunker
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.document_loaders import TextLoader
+
+embeddings = HuggingFaceEmbeddings(
+    model_name="BAAI/bge-small-zh-v1.5",
+    model_kwargs={'device': 'cpu'},
+    encode_kwargs={'normalize_embeddings': True}
+)
+
+# 初始化 SemanticChunker
+text_splitter = SemanticChunker(
+    embeddings,
+    breakpoint_threshold_type="percentile" # 也可以是 "standard_deviation", "interquartile", "gradient"
+)
+
+loader = TextLoader("../../data/C2/txt/蜂医.txt", encoding="utf-8")
+documents = loader.load()
+
+docs = text_splitter.split_documents(documents)
+
+print(f"文本被切分为 {len(docs)} 个块。\n")
+print("--- 前2个块内容示例 ---")
+for i, chunk in enumerate(docs[:2]):
+    print("=" * 60)
+    print(f'块 {i+1} (长度: {len(chunk.page_content)}):\n"{chunk.page_content}"')