Initial commit
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
import os
|
||||
import asyncio
|
||||
from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings
|
||||
from llama_index.core.node_parser import SentenceWindowNodeParser, SentenceSplitter
|
||||
from llama_index.llms.deepseek import DeepSeek
|
||||
from llama_index.embeddings.huggingface import HuggingFaceEmbedding
|
||||
from llama_index.core.postprocessor import MetadataReplacementPostProcessor
|
||||
from llama_index.core.evaluation import (
|
||||
FaithfulnessEvaluator,
|
||||
RelevancyEvaluator,
|
||||
BatchEvalRunner,
|
||||
)
|
||||
from llama_index.core.evaluation.eval_utils import get_results_df
|
||||
from llama_index.core.evaluation import DatasetGenerator, QueryResponseDataset
|
||||
|
||||
Settings.llm = DeepSeek(model="deepseek-chat", temperature=0.1, api_key=os.getenv("DEEPSEEK_API_KEY"))
|
||||
Settings.embed_model = HuggingFaceEmbedding(model_name="BAAI/bge-small-en")
|
||||
|
||||
async def main():
|
||||
# 1. 加载文档
|
||||
reader = SimpleDirectoryReader(input_files=["../../data/C3/pdf/IPCC_AR6_WGII_Chapter03.pdf"])
|
||||
documents = reader.load_data()
|
||||
|
||||
# 1.1 加载或生成响应评估数据集
|
||||
if os.path.exists("./c6_response_eval_dataset.json"):
|
||||
print("加载响应评估数据集...")
|
||||
response_eval_dataset = QueryResponseDataset.from_json("./c6_response_eval_dataset.json")
|
||||
else:
|
||||
print("生成响应评估数据集...")
|
||||
dataset_generator = DatasetGenerator.from_documents(documents[:10]) # 减少文档数量
|
||||
response_eval_dataset = await dataset_generator.agenerate_dataset_from_nodes(num=15) # 减少问题数量
|
||||
response_eval_dataset.save_json("./c6_response_eval_dataset.json")
|
||||
|
||||
|
||||
|
||||
# 2. 构建两种不同的RAG查询引擎和检索器进行对比
|
||||
# 2.1 句子窗口检索
|
||||
sentence_parser = SentenceWindowNodeParser.from_defaults(
|
||||
window_size=3,
|
||||
window_metadata_key="window",
|
||||
original_text_metadata_key="original_text",
|
||||
)
|
||||
sentence_nodes = sentence_parser.get_nodes_from_documents(documents)
|
||||
sentence_index = VectorStoreIndex(sentence_nodes)
|
||||
|
||||
sentence_query_engine = sentence_index.as_query_engine(
|
||||
similarity_top_k=2,
|
||||
node_postprocessors=[
|
||||
MetadataReplacementPostProcessor(target_metadata_key="window")
|
||||
],
|
||||
)
|
||||
sentence_retriever = sentence_index.as_retriever(similarity_top_k=2)
|
||||
|
||||
# 2.2 常规分块检索（基准）
|
||||
base_parser = SentenceSplitter(chunk_size=512)
|
||||
base_nodes = base_parser.get_nodes_from_documents(documents)
|
||||
base_index = VectorStoreIndex(base_nodes)
|
||||
|
||||
base_query_engine = base_index.as_query_engine(similarity_top_k=2)
|
||||
base_retriever = base_index.as_retriever(similarity_top_k=2)
|
||||
|
||||
# 3. 初始化响应评估器
|
||||
faithfulness_evaluator = FaithfulnessEvaluator(llm=Settings.llm)
|
||||
relevancy_evaluator = RelevancyEvaluator(llm=Settings.llm)
|
||||
|
||||
# 4. 执行响应评估对比
|
||||
print("开始执行响应评估对比...")
|
||||
evaluators = {"faithfulness": faithfulness_evaluator, "relevancy": relevancy_evaluator}
|
||||
queries = response_eval_dataset.queries
|
||||
|
||||
# 句子窗口检索响应评估
|
||||
print("\n=== 评估句子窗口检索 ===")
|
||||
sentence_runner = BatchEvalRunner(evaluators, workers=2, show_progress=True)
|
||||
sentence_response_results = await sentence_runner.aevaluate_queries(
|
||||
queries=queries, query_engine=sentence_query_engine
|
||||
)
|
||||
|
||||
# 常规分块检索响应评估
|
||||
print("\n=== 评估常规分块检索 ===")
|
||||
base_runner = BatchEvalRunner(evaluators, workers=2, show_progress=True)
|
||||
base_response_results = await base_runner.aevaluate_queries(
|
||||
queries=queries, query_engine=base_query_engine
|
||||
)
|
||||
|
||||
# 5. 分析并打印对比结果
|
||||
print("\n" + "="*60)
|
||||
print("响应评估结果对比")
|
||||
print("="*60)
|
||||
|
||||
def calc_response_score(results, metric):
|
||||
if results and results.get(metric):
|
||||
scores = results[metric]
|
||||
return sum(r.passing for r in scores) / len(scores)
|
||||
return 0
|
||||
|
||||
# 句子窗口检索结果
|
||||
sentence_faith = calc_response_score(sentence_response_results, "faithfulness")
|
||||
sentence_rel = calc_response_score(sentence_response_results, "relevancy")
|
||||
|
||||
# 常规分块检索结果
|
||||
base_faith = calc_response_score(base_response_results, "faithfulness")
|
||||
base_rel = calc_response_score(base_response_results, "relevancy")
|
||||
|
||||
print(f"\n句子窗口检索:")
|
||||
print(f" 忠实度: {sentence_faith:.1%}")
|
||||
print(f" 相关性: {sentence_rel:.1%}")
|
||||
|
||||
print(f"\n常规分块检索:")
|
||||
print(f" 忠实度: {base_faith:.1%}")
|
||||
print(f" 相关性: {base_rel:.1%}")
|
||||
|
||||
|
||||
|
||||
# 简单对比
|
||||
if sentence_faith > base_faith and sentence_rel > base_rel:
|
||||
print(f"\n✅ 句子窗口检索在两个维度上都优于常规分块检索")
|
||||
elif sentence_faith > base_faith or sentence_rel > base_rel:
|
||||
print(f"\n⚖️ 句子窗口检索在某些维度上有优势")
|
||||
else:
|
||||
print(f"\n❌ 句子窗口检索未显示明显优势")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"queries": {
|
||||
"005f4f96-be64-48ad-ab90-38066f27e553": "Here are 10 diverse questions based solely on the provided context information:",
|
||||
"f522fc46-fd0f-462b-9307-d1628b4da112": "**Citation Question**: How should the chapter \"Oceans and Coastal Ecosystems and Their Services\" be cited according to the IPCC AR6 WGII report?",
|
||||
"64581ee3-eca7-4e50-84a0-dfde6f69abda": "**Authorship Question**: Who are the Coordinating Lead Authors of Chapter 3 in the IPCC AR6 WGII report?",
|
||||
"73298e8c-51a6-4424-8d5c-aa6c1ae5eb81": "**Geographical Representation Question**: Which countries are represented by the Lead Authors of this chapter?",
|
||||
"47a221d0-a95e-49f0-a0f8-13bcb2094c13": "**Page Range Question**: What is the page range for Chapter 3 in the IPCC AR6 WGII report?",
|
||||
"64ff4db4-cd98-4741-a35c-e5d3ac8b7cd2": "**DOI Question**: What is the DOI (Digital Object Identifier) for Chapter 3 of the IPCC AR6 WGII report?",
|
||||
"df2772ec-fbb1-4bde-9fde-87ecbbe242ed": "**Contributing Authors Question**: Name three Contributing Authors from different countries listed in the chapter.",
|
||||
"6e4ed553-7dc6-4db0-8c97-f5dfe8540ed3": "**Review Editors Question**: Who are the Review Editors for this chapter, and which countries do they represent?",
|
||||
"30d8e598-4a92-41e8-bafa-f62f503e2417": "**Chapter Scientist Question**: Who is credited as the Chapter Scientist for this chapter, and what is their affiliation?",
|
||||
"dc190ac2-6d6a-496c-a742-4c799cacf9b9": "**Publisher Question**: Which publishers are responsible for the IPCC AR6 WGII report, and where are they based?",
|
||||
"90275ad3-39f0-44b4-ae1a-da47454f4fe8": "**File Metadata Question**: What is the file size and creation date of the PDF document containing Chapter 3?",
|
||||
"2d2edcb3-3f47-4a73-94ff-459290b231c4": "These questions test different aspects of the provided context, including citation format, authorship details, metadata, and structural elements of the report.",
|
||||
"43f7eea1-2b43-4a99-ab50-fba2dd3471db": "Here are 10 diverse quiz/examination questions based strictly on the provided context:",
|
||||
"ea03ffc2-674d-413a-9954-e708c4b0527b": "**Multiple Choice**: According to Section 3.2.2, which of the following is NOT listed as a physical change in the global ocean due to climate change?",
|
||||
"242acbb0-4f32-4732-8cc0-6344f4e0f17c": "a) Sea level rise"
|
||||
},
|
||||
"responses": {
|
||||
"005f4f96-be64-48ad-ab90-38066f27e553": "Here are 10 diverse questions based solely on the provided context information:\n\n1. **Authorship**: Who are the Coordinating Lead Authors of Chapter 3 in the IPCC AR6 WGII report? \n - *Answer*: Sarah R. Cooley (USA) and David S. Schoeman (Australia). \n\n2. **Chapter Title**: What is the full title of Chapter 3 in the IPCC AR6 WGII report? \n - *Answer*: \"Oceans and Coastal Ecosystems and Their Services.\" \n\n3. **Page Range**: On which pages does Chapter 3 appear in the report? \n - *Answer*: Pages 379\u2013550. \n\n4. **DOI**: What is the Digital Object Identifier (DOI) for Chapter 3? \n - *Answer*: doi:10.1017/9781009325844.005. \n\n5. **Contributing Authors**: Name one Contributing Author from Nigeria. \n - *Answer*: Ibidun Adelekan. \n\n6. **Institutional Affiliation**: Which country/institution is Shin-Ichi Ito affiliated with as a Lead Author? \n - *Answer*: Japan. \n\n7. **Chapter Scientist**: Who served as the Chapter Scientist for this chapter? \n - *Answer*: Jessica Bolin (Australia). \n\n8. **Review Editors**: Who were the Review Editors for Chapter 3? \n - *Answer*: Karim Hilmi (Morocco) and Lisa Levin (USA). \n\n9. **Publisher**: Which publishers released the IPCC AR6 WGII report? \n - *Answer*: Cambridge University Press, Cambridge, UK, and New York, NY, USA. \n\n10. **File Metadata**: What is the file size of the PDF document containing Chapter 3? \n - *Answer*: 21,752,444 bytes (or ~21.75 MB). \n\nThese questions are derived exclusively from the provided context, ensuring no external knowledge is used.",
|
||||
"f522fc46-fd0f-462b-9307-d1628b4da112": "The chapter \"Oceans and Coastal Ecosystems and Their Services\" should be cited as follows: \n\n**Cooley, S., D. Schoeman, L. Bopp, P. Boyd, S. Donner, D.Y. Ghebrehiwet, S.-I. Ito, W. Kiessling, P. Martinetto, E. Ojea, M.-F. Racault, B. Rost, and M. Skern-Mauritzen, 2022: Oceans and Coastal Ecosystems and Their Services.** In: *Climate Change 2022: Impacts, Adaptation and Vulnerability. Contribution of Working Group II to the Sixth Assessment Report of the Intergovernmental Panel on Climate Change* [H.-O. P\u00f6rtner, D.C. Roberts, M. Tignor, E.S. Poloczanska, K. Mintenbeck, A. Alegr\u00eda, M. Craig, S. Langsdorf, S. L\u00f6schke, V. M\u00f6ller, A. Okem, B. Rama (eds.)]. Cambridge University Press, Cambridge, UK and New York, NY, USA, pp. 379\u2013550, doi:10.1017/9781009325844.005. \n\nThis citation includes the coordinating lead authors, lead authors, chapter title, report title, editors, publisher details, page range, and DOI.",
|
||||
"64581ee3-eca7-4e50-84a0-dfde6f69abda": "The Coordinating Lead Authors of Chapter 3 (\"Oceans and Coastal Ecosystems and Their Services\") in the IPCC AR6 WGII report are **Sarah R. Cooley (USA)** and **David S. Schoeman (Australia)**. \n\nThis information is explicitly stated in the provided context under the \"Coordinating Lead Authors\" section.",
|
||||
"73298e8c-51a6-4424-8d5c-aa6c1ae5eb81": "Based on the context information provided, the Lead Authors of the chapter \"Oceans and Coastal Ecosystems and Their Services\" represent the following countries: \n\n1. **USA** \u2013 Sarah R. Cooley \n2. **Australia** \u2013 David S. Schoeman \n3. **France** \u2013 Laurent Bopp \n4. **Australia/UK** \u2013 Philip Boyd \n5. **Canada** \u2013 Simon Donner \n6. **Japan** \u2013 Shin-Ichi Ito \n7. **Germany** \u2013 Wolfgang Kiessling \n8. **Argentina** \u2013 Paulina Martinetto \n9. **Spain** \u2013 Elena Ojea \n10. **UK/France** \u2013 Marie-Fanny Racault \n11. **Germany** \u2013 Bj\u00f6rn Rost \n12. **Norway** \u2013 Mette Skern-Mauritzen \n13. **South Africa/Eritrea** \u2013 Dawit Yemane Ghebrehiwet \n\nThus, the countries represented by the Lead Authors are: **USA, Australia, France, UK, Canada, Japan, Germany, Argentina, Spain, Norway, South Africa, and Eritrea**. Some authors have dual affiliations (e.g., Australia/UK, UK/France, South Africa/Eritrea).",
|
||||
"47a221d0-a95e-49f0-a0f8-13bcb2094c13": "The page range for Chapter 3 (\"Oceans and Coastal Ecosystems and Their Services\") in the IPCC AR6 WGII report is **379\u2013550**, as indicated in the citation provided in the context information. \n\nAnswer: The page range for Chapter 3 is 379\u2013550.",
|
||||
"64ff4db4-cd98-4741-a35c-e5d3ac8b7cd2": "The DOI for Chapter 3 of the IPCC AR6 WGII report is **10.1017/9781009325844.005**. \n\nThis is explicitly stated in the citation provided in the context information.",
|
||||
"df2772ec-fbb1-4bde-9fde-87ecbbe242ed": "Here are three Contributing Authors from different countries listed in the chapter:\n\n1. **Johann D. Bell** (Australia) \n2. **William W. L. Cheung** (Canada) \n3. **Elisabeth Holland** (Fiji) \n\nThese authors represent three distinct countries: Australia, Canada, and Fiji.",
|
||||
"6e4ed553-7dc6-4db0-8c97-f5dfe8540ed3": "The Review Editors for this chapter are **Karim Hilmi** (Morocco) and **Lisa Levin** (USA). They represent Morocco and the United States, respectively. \n\n(Answer derived directly from the provided context information.)",
|
||||
"30d8e598-4a92-41e8-bafa-f62f503e2417": "The Chapter Scientist credited for this chapter is **Jessica Bolin**, and their affiliation is **Australia**. \n\n(Source: Context information provided, under the \"Chapter Scientist\" section.)",
|
||||
"dc190ac2-6d6a-496c-a742-4c799cacf9b9": "The IPCC AR6 WGII report is published by two publishers: \n\n1. **Cambridge University Press** (based in Cambridge, UK) \n2. **Cambridge University Press** (based in New York, NY, USA) \n\nThis information is derived from the citation provided in the context: \n*\"Cambridge University Press, Cambridge, UK and New York, NY, USA, pp. 379\u2013550, doi:10.1017/9781009325844.005.\"* \n\n(Note: While the citation lists \"Cambridge University Press\" twice with different locations, it does not specify a separate publisher for the U.S. edition, implying the same publisher operates in both locations.)",
|
||||
"90275ad3-39f0-44b4-ae1a-da47454f4fe8": "The file size of the PDF document containing Chapter 3 is **21,752,444 bytes**, and the creation date is **2025-07-15**. \n\n(Source: Context metadata provided.)",
|
||||
"2d2edcb3-3f47-4a73-94ff-459290b231c4": "Here are the answers to different aspects of the provided context: \n\n1. **Citation Format**: \n The chapter should be cited as: \n *Cooley, S., D. Schoeman, L. Bopp, P. Boyd, S. Donner, D.Y. Ghebrehiwet, S.-I. Ito, W. Kiessling, P. Martinetto, E. Ojea, M.-F. Racault, B. Rost, and M. Skern-Mauritzen, 2022: Oceans and Coastal Ecosystems and Their Services. In: Climate Change 2022: Impacts, Adaptation and Vulnerability. Contribution of Working Group II to the Sixth Assessment Report of the Intergovernmental Panel on Climate Change [H.-O. P\u00f6rtner, D.C. Roberts, M. Tignor, E.S. Poloczanska, K. Mintenbeck, A. Alegr\u00eda, M. Craig, S. Langsdorf, S. L\u00f6schke, V. M\u00f6ller, A. Okem, B. Rama (eds.)]. Cambridge University Press, Cambridge, UK and New York, NY, USA, pp. 379\u2013550, doi:10.1017/9781009325844.005.* \n\n2. **Authorship Details**: \n - **Coordinating Lead Authors**: Sarah R. Cooley (USA), David S. Schoeman (Australia) \n - **Lead Authors**: Laurent Bopp (France), Philip Boyd (Australia/UK), Simon Donner (Canada), Shin-Ichi Ito (Japan), Wolfgang Kiessling (Germany), Paulina Martinetto (Argentina), Elena Ojea (Spain), Marie-Fanny Racault (UK/France), Bj\u00f6rn Rost (Germany), Mette Skern-Mauritzen (Norway), Dawit Yemane Ghebrehiwet (South Africa/Eritrea) \n - **Contributing Authors, Review Editors, and Chapter Scientist** are also listed in the context. \n\n3. **Metadata**: \n - **Page Label**: 379 \n - **File Name**: IPCC_AR6_WGII_Chapter03.pdf \n - **File Path**: ..\\..\\data\\C3\\pdf\\IPCC_AR6_WGII_Chapter03.pdf \n - **File Type**: application/pdf \n - **File Size**: 21,752,444 bytes (~21.7 MB) \n - **Creation Date**: 2025-07-15 \n - **Last Modified Date**: 2025-07-15 \n\n4. **Structural Elements**: \n - The chapter is part of **Working Group II (WGII)** of the **Sixth Assessment Report (AR6)** of the IPCC. \n - It covers **Oceans and Coastal Ecosystems and Their Services** (pages 379\u2013550). \n - The **DOI** for the chapter is **10.1017/9781009325844.005**. \n\nLet me know if you need further details!",
|
||||
"43f7eea1-2b43-4a99-ab50-fba2dd3471db": "Here are 10 diverse quiz/examination questions based strictly on the provided context from Chapter 3 of the IPCC AR6 WGII report: \n\n1. **Multiple Choice**: Which section of Chapter 3 discusses the challenges of attributing changes in marine ecosystems specifically to climate change? \n a) 3.1 Point of Departure \n b) FAQ 3.1 \n c) 3.2.1 Introduction \n d) Box 3.1 \n\n2. **True/False**: According to the table of contents, the chapter includes a FAQ about marine heatwaves and their impacts on marine life and human communities. (True/False) \n\n3. **Short Answer**: Name two types of climate-induced drivers discussed in Section 3.2 (Observed Trends and Projections of Climatic Impact-Drivers in the Global Ocean). \n\n4. **Fill in the Blank**: Section 3.3.4 focuses on __________ and evolutionary adaptation in marine organisms in response to climate drivers. \n\n5. **Matching**: Match the following subsections to their primary focus: \n - 3.2.2 \n - 3.2.3 \n - 3.3.3 \n - 3.4.2 \n Options: \n a) Chemical changes in the ocean \n b) Coastal ecosystems and seas \n c) Physical changes in the ocean \n d) Responses to multiple drivers \n\n6. **Short Answer**: What is the title of Box 3.2, and what organisms does it focus on? \n\n7. **Multiple Choice**: Which FAQ addresses the concept of \"tipping points\" in the ocean? \n a) FAQ 3.1 \n b) FAQ 3.2 \n c) FAQ 3.3 \n d) None of the above \n\n8. **True/False**: Section 3.3.5 discusses ecological responses to single drivers only. (True/False) \n\n9. **Short Answer**: In which section would you find a discussion of acclimation and adaptation mechanisms in marine species? \n\n10. **Essay Prompt (Brief)**: Based on the table of contents, explain how Chapter 3 structures its analysis of climate impacts on oceans and coastal ecosystems (e.g., drivers, biological responses, observed/projected impacts). \n\nThese questions test comprehension of the chapter's structure, key themes, and specific details while adhering strictly to the provided context. Let me know if you'd like adjustments or additional question types!",
|
||||
"ea03ffc2-674d-413a-9954-e708c4b0527b": "To answer this question, we need to refer to **Section 3.2.2 (Physical Changes)** in the provided context. However, the context does not explicitly list the specific physical changes discussed in that section. Since the detailed content of Section 3.2.2 is not included in the given context, we cannot definitively identify which option is **NOT** listed as a physical change.\n\nIf this were a real exam question, you would need to refer to the actual text of **Section 3.2.2** in the IPCC AR6 WGII Chapter 3 document to determine the correct answer. Without that specific information, we cannot provide an accurate response. \n\nWould you like me to infer possible physical changes based on general climate science knowledge? (Note: This would go beyond the given context.)",
|
||||
"242acbb0-4f32-4732-8cc0-6344f4e0f17c": "The context information provided does not explicitly mention \"sea level rise\" in the listed sections or headings. However, given that the chapter focuses on \"Oceans and Coastal Ecosystems and Their Services,\" it is likely that sea level rise is discussed in sections such as **3.2.2 Physical Changes** (which may cover ocean warming, ice melt, and associated sea level changes) or **3.4.2 Coastal Ecosystems and Seas** (which may address impacts on coastal areas). \n\nFor a precise answer, you would need to refer to the content of these sections in the actual document (e.g., page 392 for physical changes or page 410 for coastal ecosystems). Would you like me to infer potential connections based on typical IPCC reporting?"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user