我正在编辑【python】代码，遇到了 【Traceback (most recent call last):
File "D:\python\python3.11.5\Lib\site-packages\langchain\_api\module_import.py", line 69, in import_by_name
module = importlib.import_module(new_module)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\python\python3.11.5\Lib\importlib\__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<frozen importlib._bootstrap>", line 1204, in _gcd_import
File "<frozen importlib._bootstrap>", line 1176, in _find_and_load
File "<frozen importlib._bootstrap>", line 1126, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
File "<frozen importlib._bootstrap>", line 1204, in _gcd_import
File "<frozen importlib._bootstrap>", line 1176, in _find_and_load
File "<frozen importlib._bootstrap>", line 1140, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'langchain_community'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\code\nepu_spider\rag\rag_sys.py", line 11, in <module>
from langchain.embeddings import HuggingFaceEmbeddings
File "D:\python\python3.11.5\Lib\site-packages\langchain\embeddings\__init__.py", line 167, in __getattr__
return _import_attribute(name)
^^^^^^^^^^^^^^^^^^^^^^^
File "D:\python\python3.11.5\Lib\site-packages\langchain\_api\module_import.py", line 72, in import_by_name
raise ModuleNotFoundError(
ModuleNotFoundError: Module langchain_community.embeddings not found. Please install langchain-community to access this module. You can install it using `pip install -U langchain-community`
】
，请帮我检查并改正错误点。我的原始代码如下：
【import os
import json
import pickle
from langchain_core.language_models import LLM
import install_faiss
import numpy as np
import requests
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import FAISS
from langchain.chains import RetrievalQA
from langchain.prompts import PromptTemplate
from langchain.llms import OpenAI
from langchain.docstore.document import Document
from typing import List, Dict, Any, Tuple, Optional
# 自定义智谱AI的LangChain包装器
class ZhipuAILLM(LLM):
def __init__(self, api_key: str, model: str = "glm-4", temperature: float = 0.7, zhipuai=None):
"""初始化智谱AI模型"""
self.api_key = api_key
self.model = model
self.temperature = temperature
zhipuai.api_key = api_key # 设置API密钥
def _call(self, prompt: str, stop: Optional[List[str]] = None, zhipuai=None) -> str:
"""调用智谱AI API生成文本"""
try:
response = zhipuai.model_api.invoke(
model=self.model,
prompt=[{"role": "user", "content": prompt}],
temperature=self.temperature,
top_p=0.7,
)
if response["code"] == 200:
return response["data"]["choices"][0]["content"]
else:
raise ValueError(f"智谱AI API错误: {response['msg']}")
except Exception as e:
raise RuntimeError(f"调用智谱AI失败: {str(e)}")
@property
def _llm_type(self) -> str:
return "zhipuai"
class RAGSystem:
def __init__(self, config: Dict[str, Any]):
"""初始化RAG系统"""
self.config = config
self.embedding_model = self._load_embedding_model()
self.llm = self._load_llm()
self.vector_db = None
self.doc_id_map = {}
# 自动加载或创建索引
self._initialize_index()
def _load_embedding_model(self) -> HuggingFaceEmbeddings:
"""加载嵌入模型"""
model_name = self.config.get("embedding_model", "sentence-transformers/all-mpnet-base-v2")
return HuggingFaceEmbeddings(
model_name=model_name,
model_kwargs={"device": self.config.get("embedding_device", "cpu")}
)
def _load_llm(self):
"""加载大语言模型 - 使用智谱AI"""
llm_provider = self.config.get("llm_provider", "zhipuai")
if llm_provider == "zhipuai":
return ZhipuAILLM(
api_key=self.config["zhipuai_api_key"], # 使用智谱AI密钥
model=self.config.get("llm_model", "glm-4"),
temperature=self.config.get("temperature", 0.7)
)
elif llm_provider == "openai":
# 保留OpenAI支持
from langchain.llms import OpenAI
return OpenAI(
api_key=self.config["openai_api_key"],
model_name=self.config.get("llm_model", "gpt-3.5-turbo"),
temperature=self.config.get("temperature", 0.7)
)
else:
raise ValueError(f"不支持的LLM提供者: {llm_provider}")
def _initialize_index(self):
"""初始化索引：加载现有或创建新索引"""
index_path = self.config["index_path"]
if os.path.exists(index_path):
print(f"加载现有索引: {index_path}")
self._load_vector_index()
else:
print(f"创建新索引: {index_path}")
self._create_new_index()
def _create_new_index(self):
"""创建新索引"""
data_dir = self.config["data_dir"]
if not os.path.exists(data_dir):
print(f"数据目录不存在: {data_dir}")
if self.config.get("auto_download", False):
self._download_sample_data()
else:
raise FileNotFoundError(f"数据目录不存在: {data_dir}")
documents = self._load_and_process_documents()
self._create_vector_index(documents)
def _download_sample_data(self):
"""下载示例数据"""
print("下载示例数据...")
data_dir = self.config["data_dir"]
os.makedirs(data_dir, exist_ok=True)
sample_urls = [
"https://raw.githubusercontent.com/langchain-ai/langchain/master/docs/docs_skeleton.json"
]
for url in sample_urls:
response = requests.get(url)
filename = os.path.basename(url)
file_path = os.path.join(data_dir, filename)
with open(file_path, "wb") as f:
f.write(response.content)
print(f"下载完成: {filename}")
def _load_and_process_documents(self) -> List[Document]:
"""加载并处理文档"""
documents = []
data_dir = self.config["data_dir"]
# 支持多种文件格式
for filename in os.listdir(data_dir):
file_path = os.path.join(data_dir, filename)
if filename.endswith(".json") or filename.endswith(".jsonl"):
documents.extend(self._load_json_documents(file_path))
elif filename.endswith(".txt"):
documents.extend(self._load_text_documents(file_path))
if not documents:
raise ValueError(f"在 {data_dir} 中没有找到可处理的文档")
# 文本分块
return self._split_documents(documents)
def _load_json_documents(self, file_path: str) -> List[Document]:
"""加载JSON或JSONL文档"""
documents = []
with open(file_path, "r") as f:
if file_path.endswith(".jsonl"):
# 处理JSONL文件
for line in f:
try:
data = json.loads(line)
doc = self._create_document_from_data(data)
documents.append(doc)
except json.JSONDecodeError:
print(f"跳过无效的JSON行: {line.strip()}")
else:
# 处理JSON文件
try:
data = json.load(f)
if isinstance(data, list):
for item in data:
doc = self._create_document_from_data(item)
documents.append(doc)
elif isinstance(data, dict):
doc = self._create_document_from_data(data)
documents.append(doc)
except json.JSONDecodeError:
print(f"无效的JSON文件: {file_path}")
return documents
def _load_text_documents(self, file_path: str) -> List[Document]:
"""加载纯文本文档"""
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
return [Document(
page_content=content,
metadata={
"source": file_path,
"title": os.path.basename(file_path),
"category": "text"
}
)]
def _create_document_from_data(self, data: Dict) -> Document:
"""从数据创建文档对象"""
return Document(
page_content=data.get("content", data.get("text", "")),
metadata={
"source": data.get("url", data.get("source", "")),
"title": data.get("title", ""),
"category": data.get("category", "unknown"),
"timestamp": data.get("timestamp", "")
}
)
def _split_documents(self, documents: List[Document]) -> List[Document]:
"""分割文档为块"""
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=self.config.get("chunk_size", 1000),
chunk_overlap=self.config.get("chunk_overlap", 200),
length_function=len
)
return text_splitter.split_documents(documents)
def _create_vector_index(self, documents: List[Document]):
"""创建FAISS向量索引"""
# 创建向量数据库
self.vector_db = FAISS.from_documents(
documents=documents,
embedding=self.embedding_model
)
# 保存索引
os.makedirs(os.path.dirname(self.config["index_path"]), exist_ok=True)
self.vector_db.save_local(self.config["index_path"])
# 创建文档ID映射
for idx, doc in enumerate(documents):
self.doc_id_map[idx] = {
"source": doc.metadata["source"],
"title": doc.metadata["title"],
"category": doc.metadata["category"]
}
# 保存映射表
map_path = os.path.join(os.path.dirname(self.config["index_path"]), "doc_id_map.pkl")
with open(map_path, "wb") as f:
pickle.dump(self.doc_id_map, f)
print(f"✅ 向量索引已创建并保存至 {self.config['index_path']}")
def _load_vector_index(self):
"""加载现有的FAISS向量索引"""
index_path = self.config["index_path"]
# 加载向量数据库
self.vector_db = FAISS.load_local(
folder_path=index_path,
embeddings=self.embedding_model
)
# 加载文档映射表
map_path = os.path.join(os.path.dirname(index_path), "doc_id_map.pkl")
if os.path.exists(map_path):
with open(map_path, "rb") as f:
self.doc_id_map = pickle.load(f)
print(f"✅ 文档映射表已加载")
else:
print("⚠️ 文档映射表未找到，将使用空映射")
def ask_question(self, question: str, history: Optional[List] = None) -> Dict:
"""提问并获取答案"""
if self.vector_db is None:
raise ValueError("向量数据库未初始化")
# 创建检索器
retriever = self.vector_db.as_retriever(
search_kwargs={
"k": self.config.get("retrieval_top_k", 5),
"score_threshold": self.config.get("score_threshold", 0.4)
}
)
# 创建问答链
qa_chain = self._create_qa_chain(retriever)
# 执行问答
result = qa_chain({"query": question})
# 提取源文档
source_docs = result["source_documents"]
doc_ids = [doc.metadata.get("doc_id", idx) for idx, doc in enumerate(source_docs)]
# 获取完整上下文
full_contexts = [self.doc_id_map.get(did, {"title": "未知", "source": ""}) for did in doc_ids]
return {
"question": question,
"answer": result["result"],
"source_documents": source_docs,
"full_contexts": full_contexts
}
def _create_qa_chain(self, retriever) -> Any:
"""创建问答链"""
# 自定义提示模板
prompt_template = """
基于以下上下文信息，请以专业、准确的方式回答用户的问题。如果上下文信息不足以回答问题，请如实告知用户。
上下文信息:
{context}
问题: {question}
请提供详细的回答:
"""
QA_PROMPT = PromptTemplate(
template=prompt_template,
input_variables=["context", "question"]
)
# 创建问答链
return RetrievalQA.from_chain_type(
llm=self.llm,
chain_type="stuff",
retriever=retriever,
return_source_documents=True,
chain_type_kwargs={"prompt": QA_PROMPT}
)
def add_document(self, content: str, metadata: Dict):
"""添加单个文档到索引"""
if self.vector_db is None:
raise ValueError("向量数据库未初始化")
# 创建文档对象
doc = Document(page_content=content, metadata=metadata)
# 分割文档
split_docs = self._split_documents([doc])
# 添加到索引
self.vector_db.add_documents(split_docs)
# 更新文档映射
start_idx = max(self.doc_id_map.keys()) + 1 if self.doc_id_map else 0
for idx, doc in enumerate(split_docs):
self.doc_id_map[start_idx + idx] = {
"source": doc.metadata["source"],
"title": doc.metadata["title"],
"category": doc.metadata["category"]
}
print(f"✅ 添加了 {len(split_docs)} 个文档块")
def save_index(self):
"""保存索引到磁盘"""
if self.vector_db is None:
raise ValueError("向量数据库未初始化")
# 保存索引
self.vector_db.save_local(self.config["index_path"])
# 保存映射表
map_path = os.path.join(os.path.dirname(self.config["index_path"]), "doc_id_map.pkl")
with open(map_path, "wb") as f:
pickle.dump(self.doc_id_map, f)
print(f"✅ 索引已保存至 {self.config['index_path']}")
def create_default_config() -> Dict:
"""创建默认配置 - 使用智谱AI"""
return {
"data_dir": "data",
"index_path": "index/faiss_index",
"embedding_model": "sentence-transformers/all-mpnet-base-v2",
"embedding_device": "cpu",
"llm_provider": "zhipuai", # 默认使用智谱AI
"zhipuai_api_key": "1fc6d23e95224503aa94bfcca6a31903.FXoiEbfDgymrE9FA", # 您的智谱AI密钥
"llm_model": "glm-4", # 智谱AI的GLM-4模型
"temperature": 0.7,
"chunk_size": 1000,
"chunk_overlap": 200,
"retrieval_top_k": 5,
"score_threshold": 0.4,
"auto_download": True
}
def interactive_cli():
"""交互式命令行界面 - 适配智谱AI"""
config = create_default_config()
# 设置智谱AI API密钥
api_key = input("请输入智谱AI API密钥（或直接回车使用默认值）: ")
if api_key.strip():
config["zhipuai_api_key"] = api_key
# 选择模型
model_choice = input("请选择模型 (1=GLM-4, 2=GLM-3-Turbo, 回车使用GLM-4): ")
if model_choice == "2":
config["llm_model"] = "glm-3-turbo"
# 初始化RAG系统
rag = RAGSystem(config)
# 交互问答
print("\nRAG系统已就绪（使用智谱AI），输入问题开始查询（输入'q'退出）")
history = []
while True:
question = input("\n>>> 问题: ")
if question.lower() in ['q', 'quit', 'exit']:
break
try:
result = rag.ask_question(question, history)
# 显示结果
print(f"\n💡 答案: {result['answer']}")
if result["full_contexts"]:
print("\n📚 信息来源:")
for i, ctx in enumerate(result["full_contexts"]):
print(f" {i + 1}. {ctx['title']} ({ctx['category']})")
print(f" 来源: {ctx['source']}")
# 添加到历史
history.append({"question": question, "answer": result["answer"]})
except Exception as e:
print(f"❌ 错误: {str(e)}")
# 保存索引
rag.save_index()
print("\n索引已保存，再见！")
if __name__ == "__main__":
interactive_cli()
】
最新发布
扫一扫
30万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
