laion/wikipedia_de_retival_BGE-m3

import os
import pandas as pd
from pathlib import Path

import retriv
retriv.set_base_path("./retriv_wiki_de")

from retriv import DenseRetriever

"""
# Uncomment if you wanna make your own index
dr = DenseRetriever(
index_name="wiki_de-index_sentence_transf-BAAI/bge-m3_title_only_fullarticles",
model="BAAI/bge-m3",
normalize=True,
max_length=512,
use_ann=True,
)

dr = dr.index_file(
path="./wikipedia_de_filtered_fullarticles.csv", # File kind is automatically inferred
embeddings_path=None, # Default value
use_gpu=True, # Default value
batch_size=32, # Default value
show_progress=True, # Default value
callback=lambda doc: { # Callback defaults to None.
"id": doc["id"],
"text": doc["title"],
},
)
"""

from retriv import DenseRetriever

# loading the wikipedia de text data
file_path = "./wikipedia_de_filtered_fullarticles.csv" # CSV with fulltext
df = pd.read_csv(file_path)

file_path = "./wikipedia_de_filtered_300wordchunks.csv" # CSV with fulltext
df2 = pd.read_csv(file_path)

# loading the retrievers
dr = DenseRetriever.load("wiki_de-index_sentence_transf-BAAI/bge-m3_title_only_fullarticles") # the embeddings here are made from the titles of the wikipedia pages, but can be matched to the full texts in the wikipedia_de_filtered_fullarticles.csv

result = dr.search(
query="was is der doppelspaltversuch?", # What to search for
return_docs=True, # Default value, return the text of the documents
cutoff=3, # Default value, number of results to return
)
print(df)

for res in result:

id_query = int(res["id"])-1
row = df.iloc[id_query]

print(row)
# Extracting 'text' and 'url' from the resulting row
result_text = row['text']
result_url = row['url']
print(result_url,result_text[:1000])

print("###################")
print("+++++++++++++++++++")

dr2 = DenseRetriever.load("wiki_de-index_sentence_transf-BAAI/bge-m3") # the embeddings here are made from 300 word segments of the articles. The IDs point to wikipedia_de_filtered_300wordchunks.csv

result = dr2.search(
query="was is der doppelspaltversuch?", # What to search for
return_docs=True, # Default value, return the text of the documents
cutoff=3, # Default value, number of results to return
)

for res in result:

id_query = int(res["id"])-1 # the "id" values start with 1, not 0 , -> need to substract 1 ;)
row = df2.iloc[id_query]

print(row)
# Extracting 'text' and 'url' from the resulting row
result_text = row['text']
result_url = row['url']
print(result_url,result_text)

print("########")

import os
import pandas as pd
from pathlib import Path

import retriv
# 设置retriv库的基础路径为 ./retriv_wiki_de
retriv.set_base_path("./retriv_wiki_de")

from retriv import DenseRetriever

'''
# 若需自行构建索引，请取消以下注释
dr = DenseRetriever(
index_name="wiki_de-index_sentence_transf-BAAI/bge-m3_title_only_fullarticles",
model="BAAI/bge-m3",
normalize=True,
max_length=512,
use_ann=True,
)

dr = dr.index_file(
path="./wikipedia_de_filtered_fullarticles.csv", # 自动识别文件类型
embeddings_path=None, # 默认值
use_gpu=True, # 默认启用GPU加速
batch_size=32, # 默认批量大小为32
show_progress=True, # 默认显示进度条
callback=lambda doc: { # 回调函数默认值为None，此处用于提取文档ID与标题文本作为检索文本
"id": doc["id"],
"text": doc["title"],
},
)
'''

# 加载德语维基百科文本数据集
# 读取存储完整文章文本的CSV文件
file_path = "./wikipedia_de_filtered_fullarticles.csv"
df = pd.read_csv(file_path)

# 读取存储文章300词分段文本的CSV文件
file_path = "./wikipedia_de_filtered_300wordchunks.csv"
df2 = pd.read_csv(file_path)

# 加载已构建的密集检索器（DenseRetriever）
dr = DenseRetriever.load("wiki_de-index_sentence_transf-BAAI/bge-m3_title_only_fullarticles")
# 该检索器的嵌入向量基于维基百科页面标题生成，可与`wikipedia_de_filtered_fullarticles.csv`中的完整文章文本进行匹配检索

# 执行检索查询
result = dr.search(
query="was is der doppelspaltversuch?", # 待搜索查询内容
return_docs=True, # 默认值，返回匹配文档的文本内容
cutoff=3, # 默认值，返回Top3匹配结果
)
# 打印完整的数据集DataFrame
print(df)

for res in result:
# 检索器的文档ID从1开始计数，需减1以匹配Python DataFrame的0索引规则
id_query = int(res["id"]) - 1
row = df.iloc[id_query]
print(row)
# 从匹配结果行中提取`text`与`url`字段
result_text = row['text']
result_url = row['url']
# 打印URL与前1000字符的匹配文本
print(result_url, result_text[:1000])

print("###################")
print("+++++++++++++++++++")

# 加载第二个密集检索器（DenseRetriever）
dr2 = DenseRetriever.load("wiki_de-index_sentence_transf-BAAI/bge-m3")
# 该检索器的嵌入向量基于文章的300词分段生成，其文档ID与`wikipedia_de_filtered_300wordchunks.csv`中的条目一一对应

# 执行第二次检索查询
result = dr2.search(
query="was is der doppelspaltversuch?", # 待搜索查询内容
return_docs=True, # 默认值，返回匹配文档的文本内容
cutoff=3, # 默认值，返回Top3匹配结果
)

for res in result:
# 检索器的文档ID从1开始计数，需减1以匹配Python DataFrame的0索引规则
id_query = int(res["id"]) - 1
row = df2.iloc[id_query]
print(row)
# 从匹配结果行中提取`text`与`url`字段
result_text = row['text']
result_url = row['url']
# 打印URL与完整匹配文本
print(result_url, result_text)

print("########")

原始信息汇总

数据集概述

数据集文件

• wikipedia_de_filtered_fullarticles.csv：包含完整文本的CSV文件。
• wikipedia_de_filtered_300wordchunks.csv：包含每篇文章300字片段的CSV文件。

数据集处理

• 使用DenseRetriever加载和索引数据集。
  • 索引名称：wiki_de-index_sentence_transf-BAAI/bge-m3_title_only_fullarticles
  • 模型：BAAI/bge-m3
  • 参数设置：
    • 归一化：True
    • 最大长度：512
    • 使用GPU：True
    • 批处理大小：32
    • 显示进度：True
• 通过查询进行搜索，返回文档文本，默认返回前3个结果。

数据集应用示例

• 搜索查询：was is der doppelspaltversuch?
• 结果处理：
  • 从wikipedia_de_filtered_fullarticles.csv和wikipedia_de_filtered_300wordchunks.csv中提取文本和URL。
  • 打印搜索结果的文本和URL。