【RAG】CSVをFaissに格納し、メタデータを活用した検索をLLMで行う方法

from pprint import pprint

from langchain.document_loaders.csv_loader import CSVLoader

file_path="./data/Womens Clothing E-Commerce Reviews.csv"

loader = CSVLoader(file_path=file_path)
docs = loader.load()
docs = docs[:100]

pprint(docs[:5])

[Document(page_content=': 0\nClothing ID: 767\nAge: 33\nTitle: \nReview Text: Absolutely wonderful - silky and sexy and comfortable\nRating: 4\nRecommended IND: 1\nPositive Feedback Count: 0\nDivision Name: Initmates\nDepartment Name: Intimate\nClass Name: Intimates', metadata={'source': './data/rag_table_data/Womens Clothing E-Commerce Reviews.csv', 'row': 0}),
 Document(page_content=': 1\nClothing ID: 1080\nAge: 34\nTitle: \nReview Text: Love this dress!  it\'s sooo pretty.  i happened to find it in a store, and i\'m glad i did bc i never would have ordered it online bc it\'s petite.  i bought a petite and am 5\'8".  i love the length on me- hits just a little below the knee.  would definitely be a true midi on someone who is truly petite.\nRating: 5\nRecommended IND: 1\nPositive Feedback Count: 4\nDivision Name: General\nDepartment Name: Dresses\nClass Name: Dresses', metadata={'source': './data/rag_table_data/Womens Clothing E-Commerce Reviews.csv', 'row': 1}),
 Document(page_content=': 2\nClothing ID: 1077\nAge: 60\nTitle: Some major design flaws\nReview Text: I had such high hopes for this dress and really wanted it to work for me. i initially ordered the petite small (my usual size) but i found this to be outrageously small. so small in fact that i could not zip it up! i reordered it in petite medium, which was just ok. overall, the top half was comfortable and fit nicely, but the bottom half had a very tight under layer and several somewhat cheap (net) over layers. imo, a major design flaw was the net over layer sewn directly into the zipper - it c\nRating: 3\nRecommended IND: 0\nPositive Feedback Count: 0\nDivision Name: General\nDepartment Name: Dresses\nClass Name: Dresses', metadata={'source': './data/rag_table_data/Womens Clothing E-Commerce Reviews.csv', 'row': 2}),
 Document(page_content=": 3\nClothing ID: 1049\nAge: 50\nTitle: My favorite buy!\nReview Text: I love, love, love this jumpsuit. it's fun, flirty, and fabulous! every time i wear it, i get nothing but great compliments!\nRating: 5\nRecommended IND: 1\nPositive Feedback Count: 0\nDivision Name: General Petite\nDepartment Name: Bottoms\nClass Name: Pants", metadata={'source': './data/rag_table_data/Womens Clothing E-Commerce Reviews.csv', 'row': 3}),
 Document(page_content=': 4\nClothing ID: 847\nAge: 47\nTitle: Flattering shirt\nReview Text: This shirt is very flattering to all due to the adjustable front tie. it is the perfect length to wear with leggings and it is sleeveless so it pairs well with any cardigan. love this shirt!!!\nRating: 5\nRecommended IND: 1\nPositive Feedback Count: 6\nDivision Name: General\nDepartment Name: Tops\nClass Name: Blouses', metadata={'source': './data/rag_table_data/Womens Clothing E-Commerce Reviews.csv', 'row': 4})]

import pandas as pd
from langchain.schema.document import Document

docs = []
df = pd.read_csv(file_path, index_col=0)
for i, row in enumerate(df.to_dict(orient='records')):
    if i < 100:
        review = row.pop("Review Text")
        title = row.pop("Title")
        metadata = row
        metadata["row"] = i
        metadata["source"] = Path(file_path).name
        page_content = f"Title: {title}\n {review}"
        docs.append(Document(page_content=page_content, metadata=metadata))
pprint(docs[:5])

[Document(page_content='Title: nan\n Absolutely wonderful - silky and sexy and comfortable', metadata={'Clothing ID': 767, 'Age': 33, 'Rating': 4, 'Recommended IND': 1, 'Positive Feedback Count': 0, 'Division Name': 'Initmates', 'Department Name': 'Intimate', 'Class Name': 'Intimates', 'row': 0, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content='Title: nan\n Love this dress!  it\'s sooo pretty.  i happened to find it in a store, and i\'m glad i did bc i never would have ordered it online bc it\'s petite.  i bought a petite and am 5\'8".  i love the length on me- hits just a little below the knee.  would definitely be a true midi on someone who is truly petite.', metadata={'Clothing ID': 1080, 'Age': 34, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 4, 'Division Name': 'General', 'Department Name': 'Dresses', 'Class Name': 'Dresses', 'row': 1, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content='Title: Some major design flaws\n I had such high hopes for this dress and really wanted it to work for me. i initially ordered the petite small (my usual size) but i found this to be outrageously small. so small in fact that i could not zip it up! i reordered it in petite medium, which was just ok. overall, the top half was comfortable and fit nicely, but the bottom half had a very tight under layer and several somewhat cheap (net) over layers. imo, a major design flaw was the net over layer sewn directly into the zipper - it c', metadata={'Clothing ID': 1077, 'Age': 60, 'Rating': 3, 'Recommended IND': 0, 'Positive Feedback Count': 0, 'Division Name': 'General', 'Department Name': 'Dresses', 'Class Name': 'Dresses', 'row': 2, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content="Title: My favorite buy!\n I love, love, love this jumpsuit. it's fun, flirty, and fabulous! every time i wear it, i get nothing but great compliments!", metadata={'Clothing ID': 1049, 'Age': 50, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 0, 'Division Name': 'General Petite', 'Department Name': 'Bottoms', 'Class Name': 'Pants', 'row': 3, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content='Title: Flattering shirt\n This shirt is very flattering to all due to the adjustable front tie. it is the perfect length to wear with leggings and it is sleeveless so it pairs well with any cardigan. love this shirt!!!', metadata={'Clothing ID': 847, 'Age': 47, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 6, 'Division Name': 'General', 'Department Name': 'Tops', 'Class Name': 'Blouses', 'row': 4, 'source': 'Womens Clothing E-Commerce Reviews.csv'})]

import openai
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores import FAISS

faiss_index_path = "./data/faiss_index"

embeddings = OpenAIEmbeddings(client=openai.ChatCompletion)
faiss_index = FAISS.from_documents(docs, embeddings)
faiss_index.save_local(faiss_index_path)

llm = ChatOpenAI(model=MODEL_NAME, temperature=MODEL_TEMPERATURE)
embeddings = OpenAIEmbeddings(client=openai.ChatCompletion)
faiss_index = FAISS.load_local(faiss_index_path, embeddings=embeddings)

retriever = faiss_index.as_retriever()
pprint(retriever.get_relevant_documents("comfortable"))

[Document(page_content="Index: 53 Title: Flattering and comfortable\n Very soft and comfortable. the shirt has an unusual, asymmetrical seam that appears along the front, right-hand side of the garment. (the model is positioned so that you can't see this detail from the picture. i attached a picture that includes the front seam.) i actually like the seam - it creates more visual interest, and adds a little bit of ruching that helps hide my belly. the cowl neck is very well done - there are two layers that form the cowl, and i've found that it means wardrobe malfun\n", metadata={'Clothing ID': 862, 'Age': 39, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 34, 'Division Name': 'General', 'Department Name': 'Tops', 'Class Name': 'Knits', 'row': 53, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content='Index: 0 Title: nan\n Absolutely wonderful - silky and sexy and comfortable\n', metadata={'Clothing ID': 767, 'Age': 33, 'Rating': 4, 'Recommended IND': 1, 'Positive Feedback Count': 0, 'Division Name': 'Initmates', 'Department Name': 'Intimate', 'Class Name': 'Intimates', 'row': 0, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content="Index: 59 Title: Soft and comfortable\n I've been looking for bralettes that provide some support but not binding/tight for night time and casual wear. this is a light weight bra, could be a little more supportive. pretty color, with nice lines. only downside is the retailer tag on the back - itchy (i need to remove it.)\n", metadata={'Clothing ID': 368, 'Age': 52, 'Rating': 4, 'Recommended IND': 1, 'Positive Feedback Count': 1, 'Division Name': 'Initmates', 'Department Name': 'Intimate', 'Class Name': 'Intimates', 'row': 59, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content='Index: 47 Title: nan\n Very comfortable, material is good, cut out on sleeves flattering\n', metadata={'Clothing ID': 823, 'Age': 52, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 0, 'Division Name': 'General', 'Department Name': 'Tops', 'Class Name': 'Blouses', 'row': 47, 'source': 'Womens Clothing E-Commerce Reviews.csv'})]

retriever = faiss_index.as_retriever(
			search_type="similarity",
			search_kwargs={
				"filter": {"Class Name": "Blouses"},
				}
			)
pprint(retriever.get_relevant_documents("comfortable"))

[Document(page_content='Index: 47 Title: nan\n Very comfortable, material is good, cut out on sleeves flattering\n', metadata={'Clothing ID': 823, 'Age': 52, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 0, 'Division Name': 'General', 'Department Name': 'Tops', 'Class Name': 'Blouses', 'row': 47, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content="Index: 72 Title: Beautiful design\n I have a short torso and this works well for me. 34c, bought the 0. there's not much stretch to the fabric so it is fitted to my chest, but not in an uncomfortable way. definitely doesn't hang and have extra fabric like on the model. \r\n\r\nzipper goes almost all the way down to the bottom so it's easier to get on and off which makes up for the lack of stretch n the fabric.\r\n\r\nunlike another reviewer, i found it went really well with navy pants and i wore it to a business meeting under a blazer. wi\n", metadata={'Clothing ID': 850, 'Age': 27, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 4, 'Division Name': 'General', 'Department Name': 'Tops', 'Class Name': 'Blouses', 'row': 72, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content='Index: 4 Title: Flattering shirt\n This shirt is very flattering to all due to the adjustable front tie. it is the perfect length to wear with leggings and it is sleeveless so it pairs well with any cardigan. love this shirt!!!\n', metadata={'Clothing ID': 847, 'Age': 47, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 6, 'Division Name': 'General', 'Department Name': 'Tops', 'Class Name': 'Blouses', 'row': 4, 'source': 'Womens Clothing E-Commerce Reviews.csv'}),
 Document(page_content="Index: 83 Title: Just lovely\n I purchased this top in an antro store last week. the quality is wonderful and the greenish blue color is very unique. the blouse has a beautiful stretchy camsiole that is attached at the shoulders, but can be removed by snaps. i tooks great when worn two different ways. i wore it with the campole out, and it peeks through the blouse at the bottom. it also looks great when the camisole is worn in. my only wish is that it would come in a petite size as it's a touch big at the shoulders and neckli\n", metadata={'Clothing ID': 822, 'Age': 68, 'Rating': 5, 'Recommended IND': 1, 'Positive Feedback Count': 3, 'Division Name': 'General', 'Department Name': 'Tops', 'Class Name': 'Blouses', 'row': 83, 'source': 'Womens Clothing E-Commerce Reviews.csv'})]