From 2142a9a1f0db44672697b35ac5a5cb34af7343e9 Mon Sep 17 00:00:00 2001 From: Matthew Farrellee Date: Thu, 30 May 2024 13:44:15 -0400 Subject: [PATCH] add rerank example using duckduckgo, bm25 and faiss --- .../docs/retrievers/nvidia_rerank.ipynb | 457 ++++++++++++++---- 1 file changed, 371 insertions(+), 86 deletions(-) diff --git a/libs/ai-endpoints/docs/retrievers/nvidia_rerank.ipynb b/libs/ai-endpoints/docs/retrievers/nvidia_rerank.ipynb index 0c36ec6a..3164e8e0 100644 --- a/libs/ai-endpoints/docs/retrievers/nvidia_rerank.ipynb +++ b/libs/ai-endpoints/docs/retrievers/nvidia_rerank.ipynb @@ -34,7 +34,9 @@ "\n", "Two important use cases:\n", "- Combining results from multiple data sources\n", - "- Enhancing accuracy for single data sources" + "- Enhancing accuracy for single data sources\n", + "\n", + "We'll demonstrate the former below." ] }, { @@ -46,11 +48,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ - "%pip install --upgrade --quiet langchain-nvidia-ai-endpoints" + "%pip install --upgrade --quiet langchain-nvidia-ai-endpoints" ] }, { @@ -72,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -101,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -134,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -151,31 +161,191 @@ "source": [ "#### BM25 relevant documents\n", "\n", - "Below we assume you have ElasticSearch running with documents stored in a `langchain-index` store." + "Let's create a BM25 index that we can query. We'll use the [`BM25Retriever`](hhttps://python.langchain.com/v0.2/docs/integrations/retrievers/bm25/) and web search results from [DuckDuckGo](https://duckduckgo.com/)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install --upgrade --quiet langchain-community duckduckgo-search beautifulsoup4 rank_bm25" + ] + }, + { + "cell_type": "code", + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "%pip install --upgrade --quiet langchain-community elasticsearch" + "from langchain_community.utilities import DuckDuckGoSearchAPIWrapper\n", + "from langchain_community.retrievers import BM25Retriever\n", + "from langchain.docstore.document import Document\n", + "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", + "import requests\n", + "from bs4 import BeautifulSoup" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "import elasticsearch\n", - "from langchain_community.retrievers import ElasticSearchBM25Retriever\n", - "\n", - "bm25_retriever = ElasticSearchBM25Retriever(\n", - " client=elasticsearch.Elasticsearch(\"http://localhost:9200\"),\n", - " index_name=\"langchain-index\"\n", - ")" + "from typing import List\n", + "\n", + "def build_documents(query, search_util, text_splitter, source) -> List[Document]:\n", + " documents = []\n", + " print(f\"Building documents for {query}\")\n", + " for result in search_util(query):\n", + " print(f\"Processing {result['title']} - {result['link']}\")\n", + " try:\n", + " text = BeautifulSoup(requests.get(result[\"link\"]).text, \"html.parser\").get_text()\n", + " for text in text_splitter.split_text(text):\n", + " documents.append(\n", + " Document(\n", + " page_content=text,\n", + " metadata={\n", + " \"title\": result[\"title\"],\n", + " \"url\": result[\"link\"],\n", + " \"source\": source,\n", + " },\n", + " )\n", + " )\n", + " except Exception as e:\n", + " print(f\"Skipping due to connection error: {e}\")\n", + " print(f\"Done building {len(documents)} documents\")\n", + " return documents " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Building documents for What is the meaning of life?\n", + "Processing Meaning of life - Wikipedia - https://en.wikipedia.org/wiki/Meaning_of_life\n", + "Processing Life | Definition, Origin, Evolution, Diversity, & Facts | Britannica - https://www.britannica.com/science/life\n", + "Processing 4 philosophical answers to the meaning of life - Big Think - https://bigthink.com/thinking/four-philosophical-answers-meaning-of-life/\n", + "Processing 5 Philosophical Answers to the Meaning of Life - WorldAtlas - https://www.worldatlas.com/philosophy/5-philosophical-answers-to-the-meaning-of-life.html\n", + "Processing The Ultimate Meaning of Life | Psychology Today - https://www.psychologytoday.com/us/blog/mindbloggling/202310/the-ultimate-meaning-of-life\n", + "Processing The Meaning of Life: Exploring Different Perspectives and ... - Medium - https://theinstituteofintellect.medium.com/the-meaning-of-life-8ff69decb5c6\n", + "Processing 'What Is the Point of Life?': Why You Might Feel This Way - Verywell Mind - https://www.verywellmind.com/whats-the-point-of-life-why-you-might-feel-this-way-5272182\n", + "Processing Meaning of Life VS. Purpose of Life (the Difference!) - LonerWolf - https://lonerwolf.com/meaning-of-life/\n", + "Processing What Is the Meaning of Life? | Core Christianity - https://corechristianity.com/resources/articles/what-is-the-meaning-of-life-\n", + "Processing Definition, Measures, Applications, & Facts | Britannica - https://www.britannica.com/topic/quality-of-life\n", + "Processing The Meaning Behind The Song: Full of life by Christine and the Queens - https://oldtimemusic.com/w3/the-meaning-behind-the-song-full-of-life-by-christine-and-the-queens/\n", + "Processing What Is Quality of Life? Why It's Important and How to Improve It - https://www.investopedia.com/terms/q/quality-of-life.asp\n", + "Processing \"What is the Meaning of Life?\" in the Works of Gilgamesh and ... - IvyPanda - https://ivypanda.com/essays/what-is-the-meaning-of-life/\n", + "Processing The Meaning Behind The Song: My Ordinary Life by The Living Tombstone - https://oldtimemusic.com/the-meaning-behind-the-song-my-ordinary-life-by-the-living-tombstone-2/\n", + "Processing Term Life Insurance: What It Is, Different Types, Pros and Cons - https://www.investopedia.com/terms/t/termlife.asp\n", + "Processing The Circle of Life: Unlocking Spiritual Meanings - https://spiritualityshepherd.com/circle-of-life-spiritual-meaning/\n", + "Processing Spirituality: Definition, Types, Benefits, and How to Practice - https://www.verywellmind.com/how-spirituality-can-benefit-mental-and-physical-health-3144807\n", + "Processing The Meaning Behind The Song: Life On Mars by David Bowie - https://oldtimemusic.com/the-meaning-behind-the-song-life-on-mars-by-david-bowie/\n", + "Processing What Does Philosophy Say About the Meaning of Life? - Exploring your mind - https://exploringyourmind.com/what-does-philosophy-say-about-the-meaning-of-life/\n", + "Processing Quiz: What is The Meaning of Life? - Trivia & Questions - ProProfs - https://www.proprofs.com/quiz-school/quizzes/meaning-of-life-quiz\n", + "Processing 56 Names that mean 'Life' | Nameberry - https://nameberry.com/baby-names/377/names-that-mean-life\n", + "Processing The Symbolic Significance of the Tree of Life Across Different Cultures ... - https://thewitness.org/what-is-the-meaning-of-the-tree-of-life/\n", + "Processing The Meaning Behind The Song: What Is Life by George Harrison - https://oldtimemusic.com/the-meaning-behind-the-song-what-is-life-by-george-harrison/\n", + "Processing What is Term Life Insurance? - ValuePenguin - https://www.valuepenguin.com/life-insurance/what-is-term-life-insurance\n", + "Processing The Meaning of Life - resilience - https://www.resilience.org/stories/2023-09-15/the-meaning-of-life/\n", + "Processing What Is The Meaning Of Life? (Unveiling the Meaning From Religious ... - https://inspirationfeed.com/what-is-the-meaning-of-life/\n", + "Processing 100 Life Quotes To Transform Your Life Today - Wisdom Quotes - https://wisdomquotes.com/life-quotes/\n", + "Processing The Meaning Behind The Song: Reflections of My Life by Marmalade - https://oldtimemusic.com/the-meaning-behind-the-song-reflections-of-my-life-by-marmalade/\n", + "Processing What does it mean that \"in Him was life, and the life was the light of ... - https://www.gotquestions.org/in-Him-was-life-light-men.html\n", + "Processing Neoteny and the meaning of life. - APA PsycNet - https://psycnet.apa.org/fulltext/2024-14510-001.html\n", + "Processing Detecting the Meaning of Life's Moments | Psychology Today - https://www.psychologytoday.com/us/blog/the-meaningful-life/202403/detecting-the-meaning-of-lifes-moments\n", + "Processing Biology | Definition, History, Concepts, Branches, & Facts - https://www.britannica.com/science/biology\n", + "Processing What's the Meaning of Life: Simplified - Insideout Mastery - https://insideoutmastery.com/meaning-of-life/\n", + "Processing Slice-of-life Definition & Meaning - Merriam-Webster - https://www.merriam-webster.com/dictionary/slice-of-life\n", + "Processing Exploring The Circle Of Life Spiritual Meaning: Origins, Symbolism, And ... - https://innerhunches.com/circle-of-life-spiritual-meaning/\n", + "Processing Defining Life | SpringerLink - https://link.springer.com/chapter/10.1007/978-3-031-23397-5_2\n", + "Processing The Meaning of Life - RTÉ - https://www.rte.ie/tv/programmes/1402684-the-meaning-of-life/\n", + "Processing What is the Definition of Life? (with picture) - AllTheScience - https://www.allthescience.org/what-is-the-definiton-of-life.htm\n", + "Processing The Tree of Life: Understanding its Symbolism and Significance - https://symbolsage.com/tree-of-life-symbol-and-meaning/\n", + "Processing What Is the Meaning of Life? (Answering the Big Question) - UpJourney - https://upjourney.com/what-is-the-meaning-of-life\n", + "Processing Unveiling the Deeper Meaning and True Purpose of Life - https://spiritualitypedia.com/spirituality/unveiling-the-deeper-meaning-and-true-purpose-of-life/\n", + "Processing \"I am the Bread of Life\" What Did Jesus Mean? John 8:35 Verse - https://www.biblestudytools.com/bible-study/topical-studies/what-did-jesus-mean-when-he-said-i-am-the-bread-of-life.html\n", + "Processing What Is Term Insurance? How Does It Work, and What Are the Types? - https://www.investopedia.com/ask/answers/08/term-life-insurance.asp\n", + "Processing The Meaning Behind The Song: That's Life by Frank Sinatra - https://oldtimemusic.com/the-meaning-behind-the-song-thats-life-by-frank-sinatra/\n", + "Processing Self-Esteem: Influences, Traits, and How to Improve It - Verywell Mind - https://www.verywellmind.com/what-is-self-esteem-2795868\n", + "Processing What Is The Meaning Of Spiritual Life? Self-Discovery - https://spiritualdesk.com/what-is-the-meaning-of-spiritual-life/\n", + "Processing Biodiversity | Definition & Facts | Britannica - https://www.britannica.com/science/biodiversity\n", + "Processing What Does the Bible Say about Life - Meaning and Importance - https://www.biblestudytools.com/bible-study/topical-studies/what-the-bible-says-about-life.html\n", + "Processing What To Do When You Can't Stop Wondering, \"What Is The Meaning Of Life ... - https://www.betterhelp.com/advice/midlife-crisis/what-to-do-when-you-cant-stop-wondering-what-is-the-meaning-of-life/\n", + "Processing When a student asked her cosmology prof the meaning of life - Big Think - https://bigthink.com/thinking/cosmogenesis-meaning-of-life/\n", + "Processing Shelf life Definition & Meaning - Merriam-Webster - https://www.merriam-webster.com/dictionary/shelf life\n", + "Processing Meaning of \"Life Is a Highway\" by Rascal Flatts - https://www.songmeaningsandfacts.com/meaning-of-life-is-a-highway-by-rascal-flatts/\n", + "Processing The Philosophy of Ikigai: 3 Examples About Finding Purpose - https://positivepsychology.com/ikigai/\n", + "Processing \"I Am the Way, the Truth, and the Life\" - Meaning Behind Jesus' Words - https://www.crosswalk.com/faith/bible-study/i-am-the-way-the-truth-and-the-life-what-jesus-really-meant.html\n", + "Processing What is the meanning of life? - Answers - https://www.answers.com/Q/What_is_the_meanning_of_life\n", + "Processing The Meaning Behind The Song: Walk of Life by Dire Straits - https://oldtimemusic.com/the-meaning-behind-the-song-walk-of-life-by-dire-straits/\n", + "Processing What Is a Life Estate? - Investopedia - https://www.investopedia.com/terms/l/life-estate.asp\n", + "Processing The Meaning of Life Is Surprisingly Simple (2024) - https://genesyssm.com/article/the-meaning-of-life-is-surprisingly-simple\n", + "Processing Exploring the Meaning of Spiritual Life - Spiritualdefinition - https://www.spiritualdefinition.com/what-is-a-spiritual-life\n", + "Processing The meaning of personal values and how they impact your life - BetterUp - https://www.betterup.com/blog/meaning-of-personal-values\n", + "Processing Cell | Definition, Types, Functions, Diagram, Division, Theory, & Facts - https://www.britannica.com/science/cell-biology\n", + "Processing What is the Meaning of Life? Isaiah 43:7 Devotional - https://mastershandcollection.com/blog/the-meaning-of-life/\n", + "Processing 10 Answers Siri Has for the Meaning of Life - iPhone Life - https://www.iphonelife.com/content/10-answers-siri-has-meaning-life\n", + "Processing Life-support system Definition & Meaning - Merriam-Webster - https://www.merriam-webster.com/dictionary/life-support system\n", + "Processing Purpose Vs Meaning Of Life - Definitions, How To Identify, And Uses - https://themindfool.com/purpose-vs-meaning-of-life/\n", + "Processing All walks of life Definition & Meaning - Merriam-Webster - https://www.merriam-webster.com/dictionary/all walks of life\n", + "Processing What Was The Meaning Of Life For The Ancient Romans - https://www.learnancientrome.com/what-was-the-meaning-of-life-for-the-ancient-romans/\n", + "Processing What does the Star of Life represent? - EMS1 - https://www.ems1.com/ems-advocacy/articles/the-star-of-life-a-shining-symbol-of-what-it-means-to-be-in-ems-kF2FK8KwZKEZhQaj/\n", + "Processing Soul Ending Explained: Pixar's Meaning Of Life Revealed - Screen Rant - https://screenrant.com/soul-ending-explained-meaning-life-purpose-spark/\n", + "Skipping due to connection error: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))\n", + "Processing The Celtic Tree of Life (Crann Bethadh): Meaning + History - https://www.theirishroadtrip.com/celtic-tree-of-life-symbol/\n", + "Processing Life Essay: What is The Meaning of Life - GradesFixer - https://gradesfixer.com/free-essay-examples/life-essay-what-is-the-meaning-of-life/\n", + "Processing 71 Inspiring 'Meaning of Life' Quotes for Purpose and Passion - https://psychicblaze.com/meaning-of-life-quotes/\n", + "Processing The Meaning Behind The Song: It's My Life by Bon Jovi - https://oldtimemusic.com/the-meaning-behind-the-song-its-my-life-by-bon-jovi/\n", + "Processing The Meaning Behind The Song: The Circle of Life by Elton John - https://oldtimemusic.com/the-meaning-behind-the-song-the-circle-of-life-by-elton-john/\n", + "Processing What Is Term Life Insurance? - Forbes Advisor - https://www.forbes.com/advisor/life-insurance/choosing-the-right-term-life-insurance/\n", + "Processing Life, The Universe, And Everything: Why 42 Really Is The Ultimate ... - https://www.iflscience.com/life-the-universe-and-everything-why-42-really-is-the-ultimate-answer-72379\n", + "Processing The Meaning Behind The Song: Life For Rent by Dido - https://oldtimemusic.com/the-meaning-behind-the-song-life-for-rent-by-dido/\n", + "Processing Understanding the Bible Definition of Life: A Spiritual Exploration - https://biblicaldefinitions.com/bible-definition-of-life/\n", + "Processing 25 Bible Verses About The Meaning Of Life (With Commentary) - https://scripturesavvy.com/bible-verses-about-the-meaning-of-life/\n", + "Processing What are Goals? Achieve More By Changing Your Perspectives - https://www.lifehack.org/863723/what-are-goals\n", + "Processing What Is The Definition Of Happiness? The True Meaning Of Joy - https://www.betterhelp.com/advice/happiness/how-do-we-define-happiness-the-true-meaning-of-joy/\n", + "Processing Three Paths Toward the Meaning of Life - The Atlantic - https://www.theatlantic.com/ideas/archive/2023/11/self-knowledge-philosophy-happiness/675868/\n", + "Processing What did Jesus mean when He spoke of living water? - https://www.gotquestions.org/living-water.html\n", + "Processing What is the Meaning of Life? [FREE masterclass] - YouTube - https://www.youtube.com/watch?v=N1Ey6PzL89w\n", + "Processing What Is Marriage? Definition, Purpose, Types, and Importance - https://www.marriage.com/advice/relationship/what-is-marriage/\n", + "Processing Life is Ephemeral | Reason and Meaning - https://reasonandmeaning.com/2023/07/16/life-is-ephemeral/\n", + "Processing What is the Spiritual Meaning of Life? - Spiritualdefinition - https://www.spiritualdefinition.com/what-is-the-spiritual-definition-of-life\n", + "Processing Happiness: What It Really Means and How to Find It - Verywell Mind - https://www.verywellmind.com/what-is-happiness-4869755\n", + "Processing What does it mean that everything is meaningless? - https://www.gotquestions.org/everything-is-meaningless.html\n", + "Processing Why Work-Life Balance Is So Important—and How to Nail It - Verywell Mind - https://www.verywellmind.com/why-work-life-balance-is-so-important-8374683\n", + "Processing The Deep Meaning of the Tree of Life Jewelry | Jewelry Guide - https://www.jewelryshoppingguide.com/tree-of-life-jewelry-meaning/\n", + "Processing What Is Stress? Symptoms, Causes, Treatment, Coping - Verywell Mind - https://www.verywellmind.com/stress-and-health-3145086\n", + "Processing The Flower of Life: Intersecting Circles, Infinite Meanings - https://symbolsage.com/flower-of-life-symbol-meaning/\n", + "Processing The Meaning And Significance Of The Circle Of Life Symbol - https://innerhunches.com/circle-of-life-symbol/\n", + "Processing Life expectancy | Definition & Facts | Britannica - https://www.britannica.com/science/life-expectancy\n", + "Processing The Meaning Behind The Song: In My Life by The Beatles - https://oldtimemusic.com/the-meaning-behind-the-song-in-my-life-by-the-beatles/\n", + "Processing What is the biblical definition of life? - Answer The Bible - https://www.answerthebible.com/what-is-the-biblical-definition-of-life/\n", + "Processing Is there a difference between the purpose of life and the meaning of ... - https://philosophy.stackexchange.com/questions/104691/is-there-a-difference-between-the-purpose-of-life-and-the-meaning-of-life\n", + "Processing Flower of Life Meaning and Symbolism Explained | UniGuide - https://www.uniguide.com/flower-of-life-sacred-geometry\n", + "Processing Whole Life Insurance Definition: How It Works, With Examples - Investopedia - https://www.investopedia.com/terms/w/wholelife.asp\n", + "Done building 2026 documents\n" + ] + } + ], + "source": [ + "bm25_tool = lambda query: DuckDuckGoSearchAPIWrapper().results(query, max_results=100, source=\"text\")\n", + "bm25_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200, length_function=len)\n", + "bm25_retriever = BM25Retriever.from_documents(build_documents(query, bm25_tool, bm25_splitter, \"DuckDuckGo Text\"))" ] }, { @@ -187,11 +357,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(500,\n", + " [Document(page_content='Origin of the question\\nPhilosopher in Meditation (detail) by RembrandtArthur Schopenhauer was the first to explicitly ask the question,[1] in an essay entitled \"Character\".Since a man does not alter, and his moral character remains absolutely the same all through his life; since he must play out the part which he has received, without the least deviation from the character; since neither experience, nor philosophy, nor religion can effect any improvement in him, the question arises, What is the meaning of life at all? To what purpose is it played, this farce in which everything that is essential is irrevocably fixed and determined?[5]Questions about the meaning of life, and similar, have been expressed in a broad variety of other ways, including:\\nWhat is the meaning of life? What\\'s it all about? Who are we?[6][7][8]\\nWhy are we here? What are we here for?[9][10][11]\\nWhat is the origin of life?[12]\\nWhat is the nature of life? What is the nature of reality?[12][13][14]', metadata={'title': 'Meaning of life - Wikipedia', 'url': 'https://en.wikipedia.org/wiki/Meaning_of_life', 'source': 'DuckDuckGo Text'}),\n", + " Document(page_content=\"Why are we here? What are we here for?[9][10][11]\\nWhat is the origin of life?[12]\\nWhat is the nature of life? What is the nature of reality?[12][13][14]\\nWhat is the purpose of life? What is the purpose of one's life?[13][15][16]\\nWhat is the significance of life?[16] (See also #Psychological significance and value in life)\\nWhat is meaningful and valuable in life?[17]\\nWhat is the value of life?[18]\\nWhat is the reason to live? What are we living for?[11][19]\\nThese questions have resulted in a wide range of competing answers and explications, from scientific theories, to philosophical, theological, and spiritual explanations.\", metadata={'title': 'Meaning of life - Wikipedia', 'url': 'https://en.wikipedia.org/wiki/Meaning_of_life', 'source': 'DuckDuckGo Text'}),\n", + " Document(page_content='How do our relationships contribute to the meaning of life?\\xa0\\n\\nRelationships significantly contribute to the meaning of life:\\n• Love and belonging: Relationships provide a sense of love, belonging, and acceptance, which are fundamental human needs.\\n• Growth and learning: Through relationships, we learn about ourselves and others, promoting personal growth.\\n• Support system: In times of trouble, relationships provide emotional support and encouragement.\\n• Shared experiences: Relationships allow us to share experiences, creating a sense of connection and mutual understanding.\\n\\n\\n\\n\\n\\n\\n\\nWhat is the relationship between happiness and the meaning of life?', metadata={'title': 'What Is the Meaning of Life? (Answering the Big Question) - UpJourney', 'url': 'https://upjourney.com/what-is-the-meaning-of-life', 'source': 'DuckDuckGo Text'}),\n", + " Document(page_content='What is the relationship between happiness and the meaning of life?\\xa0\\n\\nWhile happiness and life’s meaning are interrelated, they aren’t the same. Happiness is often a fleeting state of contentment or joy, whereas meaning is a deeper sense of purpose and understanding.\\xa0\\nMany find that pursuing a meaningful life leads to lasting satisfaction, even if the path isn’t always filled with happiness. In fact, the pursuit of meaning can involve challenges and hardships, but the reward is a profound sense of fulfillment.\\xa0\\nThat said, happiness can be a wonderful by-product of living a life aligned with your personal meaning.\\n\\n\\n\\n\\n\\n\\n\\nDo people search for meaning throughout their entire lives?', metadata={'title': 'What Is the Meaning of Life? (Answering the Big Question) - UpJourney', 'url': 'https://upjourney.com/what-is-the-meaning-of-life', 'source': 'DuckDuckGo Text'}),\n", + " Document(page_content='4.\\xa0\\nWhat philosophical concept suggests that life has no inherent meaning?\\n\\n A.\\n\\n\\nExistential nihilism\\xa0\\n\\n\\n B.\\n\\n\\nOptimism\\n\\n\\n C.\\n\\n\\nAltruism\\n\\n\\n D.\\n\\n\\nHumanism\\n\\n\\n\\nCorrect AnswerA. Existential nihilism\\xa0 \\n\\n5.\\xa0\\nIn \"The Hitchhiker\\'s Guide to the Galaxy,\" what is the answer to the ultimate question of life, the universe, and everything?\\n\\n A.\\n\\n\\n42\\n\\n\\n B.\\n\\n\\n21\\n\\n\\n C.\\n\\n\\n10\\n\\n\\n D.\\n\\n\\n100\\n\\n\\n\\nCorrect AnswerA. 42 \\n\\n6.\\xa0\\nAccording to Buddhism, what is the main cause of suffering in life?\\n\\n A.\\n\\n\\nAttachment to material things\\n\\n\\n B.\\n\\n\\nDesire and Ignorance\\n\\n\\n C.\\n\\n\\nLack of discipline\\n\\n\\n D.\\n\\n\\nPhysical pain\\n\\n\\n\\nCorrect AnswerB. Desire and Ignorance \\n\\n7.\\xa0\\nWhat does the philosophy of Absurdism propose regarding the search for the meaning of life?\\n\\n A.\\n\\n\\nTrying to find meaning leads people into a conflict with the world.\\n\\n\\n B.\\n\\n\\nTrying to avoid seeking answers\\n\\n\\n C.\\n\\n\\nTrying to find a higher purpose\\n\\n\\n D.\\n\\n\\nTo accept fate\\n\\n\\n\\nCorrect AnswerA. Trying to find meaning leads people into a conflict with the world.', metadata={'title': 'Quiz: What is The Meaning of Life? - Trivia & Questions - ProProfs', 'url': 'https://www.proprofs.com/quiz-school/quizzes/meaning-of-life-quiz', 'source': 'DuckDuckGo Text'})])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "bm25_docs = bm25_retriever.invoke(query)" + "bm25_retriever.k = 500\n", + "bm25_docs = bm25_retriever.invoke(query)\n", + "len(bm25_docs), bm25_docs[:5]" ] }, { @@ -205,23 +393,124 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], "source": [ - "%pip install --upgrade --quiet langchain-community langchain-nvidia-ai-endpoints faiss-gpu" + "%pip install --upgrade --quiet faiss-gpu" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Building documents for What is the meaning of life?\n", + "Processing 25 Philosophical Quotes on the Meaning of Life - https://www.msn.com/en-us/lifestyle/mind-and-soul/25-philosophical-quotes-on-the-meaning-of-life/ss-AA1nLKXP\n", + "Processing Monty Python's 'The Meaning Of Life' In Cannes: In 1983, The World's Most Serious Film Festival Went For Something Completely Different… - https://www.msn.com/en-us/movies/other/monty-python-s-the-meaning-of-life-in-cannes-in-1983-the-world-s-most-serious-film-festival-went-for-something-completely-different/ar-BB1mHt40\n", + "Processing Religion and the Meaning of Life - https://www.cambridge.org/core/books/religion-and-the-meaning-of-life/2C118CBF40B68F288B9010457F78571E\n", + "Processing Jeremy Fink and the Meaning of Life Streaming: Watch & Stream Online via Amazon Prime Video - https://www.yahoo.com/entertainment/jeremy-fink-meaning-life-streaming-042636701.html\n", + "Processing An Eastern perspective on the meaning of life - https://www.bbc.com/reel/video/p0b373r6/an-eastern-perspective-on-the-meaning-of-life\n", + "Processing The importance of sacrifice in everyday life - https://thecitizen.com/2024/05/27/the-importance-of-sacrifice-in-everyday-life/\n", + "Processing What does 'asexual' mean? Exploring the meaning of the 'A' in LGBTQIA - https://www.msn.com/en-us/news/us/what-does-asexual-mean-exploring-the-meaning-of-the-a-in-lgbtqia/ar-BB1njyCm\n", + "Processing Behind the Meaning of Taylor Swift's Post-Mortem of a Relationship, \"loml\" - https://americansongwriter.com/behind-the-meaning-of-taylor-swifts-post-mortem-of-a-relationship-loml/\n", + "Processing Reflections on the meaning of authenticity while living with MS - https://multiplesclerosisnewstoday.com/news-posts/2024/05/20/reflections-meaning-authenticity-while-living-ms/\n", + "Processing Man's Search For Meaning: The Psychology Of Cancer Patients - https://www.forbes.com/sites/forbeseq/2024/05/24/mans-search-for-meaning-the-psychology-of-cancer-patients/\n", + "Processing Unlock The Power Of Life Lessons: Find Deeper Purpose In Your Career - https://www.forbes.com/sites/kathymillerperkins/2023/11/20/unveiling-purpose-and-meaning-in-your-career-through-life-lessons/\n", + "Processing Cheriton's Lawrence West and the meaning of modesty - https://easternshorepost.com/2024/05/28/cheritons-lawrence-west-and-the-meaning-of-modesty/\n", + "Processing Opinion: Cancer taught me the true meaning of love, power of community - https://dailybruin.com/2024/05/23/opinion-cancer-taught-me-the-true-meaning-of-love-power-of-community\n", + "Processing 'The Village Next to Paradise' Review: A Promising Debut Wonders How We Can Find Meaning in a World of Brutality - https://www.msn.com/en-us/news/other/the-village-next-to-paradise-review-a-promising-debut-wonders-how-we-can-find-meaning-in-a-world-of-brutality/ar-BB1mMcHk\n", + "Processing Who Is the New Billie Eilish Album About? Inside the Meaning of 'HIT ME HARD AND SOFT' - https://www.news-journal.com/arena/parade/who-is-the-new-billie-eilish-album-about-inside-the-meaning-of-hit-me-hard/article_d9fc2df4-4ec0-56f9-bb05-6d3df68c3532.html\n", + "Processing Here where stars are born: the first images at the edge of the universe in ultra high definition - https://www.thevermilion.com/here-where-stars-are-born-the-first-images-at-the-edge-of-the-universe-in-ultra-high-definition/article_3518/\n", + "Processing What is the birthstone for April? A guide to the the gemstone's meaning, symbolism, colors and more. - https://www.usatoday.com/story/life/2023/04/01/april-birthstone-diamond/11521235002/\n", + "Processing Reflecting on the meaning of Memorial Day - https://herald-zeitung.com/opinion/reflecting-on-the-meaning-of-memorial-day/article_44c7ec4e-19ef-11ef-8d0d-273ab4155bf2.html\n", + "Processing Meaning of Life - https://www.sbs.com.au/ondemand/tv-series/meaning-of-life/season-2\n", + "Processing What's the meaning of Memorial Day? Why we honor the holiday - https://www.today.com/life/holidays/memorial-day-meaning-rcna144157\n", + "Processing The Meaning Behind \"It's My Life\" by Talk Talk and Why Their Music Was About Soul, Not Misery - https://americansongwriter.com/the-meaning-behind-its-my-life-by-talk-talk-and-why-their-music-was-about-soul-not-misery/\n", + "Processing Ecclesiastes and the Meaning of Life in the Ancient World - https://www.cambridge.org/core/books/ecclesiastes-and-the-meaning-of-life-in-the-ancient-world/8F9D3CFEF326DF6FD5E7A804C0E0DFFD\n", + "Processing St. Paul explains the meaning of suffering - https://www.catholicnewsagency.com/resource/55695/st-paul-explains-the-meaning-of-suffering\n", + "Processing Definition of Life - https://www.sciencenews.org/archive/definition-life\n", + "Processing Orgies and the Meaning of Life - https://www.avclub.com/film/reviews/orgies-and-the-meaning-of-life-2008\n", + "Processing The Meaning of Life - https://www.avclub.com/film/reviews/the-meaning-of-life-2017\n", + "Processing Scientists have come up with a new meaning of life - and it's pretty mind-blowing - https://www.msn.com/en-gb/money/technology/scientists-have-come-up-with-a-new-meaning-of-life-and-it-s-pretty-mind-blowing/ar-AA1dhxZd\n", + "Processing Psychologist identified reason for our eternal search for meaning of life - https://www.msn.com/en-us/lifestyle/mind-and-soul/psychologist-identified-reason-for-our-eternal-search-for-meaning-of-life/ar-BB1hasw0\n", + "Processing At 70, I faced a crisis about the meaning of life - here's how I overcame it - https://www.msn.com/en-us/health/wellness/at-70-i-faced-a-crisis-about-the-meaning-of-life-here-s-how-i-overcame-it/ar-BB1hHXDF\n", + "Processing The Meaning of Life | Clergy Corner - https://www.msn.com/en-us/news/technology/the-meaning-of-life-clergy-corner/ar-AA1nOGVd\n", + "Processing The Taste of Things Serves the Meaning of Life - https://www.msn.com/en-us/movies/news/the-taste-of-things-serves-the-meaning-of-life/ar-BB1i1JZ5\n", + "Processing REVIEW: 'Death's Game' Explores The Meaning Of Life, Through The Lens Of Death - https://www.msn.com/en-us/lifestyle/parenting/review-death-s-game-explores-the-meaning-of-life-through-the-lens-of-death/ar-AA1lRjpB\n", + "Processing Psychology expert shares her No. 1 tip on how to deprioritize work and live a purposeful life - https://www.msn.com/en-ph/news/other/psychology-expert-shares-her-no-1-tip-on-how-to-deprioritize-work-and-live-a-purposeful-life/ar-BB1niKz8\n", + "Processing How people without 'inner voices' could help reveal the mysteries of consciousness - https://www.msn.com/en-us/lifestyle/other/how-people-without-inner-voices-could-help-reveal-the-mysteries-of-consciousness/ar-BB1nhB9V\n", + "Processing On TV, Jean Smart and Hannah Einbinder Are Often at Odds. In Real Life? They Couldn't Be Closer. - https://www.msn.com/en-us/entertainment/news/on-tv-jean-smart-and-hannah-einbinder-are-often-at-odds-in-real-life-they-couldnt-be-closer/ar-BB1nhkei\n", + "Processing Witness One Woman's Margarita-Fueled Discovery of the Meaning of Fleetwood Mac's Rumors—and Maybe Life Itself - https://www.msn.com/en-us/entertainment/news/witness-one-woman-s-margarita-fueled-discovery-of-the-meaning-of-fleetwood-mac-s-rumors-and-maybe-life-itself/ar-BB1l21U5\n", + "Processing The meaning behind the name of Taylor Swift's new album, The Tortured Poets Department - https://www.msn.com/en-us/music/news/the-meaning-behind-the-name-of-taylor-swift-s-new-album-the-tortured-poets-department/ar-BB1lEt1P\n", + "Processing Meaning of Life - https://www.sbs.com.au/ondemand/tv-series/meaning-of-life\n", + "Processing Faith: Our life is full of choices, even when it seems God isn't listening - https://www.msn.com/en-us/lifestyle/other/faith-our-life-is-full-of-choices-even-when-it-seems-god-isn-t-listening/ar-BB1mJ5Rf\n", + "Processing Numerology Number Meanings: Messages From the Universe - https://www.msn.com/en-us/health/wellness/numerology-number-meanings-messages-from-the-universe/ar-BB1mZiP5\n", + "Processing How to use your fear of death to live your best life, according to a psychology expert - https://www.msn.com/en-us/money/careersandeducation/how-to-use-your-fear-of-death-to-live-your-best-life-according-to-a-psychology-expert/ar-BB1mTolI\n", + "Processing Opinion: Finding meaning and hope on Memorial Day - https://www.courant.com/2024/05/26/opinion-finding-meaning-and-hope-on-memorial-day/\n", + "Processing A Review: Finding Life's Meaning in 'Rewind' - https://www.msn.com/en-ph/entertainment/entertainmentnews/a-review-finding-life-s-meaning-in-rewind/ar-AA1m2W6N\n", + "Processing Demi Lovato shares emotional meaning behind her latest tattoo: 'I am so happy' - https://www.msn.com/en-us/entertainment/entertainment-celebrity/demi-lovato-shares-emotional-meaning-behind-her-latest-tattoo-i-am-so-happy/ar-BB1nhDwk\n", + "Processing Day 2 of jury deliberations in Trump criminal trial - https://us.cnn.com/politics/live-news/trump-hush-money-trial-05-30-24/index.html\n", + "Processing Former TikTok Star Convicted of 1st-Degree Murder for Killing Wife, Her Friend - https://www.msn.com/en-us/news/crime/former-tiktok-star-convicted-of-1st-degree-murder-for-killing-wife-her-friend/ar-BB1nhBwr\n", + "Processing What does the week have in store? Your tarot horoscope reading for May 20 to May 26 - https://www.msn.com/en-us/lifestyle/lifestyle-general/what-does-the-week-have-in-store-your-tarot-horoscope-reading-for-may-20-to-may-26/ar-BB1mDLzt\n", + "Processing How Different is The Boy and the Heron Compared to the Novel \"How Do You Live?\" - https://www.msn.com/en-us/movies/news/how-different-is-the-boy-and-the-heron-compared-to-the-novel-how-do-you-live/ar-BB1mtAJA\n", + "Processing Volunteers planted flowers with a special meaning at Spencer Hospital - https://www.ktiv.com/2024/05/30/volunteers-planted-flowers-with-special-meaning-spencer-hospital/\n", + "Processing What is life insurance? - https://www.msn.com/en-us/money/personalfinance/what-is-life-insurance/ar-BB1njCvH\n", + "Processing The Left's Funny Definition of Fascism - https://townhall.com/columnists/kurtschlichter/2024/05/20/the-lefts-funny-definition-of-fascism-n2639253\n", + "Processing Is There Any Meaning Behind Having A Black Front Door? Here's What We Know - https://www.msn.com/en-us/lifestyle/home-and-garden/is-there-any-meaning-behind-having-a-black-front-door-here-s-what-we-know/ar-BB1mem2s\n", + "Processing Poetry from Daily Life: Why stick to one meaning when you can have two (or more)? - https://www.yahoo.com/lifestyle/poetry-daily-life-why-stick-084008659.html\n", + "Processing These Are the Best Friendship Quotes to Share With Your BFF - https://www.msn.com/en-in/lifestyle/relationships/these-are-the-best-friendship-quotes-to-share-with-your-bff/ar-BB1lZbl6\n", + "Processing What Is Second-Degree Murder? Punishment, Types And Meaning - https://www.forbes.com/advisor/legal/criminal-defense/second-degree-murder/\n", + "Processing Connecticut Sun Keesusk jersey, explained: What is the meaning behind the WNBA team's alternate jerseys? - https://www.msn.com/en-us/entertainment/tv/connecticut-sun-keesusk-jersey-explained-what-is-the-meaning-behind-the-wnba-team-s-alternate-jerseys/ar-BB1moR02\n", + "Processing What is Pride Month? What to know about the annual observance - https://www.today.com/life/holidays/pride-month-meaning-facts-rcna29427\n", + "Processing Types of life insurance - https://www.msn.com/en-us/money/personalfinance/types-of-life-insurance/ar-BB14nKQP\n", + "Processing When Is the Perfect Time to Buy Life Insurance? - https://www.msn.com/en-us/money/personalfinance/when-is-the-perfect-time-to-buy-life-insurance/ar-BB1mz35l\n", + "Processing All About the Angel Number 111 and What It Means for Your Daily Life - https://www.msn.com/en-us/health/wellness/all-about-the-angel-number-111-and-what-it-means-for-your-daily-life/ar-BB1ncXO6\n", + "Processing The Special Meaning Behind Sofia Richie Grainge's Baby Name - https://www.elle.com/uk/life-and-culture/culture/a60922808/sofia-richie-child/\n", + "Processing Five Recent Breakthroughs Into Unraveling Life's Origin - https://www.realclearscience.com/articles/2024/05/03/five_recent_breakthroughs_into_unraveling_lifes_origin_1029233.html#!\n", + "Processing Let's Discuss the Nine of Cups Tarot Card - https://www.cosmopolitan.com/lifestyle/a60949611/nine-of-cups-tarot-card-meaning/\n", + "Processing The Most Important Number In Taylor Swift's Life May Not Be 13 - https://www.bustle.com/life/taylor-swifts-life-path-number-explained\n", + "Processing 10 Memorial Day facts about the history of the holiday - https://www.msn.com/en-us/news/us/10-memorial-day-facts-about-the-history-of-the-holiday/ar-BB1kveBr\n", + "Processing Comforting Spaces: What Makes Us Feel at Home in the World? - https://www.psychologytoday.com/us/blog/live-long-and-prosper/202405/comforting-spaces-what-makes-us-feel-at-home-in-the-world\n", + "Processing The seldom-heard stories of the people who put the fish on your plate - https://www.msn.com/en-xl/news/other/the-seldom-heard-stories-of-the-people-who-put-the-fish-on-your-plate/ar-BB1nkVw2\n", + "Processing Escaping The Prison Of Your Paycheck: How To Find Freedom And Fulfillment - https://www.forbes.com/sites/forbescoachescouncil/2024/05/29/escaping-the-prison-of-your-paycheck-how-to-find-freedom-and-fulfillment/\n", + "Processing Bill Walton's love of the Grateful Dead, explained: How basketball legend became 'Celebrity Deadhead No. 1' - https://www.msn.com/en-us/sports/nba/bill-waltons-love-of-the-grateful-dead-explained-how-basketball-legend-became-celebrity-deadhead-no-1/ar-BB1n93Xc\n", + "Processing Johnny Wactor's Ex-Fiancee Breaks Down Remembering How The Actor Saved A Co-Worker's Life - https://www.msn.com/en-us/tv/celebrity/johnny-wactor-s-ex-fiancee-breaks-down-remembering-how-the-actor-saved-a-co-worker-s-life/ar-BB1njhq9\n", + "Processing The Story and Meaning Behind \"Instant Karma! (We All Shine On),\" John Lennon's Impulsive Post-Beatles Smash - https://americansongwriter.com/the-story-and-meaning-behind-instant-karma-we-all-shine-on-john-lennons-impulsive-post-beatles-smash/\n", + "Processing The Military Played A Part In the Design of the Amish Beard - https://www.msn.com/en-us/news/opinion/the-military-played-a-part-in-the-design-of-the-amish-beard/ar-BB1ncwZK\n", + "Processing Blac Chyna Thanks Boyfriend Derrick Milano for 'Showing Me True Meaning of Love' on 1-Year Anniversary - https://www.msn.com/en-us/tv/celebrity/blac-chyna-thanks-boyfriend-derrick-milano-for-showing-me-true-meaning-of-love-on-1-year-anniversary/ar-BB1nfoLO\n", + "Processing Subconscious messages? Find out the meaning behind your dreams - https://www.msn.com/en-xl/lifestyle/other/subconscious-messages-find-out-the-meaning-behind-your-dreams/ss-AA1j22T7\n", + "Processing Tree of Life secures $1 million in federal funding for K-12 education - https://jewishchronicle.timesofisrael.com/tree-of-life-secures-1-million-in-federal-funding-for-k-12-education/\n", + "Processing This Pentecost, finding silent-yet-sacred deeds of power in everyday life - https://www.msn.com/en-us/news/us/this-pentecost-finding-silent-yet-sacred-deeds-of-power-in-everyday-life/ar-BB1mCg0s\n", + "Processing Memorial Day ceremonies in Euclid take on additional meaning - https://www.msn.com/en-us/news/us/memorial-day-ceremonies-in-euclid-take-on-additional-meaning/ar-BB1n8IHs\n", + "Processing Netflix Co-founder Marc Randolph Shares His Definition Of Success; Here's What He Said - https://www.msn.com/en-in/money/topstories/netflix-co-founder-marc-randolph-shares-his-definition-of-success-here-s-what-he-said/ar-BB1nftuC\n", + "Processing Netflix co-founder Marc Randolph shares his definition of success, prioritises relationships over work - https://www.msn.com/en-in/entertainment/bollywood/netflix-co-founder-marc-randolph-shares-his-definition-of-success-prioritises-relationships-over-work/ar-BB1nfzQx\n", + "Processing What is a branded title? Meaning, types of title brands, and the risks - https://www.msn.com/en-us/autos/news/what-is-a-branded-title-everything-you-need-to-know/ar-AA1hHLi1\n", + "Processing USD Graduates Light the Way Forward at Commencement 2024 - https://www.sandiego.edu/news/detail.php?_focus=92854\n", + "Processing Five Recent Breakthroughs Into Unraveling Life's Origin - https://www.realclearscience.com/articles/2024/05/03/five_recent_breakthroughs_into_unraveling_lifes_origin_1029233.html\n", + "Processing Rachel Aviv's Search for the Meaning of Mental Illness - https://www.psychologytoday.com/za/blog/the-biology-of-human-nature/202405/rachel-avivs-search-for-the-meaning-of-mental-illness\n", + "Processing The Meaning of Catholic marriage - https://www.catholicnewsagency.com/resource/55404/the-meaning-of-catholic-marriage\n", + "Processing The Well-Meaning Parent Who Grew Up Emotionally Neglected - https://www.psychologytoday.com/za/blog/childhood-emotional-neglect/202403/the-well-meaning-parent-who-grew-up-emotionally-neglected\n", + "Processing The Secret to Happiness Is Helping Others - https://time.com/collection/guide-to-happiness/4070299/secret-to-happiness/\n", + "Processing How Everyday Rituals Can Add Meaning to Your Life - https://greatergood.berkeley.edu/article/item/how_everyday_rituals_can_add_meaning_to_your_life\n", + "Done building 591 documents\n" + ] + } + ], "source": [ "from langchain_community.vectorstores import FAISS\n", - "from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings\n", - "\n", - "embedder = NVIDIAEmbeddings()\n", "\n", "# De-serialization relies on loading a pickle file.\n", "# Pickle files can be modified to deliver a malicious payload that\n", @@ -230,96 +519,92 @@ "# else has modified.\n", "allow_dangerous_deserialization=True\n", "\n", - "sem_retriever = FAISS.load_local(\"langchain_index\", embeddings=embeddings\n", - " allow_dangerous_deserialization=allow_dangerous_deserialization).as_retriever()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Return the relevant documents from the query `\"What is the meaning of life?\"` with FAISS semantic store." + "sem_tool = lambda query: DuckDuckGoSearchAPIWrapper().results(query, max_results=100, source=\"news\")\n", + "sem_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100, length_function=len)\n", + "sem_store = FAISS.from_documents(build_documents(query, sem_tool, sem_splitter, \"DuckDuckGo News\"), embedding=NVIDIAEmbeddings(truncate=\"END\"))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "sem_docs = sem_retriever.get_relevant_documents(query)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Combine and rank documents\n", - "\n", - "Let's combine the BM25 as well as semantic search results. The resulting `docs` will be ordered by their relevance to the query by the reranking NIM." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from langchain_nvidia_ai_endpoints import NVIDIARerank\n", - "\n", - "ranker = NVIDIARerank()\n", - "\n", - "all_docs = bm25_docs + sem_docs\n", - "\n", - "docs = ranker.compress_documents(query=query, documents=all_docs)" + "sem_retriever = sem_store.as_retriever(\n", + " search_kwargs = {\"k\": 500},\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Enhancing accuracy for single data sources\n", - "\n", - "Semantic search with vector embeddings is an efficient way to turn a large corpus of documents into a smaller corpus of relevant documents. This is done by trading accuracy for efficiency. Reranking as a tool adds accuracy back into the search by post-processing the smaller corpus of documents. Typically, ranking on the full corpus is too slow for applications." + "Return the relevant documents from the query `\"What is the meaning of life?\"` with FAISS semantic store." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(500,\n", + " [Document(page_content='Definition of Life | Science News\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nSkip to content\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\t\\tSubscribe today\\t\\n\\nEvery print subscription comes with full digital access\\n\\n\\nSubscribe now\\n\\n\\n\\n\\nMenu\\n\\n\\n\\n\\nAll Topics\\nHealth\\nHumans\\nLife\\nEarth\\nPhysics\\nSpace\\n \\n\\nMagazine\\nMenu\\n\\nAll Stories\\nMultimedia\\nReviews\\nCollections\\nCentury of Science\\nFor Educators\\nCoronavirus Outbreak\\nNewsletters\\n\\n\\nAbout\\nFor Students\\nOur Store\\n\\n\\n\\nSIGN IN\\n\\n\\n\\t\\t\\tDonate\\t\\t\\n\\n\\n\\nScience News\\n\\n\\n\\t\\t\\tINDEPENDENT JOURNALISM SINCE 1921\\t\\t\\n\\n\\nSIGN IN\\n\\n\\n\\n\\n\\n\\n\\t\\t\\t\\tSearch\\t\\t\\t\\n\\n\\n\\n\\n\\n\\nOpen search\\n\\n\\nClose search\\n\\n\\n\\n\\n\\n\\n\\nScience News\\n\\n\\n\\t\\t\\tINDEPENDENT JOURNALISM SINCE 1921\\t\\t\\n\\n\\n\\n\\n\\n\\t\\t\\t\\tAll Topics\\t\\t\\t\\n\\n\\n\\n\\t\\tEarth\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tAgriculture\\t\\t\\t\\t\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tClimate\\t\\t\\t\\t\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tOceans\\t\\t\\t\\t\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tEnvironment\\t\\t\\t\\t\\t\\n\\n\\n\\n\\n\\n\\t\\tHumans\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tAnthropology\\t\\t\\t\\t\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tHealth & Medicine\\t\\t\\t\\t\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tArchaeology\\t\\t\\t\\t\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tPsychology\\t\\t\\t\\t\\t\\n\\n\\n\\n\\n\\n\\t\\tLife\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tAnimals\\t\\t\\t\\t\\t\\n\\n\\n\\n\\t\\t\\t\\t\\t\\tPlants', metadata={'title': 'Definition of Life', 'url': 'https://www.sciencenews.org/archive/definition-life', 'source': 'DuckDuckGo News'}),\n", + " Document(page_content=\"Information\\n\\n\\n\\n\\nInformation\\n\\n\\nContents\\n\\n\\nMetrics\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\nBook description \\n\\n\\n\\n\\n\\nAs humans, we want to live meaningfully, yet we are often driven by impulse. In Religion and the Meaning of Life, Williams investigates this paradox – one with profound implications. Delving into felt realities pertinent to meaning, such as boredom, trauma, suicide, denial of death, and indifference, Williams describes ways to acquire meaning and potential obstacles to its acquisition. This book is unique in its willingness to transcend a more secular stance and explore how one's belief in God may be relevant to life's meaning. Religion and the Meaning of Life's interdisciplinary approach makes it useful to philosophers, religious studies scholars, psychologists, students, and general readers alike. The insights from this book have profound real-world applications – they can transform how readers search for meaning and, consequently, how readers see and exist in the world.\", metadata={'title': 'Religion and the Meaning of Life', 'url': 'https://www.cambridge.org/core/books/religion-and-the-meaning-of-life/2C118CBF40B68F288B9010457F78571E', 'source': 'DuckDuckGo News'}),\n", + " Document(page_content='An Eastern perspective on the meaning of lifeBritish Broadcasting CorporationWatchRegisterSign InHomeNewsSportBusinessInnovationCultureTravelEarthVideoLiveHomeNewsIsrael-Gaza WarWar in UkraineUK General ElectionIndia ElectionUS & CanadaUKUK PoliticsEnglandN. IrelandN. Ireland PoliticsScotlandScotland PoliticsWalesWales PoliticsAfricaAsiaChinaIndiaAustraliaEuropeLatin AmericaMiddle EastIn PicturesBBC VerifySportBusinessFuture of BusinessTechnology of BusinessWork CultureInnovationTechnologyScience & HealthArtificial IntelligenceCultureFilm & TVMusicArt & DesignStyleBooksEntertainment NewsTravelDestinationsAfricaAntarcticaAsiaAustralia and PacificCaribbean & BermudaCentral AmericaEuropeMiddle EastNorth AmericaSouth AmericaWorld’s TableCulture & ExperiencesAdventuresThe SpeciaListEarthNatural WondersWeather & ScienceClimate SolutionsSustainable BusinessGreen LivingVideoLiveLive NewsLive SportHomeNewsSportBusinessInnovationCultureTravelEarthVideoLiveAudioWeatherAn Eastern perspective on', metadata={'title': 'An Eastern perspective on the meaning of life', 'url': 'https://www.bbc.com/reel/video/p0b373r6/an-eastern-perspective-on-the-meaning-of-life', 'source': 'DuckDuckGo News'}),\n", + " Document(page_content='Religion and the Meaning of Life\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nSkip to main content\\n\\n\\nAccessibility help\\n\\n\\n\\n\\nWe use cookies to distinguish you from other users and to provide you with a better experience on our websites. Close this message to accept cookies or find out how to manage your cookie settings.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nLogin Alert\\n\\n\\n\\n\\n\\n\\n\\n\\nCancel\\n\\nLog in\\n\\n\\n×\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n×\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nHome\\n\\n\\n\\nLog in\\nRegister\\nBrowse subjects\\nPublications\\nOpen research\\nServices\\nAbout Cambridge Core\\n\\n\\nCart\\n\\n\\n\\n\\n\\n\\n\\nCart\\n\\n\\n\\n\\n\\n\\n\\nInstitution login\\n\\n\\nRegister\\nLog in\\n\\n\\n\\nCart\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\nHome\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nHome\\n\\n\\n\\nLog in\\nRegister\\nBrowse subjects\\nPublications\\nOpen research\\nServices\\nAbout Cambridge Core\\n\\n\\nCart\\n\\n\\n\\n\\n\\n\\n\\nCart\\n\\n\\n\\n\\n\\n\\n\\nInstitution login\\n\\n\\nRegister\\nLog in\\n\\n\\n\\nCart\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n< Back to search results', metadata={'title': 'Religion and the Meaning of Life', 'url': 'https://www.cambridge.org/core/books/religion-and-the-meaning-of-life/2C118CBF40B68F288B9010457F78571E', 'source': 'DuckDuckGo News'}),\n", + " Document(page_content='a Relationship Last: 5 Secrets Backed by ResearchDoes Spirituality Make You Happy?The Science of CryingScience Says Your Pet Is Good for Your Mental HealthOrganization Expert Marie Kondo Declutters Her Brain Like She Declutters Her ClosetsHow to Stay MarriedHow Your iPhone Photos Make You HappierGet Happy in LifeGet Happy at WorkGet Happy at HomeHappiness GuideFull ListGet Happy in LifeGet Happy at WorkGet Happy at HomeIlka & Franz—Getty ImagesThe Secret to Happiness Is Helping OthersByJenny SantiThere is a Chinese saying that goes: “If you want happiness for an hour, take a nap. If you want happiness for a day, go fishing. If you want happiness for a year, inherit a fortune. If you want happiness for a lifetime, help somebody.” For centuries, the greatest thinkers have suggested the same thing: Happiness is found in helping others.For it is in giving that we receive — Saint Francis of AssisiThe sole meaning of life is to serve humanity — Leo TolstoyWe make a living by what we get; we', metadata={'title': 'The Secret to Happiness Is Helping Others', 'url': 'https://time.com/collection/guide-to-happiness/4070299/secret-to-happiness/', 'source': 'DuckDuckGo News'})])" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "%pip install --upgrade --quiet langchain langchain-nvidia-ai-endpoints pgvector psycopg langchain-postgres" + "sem_docs = sem_retriever.invoke(query)\n", + "len(sem_docs), sem_docs[:5]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Below we assume you have Postgresql running with documents stored in a collection named `langchain-index`.\n", + "#### Combine and rank documents\n", "\n", - "We will narrow the collection to 1,000 results and further narrow it to 10 with the reranker." + "Let's combine the BM25 as well as semantic search results. The resulting `docs` will be ordered by their relevance to the query by the reranking NIM." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(page_content='The meaning of life can be derived from philosophical and religious contemplation of, and scientific inquiries about, existence, social ties, consciousness, and happiness. Many other issues are also involved, such as symbolic meaning, ontology, value, purpose, ethics, good and evil, free will, the existence of one or multiple gods, conceptions of God, the soul, and the afterlife. Scientific contributions focus primarily on describing related empirical facts about the universe, exploring the context and parameters concerning the \"how\" of life. Science also studies and can provide recommendations for the pursuit of well-being and a related conception of morality. An alternative, humanistic approach poses the question, \"What is the meaning of my life?\"', metadata={'title': 'Meaning of life - Wikipedia', 'url': 'https://en.wikipedia.org/wiki/Meaning_of_life', 'source': 'DuckDuckGo Text', 'relevance_score': 3.875}),\n", + " Document(page_content='What is the meaning of life? This is a question many people ask themselves at some point in their lives. While some people turn to religion to answer the question, others have turned to philosophy. Throughout history, this question has occupied the minds of many philosophers, and different schools of philosophy have come up with various answers to the question. These five different philosophical answers and approaches to the question of the meaning of life showcase the vast diversity of human thought. \\n1. Existentialism - There is No Meaning \\nHuman beings can decide their own life and meaning.', metadata={'title': '5 Philosophical Answers to the Meaning of Life - WorldAtlas', 'url': 'https://www.worldatlas.com/philosophy/5-philosophical-answers-to-the-meaning-of-life.html', 'source': 'DuckDuckGo Text', 'relevance_score': 3.287109375}),\n", + " Document(page_content=\"One way to approach the question of the meaning of life is to consider the various perspectives people have on this topic. For some, the meaning of life is intricately linked to religious beliefs and the idea of a higher power or divine purpose. In this view, life is regarded as a sacred gift meant to be lived in accordance with the will of a higher being. On the other hand, some believe the meaning of life is more secular, focusing on personal fulfillment, happiness, and the pursuit of one's passions and goals.\\nAnother way to think about the meaning of life is through the concept of existentialism, which explores the idea that life has no inherent meaning or purpose. According to existentialist philosophy, it is up to each individual to create their own meaning in life through their actions, choices, and experiences. This perspective can be empowering, as it places the responsibility for finding meaning squarely on the individual, rather than on external forces or beliefs.\", metadata={'title': 'Life Essay: What is The Meaning of Life - GradesFixer', 'url': 'https://gradesfixer.com/free-essay-examples/life-essay-what-is-the-meaning-of-life/', 'source': 'DuckDuckGo Text', 'relevance_score': 1.7998046875}),\n", + " Document(page_content='the meaning of life. In many religious traditions, life is seen as a gift from a divine being, and the purpose of human existence is to seek a higher spiritual connection or attain enlightenment. For example, in Hinduism, the concept of Dharma guides individuals to fulfill their moral duties and achieve spiritual liberation. Similarly, in Christianity, followers believe that the meaning of life lies in a personal relationship with God and following the teachings of Jesus Christ.Philosophical theories from different eras also contribute to our understanding of life’s purpose. Aristotle’s theory of eudaimonia suggests that the ultimate goal of human life is to achieve a state of flourishing and human excellence. According to Aristotle, this can be attained through the cultivation of virtues and the pursuit of intellectual, moral, and physical development. In contrast, utilitarian philosophers such as Jeremy Bentham and John Stuart Mill argued that the purpose of life is the maximization', metadata={'title': 'The Meaning of Life: Exploring Different Perspectives and ... - Medium', 'url': 'https://theinstituteofintellect.medium.com/the-meaning-of-life-8ff69decb5c6', 'source': 'DuckDuckGo Text', 'relevance_score': 1.5576171875}),\n", + " Document(page_content='From a scientific perspective, the meaning of life can be seen as a product of evolution and the natural processes that have shaped our existence. In this view, life is a result of the complex interactions of biological, chemical, and physical forces that have led to the development of living organisms on Earth. The meaning of life, then, can be seen as a reflection of our evolutionary history and the ways in which we have adapted to survive and thrive in our environment.\\nUltimately, the question of the meaning of life is deeply personal and influenced by a wide range of factors, including our beliefs, experiences, and values. While there may not be a definitive answer to this question, it is a topic that invites us to reflect on our own lives and the ways in which we find meaning and purpose in our existence.', metadata={'title': 'Life Essay: What is The Meaning of Life - GradesFixer', 'url': 'https://gradesfixer.com/free-essay-examples/life-essay-what-is-the-meaning-of-life/', 'source': 'DuckDuckGo Text', 'relevance_score': 1.4853515625})]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from langchain_nvidia_ai_endpoints import NVIDIAEmbeddings\n", - "from langchain.vectorstores.pgvector import PGVector\n", - "\n", - "ranker = NVIDIARerank(top_n=10)\n", - "embedder = NVIDIAEmbeddings()\n", - "\n", - "store = PGVector(embeddings=embedder,\n", - " collection_name=\"langchain-index\",\n", - " connection=\"postgresql+psycopg://langchain:langchain@localhost:6024/langchain\")\n", + "ranker = NVIDIARerank()\n", "\n", - "subset_docs = store.similarity_search(query, k=1_000)\n", + "all_docs = bm25_docs + sem_docs\n", "\n", - "docs = ranker.compress_documents(query=query, documents=subset_docs)" + "ranker.top_n = 5\n", + "docs = ranker.compress_documents(query=query, documents=all_docs)\n", + "docs" ] } ], @@ -339,7 +624,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.10.14" } }, "nbformat": 4,