From ee9abb6722c3a3438cb5313624f2bd7ba54a56e7 Mon Sep 17 00:00:00 2001 From: deedy5 <65482418+deedy5@users.noreply.github.com> Date: Tue, 5 Dec 2023 00:48:20 +0000 Subject: [PATCH] Bugfix duckduckgo_search news search (#13670) - **Description:** Bugfix duckduckgo_search news search - **Issue:** https://github.com/langchain-ai/langchain/issues/13648 - **Dependencies:** None - **Tag maintainer:** @baskaryan --------- Co-authored-by: Harrison Chase --- docs/docs/integrations/tools/ddg.ipynb | 52 ++++---- .../langchain/tools/ddg_search/tool.py | 6 +- .../langchain/utilities/duckduckgo_search.py | 117 ++++++++++-------- .../utilities/test_duckduckdgo_search_api.py | 13 +- 4 files changed, 111 insertions(+), 77 deletions(-) diff --git a/docs/docs/integrations/tools/ddg.ipynb b/docs/docs/integrations/tools/ddg.ipynb index 2f83586ff94c0..2d8daefcfa416 100644 --- a/docs/docs/integrations/tools/ddg.ipynb +++ b/docs/docs/integrations/tools/ddg.ipynb @@ -17,12 +17,12 @@ "metadata": {}, "outputs": [], "source": [ - "# !pip install duckduckgo-search" + "# !pip install -U duckduckgo-search" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 1, "id": "ac4910f8", "metadata": {}, "outputs": [], @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 2, "id": "84b8f773", "metadata": {}, "outputs": [], @@ -42,17 +42,17 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 3, "id": "068991a6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'August 4, 1961 (age 61) Honolulu Hawaii Title / Office: presidency of the United States of America (2009-2017), United States United States Senate (2005-2008), United States ... (Show more) Political Affiliation: Democratic Party Awards And Honors: Barack Hussein Obama II (/ b ə ˈ r ɑː k h uː ˈ s eɪ n oʊ ˈ b ɑː m ə / bə-RAHK hoo-SAYN oh-BAH-mə; born August 4, 1961) is an American politician who served as the 44th president of the United States from 2009 to 2017. A member of the Democratic Party, he was the first African-American president of the United States. Obama previously served as a U.S. senator representing Illinois ... Answer (1 of 12): I see others have answered President Obama\\'s name which is \"Barack Hussein Obama\". President Obama has received many comments about his name from the racists across US. It is worth noting that he never changed his name. Also, it is worth noting that a simple search would have re... What is Barack Obama\\'s full name? Updated: 11/11/2022 Wiki User ∙ 6y ago Study now See answer (1) Best Answer Copy His full, birth name is Barack Hussein Obama, II. He was named after his... Alex Oliveira July 24, 2023 4:57pm Updated 0 seconds of 43 secondsVolume 0% 00:00 00:43 The man who drowned while paddleboarding on a pond outside the Obamas\\' Martha\\'s Vineyard estate has been...'" + "\"Life After the Presidency How Tall is Obama? Books and Grammy Hobbies Movies About Obama Quotes 1961-present Who Is Barack Obama? Barack Obama was the 44 th president of the United States... facts you never knew about Barack Obama is that his immediate family spread out across three continents. Barack, who led America from 2009 to 2017, comes from a large family of seven living half-siblings. His father, Barack Obama Sr., met his mother, Ann Dunham, in 1960 and married her a year after. With a tear running from his eye, President Barack Obama recalls the 20 first-graders killed in 2012 at Sandy Hook Elementary School, while speaking in the East Room of the White House in ... Former first Lady Rosalynn Carter was laid to rest at her family's home in Plains, Ga. on Nov. 29 following three days of memorials across her home state. She passed away on Nov. 19, aged 96 ... Here are 28 of President Obama's biggest accomplishments as President of the United States. 1 - Rescued the country from the Great Recession, cutting the unemployment rate from 10% to 4.7% over ...\"" ] }, - "execution_count": 22, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 4, "id": "95635444", "metadata": {}, "outputs": [], @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 5, "id": "0133d103", "metadata": {}, "outputs": [], @@ -91,17 +91,17 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 7, "id": "439efc06", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "\"[snippet: Barack Hussein Obama II (/ b ə ˈ r ɑː k h uː ˈ s eɪ n oʊ ˈ b ɑː m ə / bə-RAHK hoo-SAYN oh-BAH-mə; born August 4, 1961) is an American politician who served as the 44th president of the United States from 2009 to 2017. A member of the Democratic Party, he was the first African-American president of the United States. Obama previously served as a U.S. senator representing Illinois ..., title: Barack Obama - Wikipedia, link: https://en.wikipedia.org/wiki/Barack_Obama], [snippet: Barack Obama, in full Barack Hussein Obama II, (born August 4, 1961, Honolulu, Hawaii, U.S.), 44th president of the United States (2009-17) and the first African American to hold the office. Before winning the presidency, Obama represented Illinois in the U.S. Senate (2005-08). He was the third African American to be elected to that body ..., title: Barack Obama | Biography, Parents, Education, Presidency, Books ..., link: https://www.britannica.com/biography/Barack-Obama], [snippet: Barack Obama 's tenure as the 44th president of the United States began with his first inauguration on January 20, 2009, and ended on January 20, 2017. A Democrat from Illinois, Obama took office following a decisive victory over Republican nominee John McCain in the 2008 presidential election. Four years later, in the 2012 presidential ..., title: Presidency of Barack Obama - Wikipedia, link: https://en.wikipedia.org/wiki/Presidency_of_Barack_Obama], [snippet: First published on Mon 24 Jul 2023 20.03 EDT. Barack Obama's personal chef died while paddleboarding near the ex-president's home on Martha's Vineyard over the weekend, Massachusetts state ..., title: Obama's personal chef dies while paddleboarding off Martha's Vineyard ..., link: https://www.theguardian.com/us-news/2023/jul/24/tafari-campbell-barack-obama-chef-drowns-marthas-vineyard]\"" + "'[snippet: 1:12. Former President Barack Obama, in a CNN interview that aired Thursday night, said he does not believe President Joe Biden will face a serious primary challenge during his 2024 reelection ..., title: Five takeaways from Barack Obama\\'s CNN interview on Biden ... - Yahoo, link: https://www.usatoday.com/story/news/politics/2023/06/23/five-takeaways-from-barack-obama-cnn-interview/70349112007/], [snippet: Democratic institutions in the United States and around the world have grown \"creaky,\" former President Barack Obama warned in an exclusive CNN interview Thursday, and it remains incumbent on ..., title: Obama warns democratic institutions are \\'creaky\\' but Trump ... - CNN, link: https://www.cnn.com/2023/06/22/politics/barack-obama-interview-cnntv/index.html], [snippet: Barack Obama was the 44 th president of the United States and the first Black commander-in-chief. He served two terms, from 2009 until 2017. The son of parents from Kenya and Kansas, Obama was ..., title: Barack Obama: Biography, 44th U.S. President, Politician, link: https://www.biography.com/political-figures/barack-obama], [snippet: Aug. 2, 2023, 5:00 PM PDT. By Mike Memoli and Kristen Welker. WASHINGTON — During a trip to the White House in June, former President Barack Obama made it clear to his former running mate that ..., title: Obama privately told Biden he would do whatever it takes to help in 2024, link: https://www.nbcnews.com/politics/white-house/obama-privately-told-biden-whatever-takes-help-2024-rcna97865], [snippet: Natalie Bookey-Baker, a vice president at the Obama Foundation who worked for then-first lady Michelle Obama in the White House, said about 2,500 alumni are expected. They are veterans of Obama ..., title: Barack Obama team reunion this week in Chicago; 2,500 alumni expected ..., link: https://chicago.suntimes.com/politics/2023/10/29/23937504/barack-obama-michelle-obama-david-axelrod-pod-save-america-jon-batiste-jen-psaki-reunion-obamaworld]'" ] }, - "execution_count": 25, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -120,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 8, "id": "21afe28d", "metadata": {}, "outputs": [], @@ -130,17 +130,17 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 9, "id": "2a4beeb9", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "\"[date: 2023-07-26T12:01:22, title: 'My heart is broken': Former Obama White House chef mourned following apparent drowning death in Edgartown, snippet: Tafari Campbell of Dumfries, Va., had been paddle boarding in Edgartown Great Pond when he appeared to briefly struggle, submerged, and did not return to the surface, authorities have said. Crews ultimately found the 45-year-old's body Monday morning., source: The Boston Globe on MSN.com, link: https://www.msn.com/en-us/news/us/my-heart-is-broken-former-obama-white-house-chef-mourned-following-apparent-drowning-death-in-edgartown/ar-AA1elNB8], [date: 2023-07-25T18:44:00, title: Obama's chef drowns paddleboarding near former president's Edgartown vacation home, snippet: Campbell was visiting Martha's Vineyard, where the Obamas own a vacation home. He was not wearing a lifejacket when he fell off his paddleboard., source: YAHOO!News, link: https://news.yahoo.com/obama-chef-drowns-paddleboarding-near-184437491.html], [date: 2023-07-26T00:30:00, title: Obama's personal chef dies while paddleboarding off Martha's Vineyard, snippet: Tafari Campbell, who worked at the White House during Obama's presidency, was visiting the island while the family was away, source: The Guardian, link: https://www.theguardian.com/us-news/2023/jul/24/tafari-campbell-barack-obama-chef-drowns-marthas-vineyard], [date: 2023-07-24T21:54:00, title: Obama's chef ID'd as paddleboarder who drowned near former president's Martha's Vineyard estate, snippet: Former President Barack Obama's personal chef, Tafari Campbell, has been identified as the paddle boarder who drowned near the Obamas' Martha's Vineyard estate., source: Fox News, link: https://www.foxnews.com/politics/obamas-chef-idd-paddleboarder-who-drowned-near-former-presidents-marthas-vineyard-estate]\"" + "'[snippet: 1:12. Former President Barack Obama, in a CNN interview that aired Thursday night, said he does not believe President Joe Biden will face a serious primary challenge during his 2024 reelection ..., title: Five takeaways from Barack Obama\\'s CNN interview on Biden ... - Yahoo, link: https://www.usatoday.com/story/news/politics/2023/06/23/five-takeaways-from-barack-obama-cnn-interview/70349112007/], [snippet: Democratic institutions in the United States and around the world have grown \"creaky,\" former President Barack Obama warned in an exclusive CNN interview Thursday, and it remains incumbent on ..., title: Obama warns democratic institutions are \\'creaky\\' but Trump ... - CNN, link: https://www.cnn.com/2023/06/22/politics/barack-obama-interview-cnntv/index.html], [snippet: Barack Obama was the 44 th president of the United States and the first Black commander-in-chief. He served two terms, from 2009 until 2017. The son of parents from Kenya and Kansas, Obama was ..., title: Barack Obama: Biography, 44th U.S. President, Politician, link: https://www.biography.com/political-figures/barack-obama], [snippet: Natalie Bookey-Baker, a vice president at the Obama Foundation who worked for then-first lady Michelle Obama in the White House, said about 2,500 alumni are expected. They are veterans of Obama ..., title: Barack Obama team reunion this week in Chicago; 2,500 alumni expected ..., link: https://chicago.suntimes.com/politics/2023/10/29/23937504/barack-obama-michelle-obama-david-axelrod-pod-save-america-jon-batiste-jen-psaki-reunion-obamaworld], [snippet: Aug. 2, 2023, 5:00 PM PDT. By Mike Memoli and Kristen Welker. WASHINGTON — During a trip to the White House in June, former President Barack Obama made it clear to his former running mate that ..., title: Obama privately told Biden he would do whatever it takes to help in 2024, link: https://www.nbcnews.com/politics/white-house/obama-privately-told-biden-whatever-takes-help-2024-rcna97865]'" ] }, - "execution_count": 27, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 10, "id": "c7ab3b55", "metadata": {}, "outputs": [], @@ -171,27 +171,27 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 11, "id": "adce16e1", "metadata": {}, "outputs": [], "source": [ - "search = DuckDuckGoSearchResults(api_wrapper=wrapper, backend=\"news\")" + "search = DuckDuckGoSearchResults(api_wrapper=wrapper, source=\"news\")" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 12, "id": "b7e77c54", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'[date: 2023-07-25T12:15:00, title: Barack + Michelle Obama: Sie trauern um Angestellten, snippet: Barack und Michelle Obama trauern um ihren ehemaligen Küchenchef Tafari Campbell. Der Familienvater verunglückte am vergangenen Sonntag und wurde in einem Teich geborgen., source: Gala, link: https://www.gala.de/stars/news/barack---michelle-obama--sie-trauern-um-angestellten-23871228.html], [date: 2023-07-25T10:30:00, title: Barack Obama: Sein Koch (†45) ist tot - diese Details sind bekannt, snippet: Tafari Campbell war früher im Weißen Haus eingestellt, arbeitete anschließend weiter für Ex-Präsident Barack Obama. Nun ist er gestorben. Diese Details sind bekannt., source: T-Online, link: https://www.t-online.de/unterhaltung/stars/id_100213226/barack-obama-sein-koch-45-ist-tot-diese-details-sind-bekannt.html], [date: 2023-07-25T05:33:23, title: Barack Obama: Sein Privatkoch ist bei einem tragischen Unfall gestorben, snippet: Barack Obama (61) und Michelle Obama (59) sind in tiefer Trauer. Ihr Privatkoch Tafari Campbell ist am Montag (24. Juli) ums Leben gekommen, er wurde nur 45 Jahre alt. Laut US-Polizei starb er bei ein, source: BUNTE.de, link: https://www.msn.com/de-de/unterhaltung/other/barack-obama-sein-privatkoch-ist-bei-einem-tragischen-unfall-gestorben/ar-AA1ejrAd], [date: 2023-07-25T02:25:00, title: Barack Obama: Privatkoch tot in See gefunden, snippet: Tafari Campbell kochte für Barack Obama im Weißen Haus - und auch privat nach dessen Abschied aus dem Präsidentenamt. Nun machte die Polizei in einem Gewässer eine traurige Entdeckung., source: SPIEGEL, link: https://www.spiegel.de/panorama/justiz/barack-obama-leibkoch-tot-in-see-gefunden-a-3cdf6377-bee0-43f1-a200-a285742f9ffc]'" + "'[snippet: When Obama left office in January 2017, a CNN poll showed him with a 60% approval rating, landing him near the top of the list of presidential approval ratings upon leaving office., title: Opinion: The real reason Trump is attacking Obamacare | CNN, link: https://www.cnn.com/2023/12/04/opinions/trump-obamacare-obama-repeal-health-care-obeidallah/index.html], [snippet: Buchempfehlung von Barack Obama. Der gut zweistündige Netflix-Film basiert auf dem gleichnamigen Roman \"Leave the World Behind\" des hochgelobten US-Autors Rumaan Alam. 2020 landete er damit unter den Finalisten des \"National Book Awards\". In Deutschland ist das Buch, das auch Barack Obama auf seiner einflussreichen Lese-Empfehlungsliste hatte ..., title: Neu bei Netflix \"Leave The World Behind\": Kritik zum ... - Prisma, link: https://www.prisma.de/news/filme/Neu-bei-Netflix-Leave-The-World-Behind-Kritik-zum-ungewoehnlichen-Endzeit-Film-mit-Julia-Roberts,46563944]'" ] }, - "execution_count": 30, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -199,6 +199,14 @@ "source": [ "search.run(\"Obama\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b133e3c1", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -217,7 +225,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.10.1" }, "vscode": { "interpreter": { diff --git a/libs/langchain/langchain/tools/ddg_search/tool.py b/libs/langchain/langchain/tools/ddg_search/tool.py index 93b5b8d08dc96..32ac8feae466a 100644 --- a/libs/langchain/langchain/tools/ddg_search/tool.py +++ b/libs/langchain/langchain/tools/ddg_search/tool.py @@ -46,11 +46,11 @@ class DuckDuckGoSearchResults(BaseTool): "Useful for when you need to answer questions about current events. " "Input should be a search query. Output is a JSON array of the query results" ) - num_results: int = 4 + max_results: int = Field(alias="num_results", default=4) api_wrapper: DuckDuckGoSearchAPIWrapper = Field( default_factory=DuckDuckGoSearchAPIWrapper ) - backend: str = "api" + backend: str = "text" args_schema: Type[BaseModel] = DDGInput def _run( @@ -59,7 +59,7 @@ def _run( run_manager: Optional[CallbackManagerForToolRun] = None, ) -> str: """Use the tool.""" - res = self.api_wrapper.results(query, self.num_results, backend=self.backend) + res = self.api_wrapper.results(query, self.max_results, source=self.backend) res_strs = [", ".join([f"{k}: {v}" for k, v in d.items()]) for d in res] return ", ".join([f"[{rs}]" for rs in res_strs]) diff --git a/libs/langchain/langchain/utilities/duckduckgo_search.py b/libs/langchain/langchain/utilities/duckduckgo_search.py index 67e89875ceca3..d258726896ab3 100644 --- a/libs/langchain/langchain/utilities/duckduckgo_search.py +++ b/libs/langchain/langchain/utilities/duckduckgo_search.py @@ -18,6 +18,8 @@ class DuckDuckGoSearchAPIWrapper(BaseModel): safesearch: str = "moderate" time: Optional[str] = "y" max_results: int = 5 + backend: str = "api" # which backend to use in DDGS.text() (api, html, lite) + source: str = "text" # which function to use in DDGS (DDGS.text() or DDGS.news()) class Config: """Configuration for this pydantic object.""" @@ -32,43 +34,69 @@ def validate_environment(cls, values: Dict) -> Dict: except ImportError: raise ImportError( "Could not import duckduckgo-search python package. " - "Please install it with `pip install duckduckgo-search`." + "Please install it with `pip install -U duckduckgo-search`." ) return values - def get_snippets(self, query: str) -> List[str]: - """Run query through DuckDuckGo and return concatenated results.""" + def _ddgs_text( + self, query: str, max_results: Optional[int] = None + ) -> List[Dict[str, str]]: + """Run query through DuckDuckGo text search and return results.""" + from duckduckgo_search import DDGS + + with DDGS() as ddgs: + ddgs_gen = ddgs.text( + query, + region=self.region, + safesearch=self.safesearch, + timelimit=self.time, + max_results=max_results or self.max_results, + backend=self.backend, + ) + if ddgs_gen: + return [r for r in ddgs_gen] + return [] + + def _ddgs_news( + self, query: str, max_results: Optional[int] = None + ) -> List[Dict[str, str]]: + """Run query through DuckDuckGo news search and return results.""" from duckduckgo_search import DDGS with DDGS() as ddgs: - results = ddgs.text( + ddgs_gen = ddgs.news( query, region=self.region, safesearch=self.safesearch, timelimit=self.time, + max_results=max_results or self.max_results, ) - if results is None: - return ["No good DuckDuckGo Search Result was found"] - snippets = [] - for i, res in enumerate(results, 1): - if res is not None: - snippets.append(res["body"]) - if len(snippets) == self.max_results: - break - return snippets + if ddgs_gen: + return [r for r in ddgs_gen] + return [] def run(self, query: str) -> str: - snippets = self.get_snippets(query) - return " ".join(snippets) + """Run query through DuckDuckGo and return concatenated results.""" + if self.source == "text": + results = self._ddgs_text(query) + elif self.source == "news": + results = self._ddgs_news(query) + else: + results = [] + + if not results: + return "No good DuckDuckGo Search Result was found" + return " ".join(r["body"] for r in results) def results( - self, query: str, num_results: int, backend: str = "api" + self, query: str, max_results: int, source: Optional[str] = None ) -> List[Dict[str, str]]: """Run query through DuckDuckGo and return metadata. Args: query: The query to search for. - num_results: The number of results to return. + max_results: The number of results to return. + source: The source to look from. Returns: A list of dictionaries with the following keys: @@ -76,38 +104,27 @@ def results( title - The title of the result. link - The link to the result. """ - from duckduckgo_search import DDGS - - with DDGS() as ddgs: - results = ddgs.text( - query, - region=self.region, - safesearch=self.safesearch, - timelimit=self.time, - backend=backend, - ) - if results is None: - return [{"Result": "No good DuckDuckGo Search Result was found"}] - - def to_metadata(result: Dict) -> Dict[str, str]: - if backend == "news": - return { - "date": result["date"], - "title": result["title"], - "snippet": result["body"], - "source": result["source"], - "link": result["url"], - } - return { - "snippet": result["body"], - "title": result["title"], - "link": result["href"], + source = source or self.source + if source == "text": + results = [ + {"snippet": r["body"], "title": r["title"], "link": r["href"]} + for r in self._ddgs_text(query, max_results=max_results) + ] + elif source == "news": + results = [ + { + "snippet": r["body"], + "title": r["title"], + "link": r["url"], + "date": r["date"], + "source": r["source"], } + for r in self._ddgs_news(query, max_results=max_results) + ] + else: + results = [] + + if results is None: + results = [{"Result": "No good DuckDuckGo Search Result was found"}] - formatted_results = [] - for i, res in enumerate(results, 1): - if res is not None: - formatted_results.append(to_metadata(res)) - if len(formatted_results) == num_results: - break - return formatted_results + return results diff --git a/libs/langchain/tests/integration_tests/utilities/test_duckduckdgo_search_api.py b/libs/langchain/tests/integration_tests/utilities/test_duckduckdgo_search_api.py index 8d228e573d6b3..74f0f25fa370c 100644 --- a/libs/langchain/tests/integration_tests/utilities/test_duckduckdgo_search_api.py +++ b/libs/langchain/tests/integration_tests/utilities/test_duckduckdgo_search_api.py @@ -1,11 +1,11 @@ import pytest -from langchain.tools.ddg_search.tool import DuckDuckGoSearchRun +from langchain.tools.ddg_search.tool import DuckDuckGoSearchResults, DuckDuckGoSearchRun def ddg_installed() -> bool: try: - from duckduckgo_search import ddg # noqa: F401 + from duckduckgo_search import DDGS # noqa: F401 return True except Exception as e: @@ -20,3 +20,12 @@ def test_ddg_search_tool() -> None: result = tool(keywords) print(result) assert len(result.split()) > 20 + + +@pytest.mark.skipif(not ddg_installed(), reason="requires duckduckgo-search package") +def test_ddg_search_news_tool() -> None: + keywords = "Tesla" + tool = DuckDuckGoSearchResults(source="news") + result = tool(keywords) + print(result) + assert len(result.split()) > 20