arxiv-synth.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9ac67f14-6a79-4686-97e5-fcc12f7e7d9a",
   "metadata": {},
   "source": [
    "# ArXiv Synth\n",
    "\n",
    "<p align=\"center\">\n",
    "   <img src=\"./arxiv-synth.png\" alt=\"ArXiv synth retro logo\" width=\"250\"/>\n",
    "</p>\n",
    "ArXiv Synth is an educational demonstration project that utilizes large language models to summarize and analyze content from arXiv based on a given query. It fetches papers from arXiv, processes them, and generates summaries using OpenAI's and Anthropic's language models.\n",
    "\n",
    "<br><br>\n",
    "The project is meant as an educational demonstration of the capabilities, not as a research tool.\n",
    "\n",
    "# Fetch the paper information from ArXiv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3c0f7f39-f54d-47a0-aa99-2bff06c35ca9",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai_api_key = #<Add API key here>\n",
    "claude_api_key = #<Add API key here>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9c273222-f87d-4624-a5e6-81ad9de14063",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import sys\n",
    "from IPython.display import display, Markdown, clear_output\n",
    "\n",
    "import arxiv\n",
    "import anthropic\n",
    "from openai import OpenAI\n",
    "\n",
    "##########################################################\n",
    "## Functions to interact with ArXiv and cache papers    ##\n",
    "##########################################################\n",
    "def fetch_papers(query, category=None, n_papers=20):\n",
    "    # Construct the API client\n",
    "    client = arxiv.Client()\n",
    "    \n",
    "    # Build the search query\n",
    "    search_query = query\n",
    "    if category:\n",
    "        search_query += f\" AND cat:{category}\"\n",
    "    \n",
    "    # Initialize the search\n",
    "    search = arxiv.Search(\n",
    "        query=search_query,\n",
    "        max_results=n_papers,\n",
    "        sort_by=arxiv.SortCriterion.SubmittedDate\n",
    "    )\n",
    "    \n",
    "    # Fetch the results\n",
    "    results = client.results(search)\n",
    "    \n",
    "    # Initialize a list to hold data dictionaries for each paper\n",
    "    papers = []\n",
    "    \n",
    "    for paper in results:\n",
    "        paper_data = {\n",
    "            'title': paper.title,\n",
    "            'abstract': paper.summary.replace('\\n', ' '),  # Replace new lines in abstracts with spaces\n",
    "            'authors': [author.name for author in paper.authors],\n",
    "            'link': paper.entry_id  # Add this line to include the paper's URL\n",
    "        }\n",
    "        papers.append(paper_data)\n",
    "    return papers\n",
    "\n",
    "    \n",
    "def write_papers(papers, filename=\"./arxiv_papers.jsonl\"):\n",
    "    # Write data to a JSONL file. Replace with DB write if you want something more interesting\n",
    "    with open(filename, 'w') as outfile:\n",
    "        for paper_data in papers:\n",
    "            json.dump(paper_data, outfile)\n",
    "            outfile.write('\\n')\n",
    "\n",
    "\n",
    "##########################################################\n",
    "## Functions to get responses from OpenAI and Anthropic ##\n",
    "##########################################################\n",
    "def get_openai(prompt,\n",
    "               model=\"gpt-3.5-turbo-0125\", \n",
    "               api_key=None, \n",
    "               system_prompt=\"You are a helpful assistant.\", \n",
    "               max_tokens=2000, \n",
    "               temperature=0.8):\n",
    "    # models \"gpt-4-turbo\", \"gpt-3.5-turbo-0125\"\n",
    "    client = OpenAI(api_key=api_key)\n",
    "    \n",
    "    response = client.chat.completions.create(\n",
    "      model=model,\n",
    "      messages=[\n",
    "        {\"role\": \"system\", \"content\": system_prompt},\n",
    "        {\"role\": \"user\", \"content\": prompt}\n",
    "      ],\n",
    "      temperature=temperature,\n",
    "      max_tokens=max_tokens,\n",
    "      top_p=1,\n",
    "      frequency_penalty=0,\n",
    "      presence_penalty=0\n",
    "    )\n",
    "    return response.choices[0].message.content\n",
    "\n",
    "def get_anthropic(prompt, \n",
    "                  model=\"claude-3-haiku-20240307\", \n",
    "                  api_key=None,\n",
    "                  system_prompt=\"You are a helpful assistant.\", \n",
    "                  max_tokens=2000, \n",
    "                  temperature=0):\n",
    "    # Other models \"claude-3-opus-20240229\", \"claude-3-sonnet-20240229\", \"claude-3-haiku-20240307\"\n",
    "\n",
    "    client = anthropic.Anthropic(\n",
    "        api_key=api_key,\n",
    "    )\n",
    "    message = client.messages.create(\n",
    "        model=model,\n",
    "        max_tokens=2000,\n",
    "        temperature=0,\n",
    "        system=f\"{system_prompt}\",\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\n",
    "                        \"type\": \"text\",\n",
    "                        \"text\": prompt\n",
    "                    }\n",
    "                ]\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "    return message.content[0].text\n",
    "\n",
    "def get_streaming_anthropic(input_for_summary, \n",
    "                            model=\"claude-3-haiku-20240307\", \n",
    "                            api_key=None,\n",
    "                            system_prompt=\"You are a helpful assitant\", \n",
    "                            max_tokens=2000, \n",
    "                            temperature=0):\n",
    "\n",
    "    client = anthropic.Anthropic(api_key=claude_api_key)\n",
    "\n",
    "    response = \"\"\n",
    "    with client.messages.stream(\n",
    "        max_tokens=max_tokens,\n",
    "        temperature=temperature,\n",
    "        system=f\"{system_prompt}\",\n",
    "        messages=[{\"role\": \"user\", \"content\": input_for_summary}],\n",
    "        model=model,\n",
    "    ) as stream:\n",
    "        for text in stream.text_stream:\n",
    "            response += text\n",
    "            clear_output(wait=True)\n",
    "            display(Markdown(response))\n",
    "            sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a9562b8-b083-4585-b71f-00ed227bed4b",
   "metadata": {},
   "source": [
    "# Collect Data from ArXiv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "510c1aa0-628f-48f3-af30-a77358fc6a39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "== Fetched 29 papers from ArXiv. ==> \n",
      "== Wrote papers to temporary cache at ./arxiv_papers.jsonl ==>\n"
     ]
    }
   ],
   "source": [
    "query= \"au:blaiszik\"\n",
    "n_papers = 40\n",
    "category = \"\"\n",
    "db_file = './arxiv_papers.jsonl'\n",
    "\n",
    "papers = fetch_papers(query, category=category, n_papers=n_papers)\n",
    "print(f\"== Fetched {len(papers)} papers from ArXiv. ==> \")\n",
    "\n",
    "write_papers(papers, db_file)\n",
    "print(f\"== Wrote papers to temporary cache at {db_file} ==>\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91cad36f-2665-4fe3-bccf-219192dd54dc",
   "metadata": {},
   "source": [
    "# (optional) Read papers back into memory from JSONL file\n",
    "- This is stubbed out for database or cached file retrieval\n",
    "- TODO: For a deeper augmented approach, we could keep a database of ArXiv papers with pre-calculated embeddings, and pull the top N matches into the context as well\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "666ab6d0-9845-46de-b750-836d51decd82",
   "metadata": {},
   "outputs": [],
   "source": [
    "papers = []\n",
    "with open('./arxiv_papers.jsonl', 'r') as file:\n",
    "    for line in file:\n",
    "        papers.append(json.loads(line))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "812ea896-ae0f-40cd-8019-96f29a9e7655",
   "metadata": {},
   "source": [
    "# Create the Prompt and Add Context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a189e274-8e75-4f29-b40d-9b3cbd5737fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_for_summary = \"\"\"Summarize the following arXiv papers at the level of an advanced Ph.D. student, making interconnections between the papers where possible.\n",
    "                       First create a summary paragraph that includes the most important breakthroughs in the contained papers. \n",
    "                       Second, create a summary tweet thread describing the papers. \n",
    "                       Next, provide a section on interconnections where the information is grouped in a structured way that makes it easy to understand, rather than by each paper separately. \n",
    "                       Each paper should always be referenced by its link in markdown format. Be sure to include markdown style links to the papers and to references to the other papers. \n",
    "                       \\n\\n\"\"\"\n",
    "for paper in papers:\n",
    "    input_for_summary += \"<paper>\\n\"\n",
    "    input_for_summary += f\"### {paper['title']}\\n\\n\"\n",
    "    input_for_summary += f\"**Abstract:** {paper['abstract']}\\n\\n\"\n",
    "    input_for_summary += f\"**Authors:** {', '.join(paper['authors'])}\\n\\n\"\n",
    "    input_for_summary += f\"[Link to paper]({paper['link']})\\n\\n\"\n",
    "    input_for_summary += \"</paper>\\n\\n\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbbc643f-a033-42f5-b7dc-95a47f577fb0",
   "metadata": {},
   "source": [
    "# Get OpenAI Summary++"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "52c1da27-0b07-40e9-a86f-e0553d759f52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "### Summary Paragraph:\n",
       "These papers cover a wide range of topics in scientific research, including advancements in machine learning, AI, high-performance computing, and materials science. Breakthroughs include the development of a benchmark dataset for studying surrogate model accuracy in protein-ligand docking, the introduction of funcX for federated function serving, and the proposal of Data Stations for secure data architecture. Other notable contributions include the use of AI and HPC for lead generation in targeting SARS-CoV-2, the creation of a data ecosystem for supporting machine learning in materials science, and the development of machine learning models for predicting atomization energies accurately. These papers collectively reflect the cutting-edge research at the intersection of technology and scientific discovery.\n",
       "\n",
       "### Tweet Thread Summary:\n",
       "1. **Twins in rotational spectroscopy:** Explore the uniqueness of rotational spectra in identifying molecules, challenging assumptions with AI methods. [Link to paper](http://arxiv.org/abs/2404.04225v1)\n",
       "2. **Trillion Parameter AI Infrastructure:** Survey envisioning TPM ecosystem for scientific discovery, highlighting technical challenges and system design. [Link to paper](http://arxiv.org/abs/2402.03480v1)\n",
       "3. **Accelerating Electronic Stopping Power Predictions:** TDDFT and ML combo reduces time for electronic stopping power assessment, enabling predictions in hours. [Link to paper](http://arxiv.org/abs/2311.00787v1)\n",
       "4. **Modular Science Factories:** Proposal for computation and AI-enabled SDLs, emphasizing modularity and scalability for diverse scientific applications. [Link to paper](http://arxiv.org/abs/2308.09793v2)\n",
       "5. **ML Transforming Materials Science:** LLMs hackathon showcases LLM applications in chemistry and materials science, highlighting rapid prototype generation. [Link to paper](http://arxiv.org/abs/2306.06283v4)\n",
       "\n",
       "### Interconnections:\n",
       "\n",
       "- **Machine Learning and Materials Science:**\n",
       "  - The paper on \"Machine Learning Prediction of Atomization Energies\" showcases the application of ML in predicting accurate energy values for organic molecules.\n",
       "  - The \"JARVIS-Leaderboard\" paper introduces a platform for benchmarking materials design methods, which includes AI approaches.\n",
       "- **AI and Scientific Discovery:**\n",
       "  - The \"funcX\" paper describes a federated function serving fabric that enables scalable and high-performance remote function execution, catering to scientific needs.\n",
       "  - \"Serverless Supercomputing\" introduces a serverless approach for high-performance function as a service, serving as a valuable resource for scientific applications.\n",
       "- **AI and Drug Discovery:**\n",
       "  - The \"Targeting SARS-CoV-2 with AI-Enabled Lead Generation\" and \"IMPECCABLE\" papers focus on AI-driven lead generation for COVID-19 drug discovery, demonstrating the potential of AI in accelerating drug screening processes.\n",
       "- **Hybrid Computing Systems:**\n",
       "  - The \"DLHub\" paper presents a system for model and data serving tailored for scientific applications, emphasizing the reproducibility and scalability of ML models in the scientific community.\n",
       "  - \"Accelerated, Scalable and Reproducible AI-driven Gravitational Wave Detection\" leverages AI models to process large datasets rapidly, showcasing the efficiency of combining AI and HPC for scientific discovery."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "system_prompt = \"You are modeling the mind of a researcher who has obtained a PhD in the field of study for the papers retrieved.\"\n",
    "\n",
    "# models \"gpt-4-turbo\", \"gpt-3.5-turbo-0125\"\n",
    "\n",
    "oai = get_openai(input_for_summary, \n",
    "                 model=\"gpt-3.5-turbo-0125\",\n",
    "                 system_prompt=system_prompt,\n",
    "                 max_tokens=2000,\n",
    "                 api_key=openai_api_key\n",
    "                )\n",
    "display(Markdown(oai))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f931af6-30c1-4c37-9c85-9901e87d1506",
   "metadata": {},
   "source": [
    "# Get Anthropic Summary++"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "66a53c71-ee8a-4b2f-ad89-29bb5ff59739",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "Summary Paragraph:\n",
       "\n",
       "The papers presented cover a wide range of topics in materials science and chemistry, showcasing how advanced computational methods, machine learning, and data infrastructure can accelerate scientific discovery. Key breakthroughs include demonstrating that rotational spectra do not uniquely identify molecules, enabling 10 million-fold speedups in electronic stopping power predictions, developing modular architectures for scalable \"science factories\", creating large-scale benchmarks for materials design methods, and leveraging large language models to transform materials research. The papers also highlight the importance of FAIR (Findable, Accessible, Interoperable, Reusable) principles for AI models and data, and present infrastructure like funcX and DLHub to enable federated, scalable, and reproducible computational workflows. Overall, these papers illustrate how the integration of AI, HPC, and data-centric approaches can drive transformative advances across materials science and chemistry.\n",
       "\n",
       "Summary Tweet Thread:\n",
       "\n",
       "1/ Fascinating new research in materials science and chemistry, showing how advanced computational methods, ML, and data infrastructure can accelerate discovery:\n",
       "\n",
       "2/ Rotational spectra don't uniquely identify molecules, enabling 10M-fold speedups in electronic stopping power predictions, and modular \"science factories\" for scalable automation.\n",
       "\n",
       "3/ Large benchmarks for materials design methods, leveraging large language models to transform research, and FAIR principles for AI models/data to enable reproducible workflows.\n",
       "\n",
       "4/ Infrastructure like funcX and DLHub enabling federated, scalable, and reproducible computational pipelines across materials science and chemistry.\n",
       "\n",
       "5/ Integration of AI, HPC, and data-centric approaches driving transformative advances - a glimpse of the future of materials research!\n",
       "\n",
       "Interconnections:\n",
       "\n",
       "The papers presented showcase how advanced computational methods, machine learning, and data infrastructure are transforming materials science and chemistry research. Several key interconnections emerge:\n",
       "\n",
       "**Rotational Spectra and Molecular Identification**: The paper on \"Twins in rotational spectroscopy\" demonstrates that rotational spectra do not uniquely identify molecules, challenging a long-held assumption in the field. This has implications for techniques like rotational spectroscopy that rely on this assumption for structural determination.\n",
       "\n",
       "**Accelerating Computational Methods**: Multiple papers focus on enabling massive speedups in computational methods, from the 10 million-fold acceleration in electronic stopping power predictions to the real-time analysis of flame spray pyrolysis experiments. These advances are critical for enabling high-throughput computational screening and optimization.\n",
       "\n",
       "**Modular and Scalable Architectures**: The paper on \"Towards a Modular Architecture for Science Factories\" presents a framework for building scalable, reconfigurable experimental platforms, which can be combined with the computational infrastructure described in other papers to create integrated \"science factories\".\n",
       "\n",
       "**Benchmarking and Reproducibility**: The \"JARVIS-Leaderboard\" paper and the work on FAIR principles for AI models highlight the importance of rigorous benchmarking and reproducibility in materials research. These efforts provide critical foundations for validating and comparing the various computational and machine learning methods described in the other papers.\n",
       "\n",
       "**Leveraging Large Language Models**: The paper on how large language models can transform materials science demonstrates the potential of these powerful AI models to accelerate research across diverse domains, from knowledge extraction to experimental design.\n",
       "\n",
       "**Data-Centric Computational Frameworks**: Several papers, including those on funcX, DLHub, and the Data Station, describe infrastructure for enabling federated, scalable, and reproducible computational workflows that tightly integrate data management, AI/ML, and high-performance computing. These frameworks are crucial for realizing the full potential of the computational and machine learning advances presented in the other papers.\n",
       "\n",
       "By considering the interconnections between these diverse research directions, it becomes clear that the integration of advanced computational methods, machine learning, and data-centric approaches is driving transformative progress across materials science and chemistry."
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "system_prompt = \"You are modeling the mind of a researcher who has obtained a PhD in the field of study for the papers retrieved.\"\n",
    "\n",
    "# Other models \"claude-3-opus-20240229\", \"claude-3-sonnet-20240229\", \"claude-3-haiku-20240307\"\n",
    "\n",
    "anth = get_anthropic(input_for_summary, \n",
    "                         api_key=claude_api_key,\n",
    "                         model=\"claude-3-haiku-20240307\",\n",
    "                         system_prompt=system_prompt, \n",
    "                         max_tokens=2000, \n",
    "                         temperature=0)\n",
    "display(Markdown(anth))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78ca4831-38e9-46aa-b7c0-7c4ad16adad8",
   "metadata": {},
   "source": [
    "# Anthropic Streaming Example\n",
    "TODO: fix markdown rendering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "dfd4e9fe-86f2-4221-968b-9362a1e41934",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "Summary Paragraph:\n",
       "\n",
       "The collection of papers presented here highlights several important breakthroughs in the application of machine learning and artificial intelligence to accelerate scientific discovery across a range of domains, including materials science, chemistry, and astrophysics. Key advances include the development of benchmark datasets and evaluation frameworks to rigorously assess the performance of machine learning models [JARVIS-Leaderboard, Benchmark tests of atom segmentation], the creation of federated and modular platforms to enable scalable and reproducible AI-driven experimentation and analysis [funcX, Towards a Modular Architecture for Science Factories], and the establishment of FAIR principles to promote the findability, accessibility, interoperability, and reusability of AI models and scientific data [FAIR for AI, Community Action on FAIR Data]. Additionally, the papers showcase successful applications of machine learning to problems such as predicting molecular properties [Machine Learning Prediction of Accurate Atomization Energies], automating experimental workflows [Deep Learning for Automated Experimentation], and accelerating the detection of gravitational waves [Accelerated, Scalable and Reproducible AI-driven Gravitational Wave Detection]. Collectively, these advances demonstrate the transformative potential of AI and machine learning to revolutionize scientific discovery across a wide range of disciplines.\n",
       "\n",
       "Summary Tweet Thread:\n",
       "\n",
       "1/ This thread summarizes a collection of papers showcasing breakthroughs in applying machine learning and AI to accelerate scientific discovery. Key advances include:\n",
       "\n",
       "2/ Development of benchmark datasets and evaluation frameworks to rigorously assess ML model performance [JARVIS-Leaderboard, Benchmark tests of atom segmentation]\n",
       "\n",
       "3/ Creation of federated and modular platforms to enable scalable and reproducible AI-driven experimentation and analysis [funcX, Towards a Modular Architecture for Science Factories]\n",
       "\n",
       "4/ Establishment of FAIR principles to promote the findability, accessibility, interoperability, and reusability of AI models and scientific data [FAIR for AI, Community Action on FAIR Data]\n",
       "\n",
       "5/ Successful applications of ML to predict molecular properties [Machine Learning Prediction of Accurate Atomization Energies], automate experimental workflows [Deep Learning for Automated Experimentation], and accelerate gravitational wave detection [Accelerated, Scalable and Reproducible AI-driven Gravitational Wave Detection]\n",
       "\n",
       "6/ These advances demonstrate the transformative potential of AI/ML to revolutionize scientific discovery across disciplines. The future of data-driven science is bright!\n",
       "\n",
       "Interconnections:\n",
       "\n",
       "The papers presented here can be grouped into several interconnected themes:\n",
       "\n",
       "**Benchmark Datasets and Evaluation Frameworks**\n",
       "- [JARVIS-Leaderboard] and [Benchmark tests of atom segmentation] describe the development of benchmark datasets and evaluation frameworks to rigorously assess the performance of machine learning models in materials science and microscopy applications.\n",
       "- These benchmark efforts are crucial for promoting reproducibility, enabling fair comparisons between different modeling approaches, and accelerating progress in the field.\n",
       "\n",
       "**Federated and Modular AI Platforms**\n",
       "- [funcX] and [Towards a Modular Architecture for Science Factories] present the development of federated and modular platforms that enable scalable, flexible, and reproducible AI-driven experimentation and analysis.\n",
       "- These platforms decouple the management of computational resources from the execution of scientific workflows, allowing for efficient utilization of diverse computing infrastructure (e.g., clouds, clusters, supercomputers) and"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Other models \"claude-3-opus-20240229\", \"claude-3-sonnet-20240229\", \"claude-3-haiku-20240307\"\n",
    "\n",
    "# Call the function with your input\n",
    "get_streaming_anthropic(input_for_summary, \n",
    "                             model=\"claude-3-haiku-20240307\",\n",
    "                             api_key=claude_api_key,\n",
    "                             system_prompt=system_prompt, \n",
    "                             max_tokens=750, \n",
    "                             temperature=0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ead2fad-e404-4609-91fc-1fdb1f287678",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}