From d417e4b3727cfc42abc6cc2968ee46358453ed04 Mon Sep 17 00:00:00 2001 From: Aayush Kataria Date: Sun, 15 Dec 2024 13:26:32 -0800 Subject: [PATCH] Community: Azure CosmosDB No Sql Vector Store: Full Text and Hybrid Search Support (#28716) Thank you for contributing to LangChain! - Added [full text](https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search) and [hybrid search](https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/hybrid-search) support for Azure CosmosDB NoSql Vector Store - Added a new enum called CosmosDBQueryType which supports the following values: - VECTOR = "vector" - FULL_TEXT_SEARCH = "full_text_search" - FULL_TEXT_RANK = "full_text_rank" - HYBRID = "hybrid" - User now needs to provide this query_type to the similarity_search method for the vectorStore to make the correct query api call. - Added a couple of work arounds as for the FULL_TEXT_RANK and HYBRID query functions we don't support parameterized queries right now. I have added TODO's in place, and will remove these work arounds by end of January. - Added necessary test cases and updated the - [x] **Add tests and docs**: If you're adding a new integration, please include 1. a test for the integration, preferably unit tests that do not rely on network access, 2. an example notebook showing its use. It lives in `docs/docs/integrations` directory. - [x] **Lint and test**: Run `make format`, `make lint` and `make test` from the root of the package(s) you've modified. See contribution guidelines for more: https://python.langchain.com/docs/contributing/ Additional guidelines: - Make sure optional dependencies are imported within a function. - Please do not add dependencies to pyproject.toml files (even optional ones) unless they are required for unit tests. - Most PRs should not touch more than one package. - Changes should be backwards compatible. - If you are adding something to community, do not re-import it in langchain. If no one reviews your PR within a few days, please @-mention one of baskaryan, efriis, eyurtsev, ccurme, vbarda, hwchase17. --------- Co-authored-by: Erick Friis --- .../vectorstores/azure_cosmos_db_no_sql.ipynb | 450 +++++++++++-- .../vectorstores/azure_cosmos_db.py | 1 + .../vectorstores/azure_cosmos_db_no_sql.py | 602 +++++++++++++++--- .../test_azure_cosmos_db_no_sql.py | 370 +++++++++-- 4 files changed, 1261 insertions(+), 162 deletions(-) diff --git a/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb b/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb index 9e3109a01d89f..7a231f764e276 100644 --- a/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb +++ b/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb @@ -11,7 +11,12 @@ " \n", "Azure Cosmos DB is the database that powers OpenAI's ChatGPT service. It offers single-digit millisecond response times, automatic and instant scalability, along with guaranteed speed at any scale. \n", "\n", - "[Azure Cosmos DB for NoSQL](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/vector-search) now offers vector indexing and search in preview. This feature is designed to handle high-dimensional vectors, enabling efficient and accurate vector search at any scale. You can now store vectors directly in the documents alongside your data. This means that each document in your database can contain not only traditional schema-free data, but also high-dimensional vectors as other properties of the documents. This colocation of data and vectors allows for efficient indexing and searching, as the vectors are stored in the same logical unit as the data they represent. This simplifies data management, AI application architectures, and the efficiency of vector-based operations.\n", + "Azure Cosmos DB for NoSQL now offers vector indexing and search in preview. This feature is designed to handle high-dimensional vectors, enabling efficient and accurate vector search at any scale. You can now store vectors directly in the documents alongside your data. This means that each document in your database can contain not only traditional schema-free data, but also high-dimensional vectors as other properties of the documents. This colocation of data and vectors allows for efficient indexing and searching, as the vectors are stored in the same logical unit as the data they represent. This simplifies data management, AI application architectures, and the efficiency of vector-based operations.\n", + "\n", + "Please refer here for more details:\n", + "- [Vector Search](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/vector-search)\n", + "- [Full Text Search](https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search)\n", + "- [Hybrid Search](https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/hybrid-search)\n", "\n", "[Sign Up](https://azure.microsoft.com/en-us/free/) for lifetime free access to get started today." ] @@ -36,12 +41,12 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 1, "id": "c507b0e8", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:36:53.595385Z", - "start_time": "2024-05-25T01:36:53.571737Z" + "end_time": "2024-12-13T21:14:23.285152Z", + "start_time": "2024-12-13T21:14:23.277816Z" } }, "outputs": [], @@ -64,12 +69,12 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 2, "id": "8205cd27", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:02.731634Z", - "start_time": "2024-05-25T01:43:00.383956Z" + "end_time": "2024-12-13T21:14:36.109885Z", + "start_time": "2024-12-13T21:14:30.764272Z" } }, "outputs": [], @@ -83,12 +88,12 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 3, "id": "8d33cceb", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:02.787966Z", - "start_time": "2024-05-25T01:43:02.763502Z" + "end_time": "2024-12-13T21:14:38.909571Z", + "start_time": "2024-12-13T21:14:38.813059Z" } }, "outputs": [], @@ -101,12 +106,12 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 4, "id": "6a80f1c2", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:04.582560Z", - "start_time": "2024-05-25T01:43:04.578948Z" + "end_time": "2024-12-13T21:14:41.079019Z", + "start_time": "2024-12-13T21:14:41.075776Z" } }, "outputs": [ @@ -114,7 +119,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "page_content='GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction' metadata={'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 0}\n" + "page_content='GPT-4 Technical Report\n", + "OpenAI∗\n", + "Abstract\n", + "We report the development of GPT-4, a large-scale, multimodal model which can\n", + "accept image and text inputs and produce text outputs. While less capable than\n", + "humans in many real-world scenarios, GPT-4 exhibits human-level performance\n", + "on various professional and academic benchmarks, including passing a simulated\n", + "bar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\n", + "based model pre-trained to predict the next token in a document. The post-training\n", + "alignment process results in improved performance on measures of factuality and\n", + "adherence to desired behavior. A core component of this project was developing\n", + "infrastructure and optimization methods that behave predictably across a wide\n", + "range of scales. This allowed us to accurately predict some aspects of GPT-4’s\n", + "performance based on models trained with no more than 1/1,000th the compute of\n", + "GPT-4.\n", + "1 Introduction' metadata={'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 0}\n" ] } ], @@ -132,12 +152,12 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 5, "id": "04c72ccc", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:13.279497Z", - "start_time": "2024-05-25T01:43:13.275379Z" + "end_time": "2024-12-13T21:15:39.050171Z", + "start_time": "2024-12-13T21:15:39.046271Z" } }, "outputs": [], @@ -146,7 +166,8 @@ " \"indexingMode\": \"consistent\",\n", " \"includedPaths\": [{\"path\": \"/*\"}],\n", " \"excludedPaths\": [{\"path\": '/\"_etag\"/?'}],\n", - " \"vectorIndexes\": [{\"path\": \"/embedding\", \"type\": \"quantizedFlat\"}],\n", + " \"vectorIndexes\": [{\"path\": \"/embedding\", \"type\": \"diskANN\"}],\n", + " \"fullTextIndexes\": [{\"path\": \"/text\"}],\n", "}\n", "\n", "vector_embedding_policy = {\n", @@ -158,17 +179,22 @@ " \"dimensions\": 1536,\n", " }\n", " ]\n", + "}\n", + "\n", + "full_text_policy = {\n", + " \"defaultLanguage\": \"en-US\",\n", + " \"fullTextPaths\": [{\"path\": \"/text\", \"language\": \"en-US\"}],\n", "}" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 10, "id": "4ebad8ef01a6c04f", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:48:42.981276Z", - "start_time": "2024-05-25T01:44:55.468667Z" + "end_time": "2024-12-13T21:24:55.719198Z", + "start_time": "2024-12-13T21:22:47.334006Z" } }, "outputs": [], @@ -177,7 +203,7 @@ "from langchain_community.vectorstores.azure_cosmos_db_no_sql import (\n", " AzureCosmosDBNoSqlVectorSearch,\n", ")\n", - "from langchain_openai import AzureOpenAIEmbeddings\n", + "from langchain_openai import OpenAIEmbeddings\n", "\n", "HOST = \"AZURE_COSMOS_DB_ENDPOINT\"\n", "KEY = \"AZURE_COSMOS_DB_KEY\"\n", @@ -188,11 +214,11 @@ "partition_key = PartitionKey(path=\"/id\")\n", "cosmos_container_properties = {\"partition_key\": partition_key}\n", "\n", - "openai_embeddings = AzureOpenAIEmbeddings(\n", - " azure_deployment=OPENAI_EMBEDDINGS_MODEL_DEPLOYMENT,\n", - " api_version=OPENAI_API_VERSION,\n", - " azure_endpoint=OPENAI_API_BASE,\n", - " openai_api_key=OPENAI_API_KEY,\n", + "openai_embeddings = OpenAIEmbeddings(\n", + " deployment=\"smart-agent-embedding-ada\",\n", + " model=\"text-embedding-ada-002\",\n", + " chunk_size=1,\n", + " openai_api_key=\"OPENAI_API_KEY\",\n", ")\n", "\n", "# insert the documents in AzureCosmosDBNoSql with their embedding\n", @@ -203,8 +229,11 @@ " database_name=database_name,\n", " container_name=container_name,\n", " vector_embedding_policy=vector_embedding_policy,\n", + " full_text_policy=full_text_policy,\n", " indexing_policy=indexing_policy,\n", " cosmos_container_properties=cosmos_container_properties,\n", + " cosmos_database_properties={},\n", + " full_text_search_enabled=True,\n", ")" ] }, @@ -212,18 +241,16 @@ "cell_type": "markdown", "id": "3f5ff6adb7c8ad48", "metadata": {}, - "source": [ - "## Querying Data" - ] + "source": "## Vector Search" }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 22, "id": "7cf7a2cc", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:48:47.576539Z", - "start_time": "2024-05-25T01:48:46.693940Z" + "end_time": "2024-12-13T21:34:31.438548Z", + "start_time": "2024-12-13T21:34:30.456291Z" } }, "outputs": [ @@ -248,6 +275,8 @@ ], "source": [ "# Perform a similarity search between the embedding of the query and the embeddings of the documents\n", + "import json\n", + "\n", "query = \"What were the compute requirements for training GPT 4\"\n", "results = vector_search.similarity_search(query)\n", "\n", @@ -258,18 +287,75 @@ "cell_type": "markdown", "id": "30d77484c71fe192", "metadata": {}, + "source": "## Vector Search with Score" + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "6371c107306cd76d", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:45:46.356588Z", + "start_time": "2024-12-13T21:45:45.770116Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 1: 0.8394796122122777\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":43,\"id\":\"e5610de3-8af6-43b9-8266-51c26d76eaa3\"},\"page_content\":\"2 GPT-4 Observed Safety Challenges\\nGPT-4 demonstrates increased performance in areas such as reasoning, knowledge retention, and\\ncoding, compared to earlier models such as GPT-2[ 22] and GPT-3.[ 10] Many of these improvements\\nalso present new safety challenges, which we highlight in this section.\\nWe conducted a range of qualitative and quantitative evaluations of GPT-4. These evaluations\\nhelped us gain an understanding of GPT-4’s capabilities, limitations, and risks; prioritize our\\nmitigation efforts; and iteratively test and build safer versions of the model. Some of the specific\\nrisks we explored are:6\\n•Hallucinations\\n•Harmful content\\n•Harms of representation, allocation, and quality of service\\n•Disinformation and influence operations\\n•Proliferation of conventional and unconventional weapons\\n•Privacy\\n•Cybersecurity\\n•Potential for risky emergent behaviors\\n•Interactions with other systems\\n•Economic impacts\\n•Acceleration\\n•Overreliance\",\"type\":\"Document\"}\n", + "Score 2: 0.8299261339098007\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"cddfb7ac-d953-46f4-8a48-76655f116bcf\"},\"page_content\":\"GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\",\"type\":\"Document\"}\n", + "Score 3: 0.8286253517208215\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":3,\"id\":\"4f3152cd-c543-4a4f-b94e-c52c4139c4a8\"},\"page_content\":\"plan to refine these methods and register performance predictions across various capabilities before\\nlarge model training begins, and we hope this becomes a common goal in the field.\\n4 Capabilities\\nWe tested GPT-4 on a diverse set of benchmarks, including simulating exams that were originally\\ndesigned for humans.4We did no specific training for these exams. A minority of the problems in the\\nexams were seen by the model during training; for each exam we run a variant with these questions\\nremoved and report the lower score of the two. We believe the results to be representative. For further\\ndetails on contamination (methodology and per-exam statistics), see Appendix C.\\nExams were sourced from publicly-available materials. Exam questions included both multiple-\\nchoice and free-response questions; we designed separate prompts for each format, and images were\\nincluded in the input for questions which required it. The evaluation setup was designed based\",\"type\":\"Document\"}\n", + "Score 4: 0.8278858118680015\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":28,\"id\":\"18b4f43d-27d2-404e-9d66-f9328a3588c6\"},\"page_content\":\"overall GPT-4 training budget. When mixing in data from these math benchmarks, a portion of the\\ntraining data was held back, so each individual training example may or may not have been seen by\\nGPT-4 during training.\\nWe conducted contamination checking to verify the test set for GSM-8K is not included in the training\\nset (see Appendix D). We recommend interpreting the performance results reported for GPT-4\\nGSM-8K in Table 2 as something in-between true few-shot transfer and full benchmark-specific\\ntuning.\\nF Multilingual MMLU\\nWe translated all questions and answers from MMLU [ 49] using Azure Translate. We used an\\nexternal model to perform the translation, instead of relying on GPT-4 itself, in case the model had\\nunrepresentative performance for its own translations. We selected a range of languages that cover\\ndifferent geographic regions and scripts, we show an example question taken from the astronomy\",\"type\":\"Document\"}\n", + "Score 5: 0.8272138555588135\n", + "\n", + "\n" + ] + } + ], "source": [ - "## Similarity Search with Score" + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" ] }, + { + "cell_type": "markdown", + "id": "9c4ffb492375192d", + "metadata": {}, + "source": "## Vector Search with filtering" + }, { "cell_type": "code", - "execution_count": 43, - "id": "6371c107306cd76d", + "execution_count": 50, + "id": "461c6ac2ba3cee2a", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:51:15.012519Z", - "start_time": "2024-05-25T01:51:13.597319Z" + "end_time": "2024-12-13T21:53:49.420189Z", + "start_time": "2024-12-13T21:53:47.779582Z" } }, "outputs": [ @@ -277,37 +363,303 @@ "name": "stdout", "output_type": "stream", "text": [ - "(Document(page_content='performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.', metadata={'id': '95cc87c2-27f5-40d2-afc2-a354e8f339e4', 'embedding': [-0.025839418172836304, -0.004486586898565292, -0.003380518639460206, -0.014719882979989052, 0.01754477061331272, 0.01754477061331272, -0.011216467246413231, 0.006923745386302471, -0.018334077671170235, -0.027348794043064117, 0.011860375292599201, 0.030990684404969215, -0.03522801771759987, -0.020854320377111435, -0.008370808325707912, 0.022931445389986038, 0.016727767884731293, -0.007706128526479006, 0.0006555921281687915, -0.017406295984983444, 0.014020584523677826, 0.00848851166665554, -0.018237145617604256, -0.018320230767130852, 0.010136363096535206, 0.011811909265816212, 0.031156854704022408, -0.037000495940446854, -0.014719882979989052, 0.0030897213146090508, 0.015052222646772861, -0.016893938183784485, -0.022045204415917397, -0.0068510458804667, 0.0028629687149077654, -0.007408407516777515, 0.009291666559875011, -0.010489474050700665, 0.004389654379338026, -0.004638909362256527, -0.0011874223127961159, 0.015647664666175842, 0.02370690368115902, -0.011583426035940647, 0.0017006449634209275, 0.0028266189619898796, 0.01800173707306385, -0.0284427460283041, 0.0016772772651165724, 0.004631985444575548, 0.018237145617604256, 0.03400943800806999, -0.003780364990234375, 0.0003414271923247725, -0.018791044130921364, 0.007733823731541634, -0.00014767050743103027, 0.00948553066700697, 0.017974043264985085, 0.0010601985268294811, -0.011922689154744148, -0.00823233276605606, -0.024759313091635704, 0.03373248875141144, -0.005840179044753313, -0.01509376522153616, -0.037055883556604385, 0.015924613922834396, -0.00361419515684247, -0.0017569003393873572, -0.0026569871697574854, 0.008675453253090382, 0.0020771236158907413, -0.025077804923057556, 0.037305139005184174, -0.010510245338082314, 0.0018901824951171875, -0.009921726770699024, -0.0230422243475914, 0.012552750296890736, 0.018749501556158066, -0.009520149789750576, 0.014103669673204422, 0.03403713181614876, 0.005663623567670584, -0.004663142375648022, 0.01595230959355831, -0.003087990451604128, -0.01701856590807438, -0.0017421874217689037, 0.017157040536403656, -0.0024908173363655806, 0.017710940912365913, 0.014096745289862156, -0.015329171903431416, 0.01995423436164856, 0.0031260710675269365, 0.02880278043448925, -0.014346000738441944, -0.019871149212121964, -0.011043373495340347, 0.0028923945501446724, -0.020341964438557625, -0.014415238052606583, -0.023208394646644592, 0.008363883942365646, 0.018167907372117043, 0.00011521545093273744, 0.0036834324710071087, 0.005642852280288935, -0.02021733671426773, 0.02186518721282482, -0.011486493051052094, -0.010780271142721176, -0.01595230959355831, 0.00623483257368207, 0.020868169143795967, -0.008017697371542454, 0.0008421005331911147, -0.0020546214655041695, 0.02577017992734909, -0.016783159226179123, 0.014359847642481327, 0.0021827106829732656, 0.026822589337825775, 0.006653719116002321, -0.018140213564038277, -0.031101463362574577, 0.003544957609847188, -0.0004048227274324745, 0.012711996212601662, 0.002544476417824626, 0.0274041835218668, -0.007678433787077665, -0.03085220977663994, 0.010191753506660461, -0.024496210739016533, 0.005992501508444548, -0.038440633565187454, -0.018278688192367554, 0.017461685463786125, 0.02776421792805195, -0.015924613922834396, -0.009000868536531925, -0.007879221811890602, 0.028415050357580185, 0.015661511570215225, 0.010960289277136326, 0.011770366691052914, 0.011708053760230541, 0.020134251564741135, -0.01779402606189251, -0.008973173797130585, 0.027736524119973183, 0.011631892062723637, -0.011396484449505806, 0.00421656109392643, 0.02277912199497223, -0.009499378502368927, -0.024662381038069725, -0.01769709214568138, 0.018860282376408577, -0.005151266697794199, 0.006986059248447418, 0.01851409487426281, 0.019871149212121964, 0.00910472497344017, 0.02027272619307041, -4.362500476418063e-05, -0.005341669544577599, -0.00907703023403883, 0.02448236383497715, -0.033870961517095566, 0.02351303957402706, -0.013265895657241344, 0.0006915090489201248, 0.013134344480931759, -0.007366864942014217, 0.0021740561351180077, -0.016340039670467377, 0.008142324164509773, 0.018707958981394768, -0.00318838469684124, -0.004337726626545191, -0.015619969926774502, -0.02711338736116886, 0.013979041948914528, -0.0014903361443430185, -0.01794634759426117, -0.01959419995546341, 0.01985730230808258, 0.0018174831056967378, 0.004143861588090658, -0.007408407516777515, -0.6279006004333496, -0.010441008023917675, -0.0062002139165997505, -0.015107612125575542, 0.025797875598073006, 0.013930575922131538, 0.014290610328316689, -0.0011727093951776624, -0.024191565811634064, 0.020646609365940094, -0.008758537471294403, -0.015287629328668118, -0.01267737802118063, -0.025216281414031982, -0.009201657958328724, -0.03353862091898918, -0.017420142889022827, -0.018237145617604256, 0.0038149836473166943, 0.02046659030020237, -0.02921820618212223, 0.014706035144627094, 0.0013007986126467586, 0.019774217158555984, -0.0291628148406744, 0.0003647948324214667, -0.00884854607284069, -0.013736710883677006, 0.005535534583032131, 0.01609078422188759, -0.03594808652997017, 0.023263784125447273, 0.0034341777209192514, -0.004334264434874058, 0.04134860634803772, -0.0021809798199683428, -0.029550544917583466, 0.014006736688315868, -0.008481588214635849, 0.026656419038772583, -0.029550544917583466, -0.012739690952003002, -0.0017214161343872547, 0.008675453253090382, -0.013868262059986591, 0.037166666239500046, 0.0016314075328409672, -0.014899900183081627, -0.006425235886126757, -0.01445678062736988, -0.012130402028560638, 0.010468702763319016, 0.024302346631884575, -0.0021619393955916166, -0.01093259360641241, 0.00438619265332818, 0.03539418429136276, 0.001936917775310576, -0.015412257052958012, -0.028553524985909462, -0.0023211855441331863, 0.005892107263207436, -0.003551881294697523, -0.0402408093214035, -0.01887412928044796, 0.019109537824988365, 0.008024620823562145, -0.0028196952771395445, 0.0032281961757689714, -0.041071657091379166, -0.010413313284516335, 0.019940387457609177, 0.0026310232933610678, -0.008890088647603989, 0.003956920467317104, 0.005310512613505125, 0.021879035979509354, -0.016963176429271698, 0.0009398984257131815, 0.026933370158076286, 0.011631892062723637, -0.014152135699987411, 0.005313974339514971, -0.00042927221511490643, 0.019303401932120323, 0.009263970889151096, -0.005275893956422806, 0.015066069550812244, 0.024468515068292618, 0.006937592756003141, 0.007858450524508953, 0.024565448984503746, -0.020480439066886902, -0.023803835734725, 0.012151173315942287, -0.0008226275094784796, -0.02891356125473976, -0.0052412752993404865, 0.005030101165175438, -0.010468702763319016, -0.009430141188204288, -0.024662381038069725, 0.03533879667520523, 0.028359660878777504, 0.03508954122662544, 0.017475532367825508, -0.014706035144627094, 0.0012436778051778674, -0.006013272795826197, -0.006767961196601391, -0.008107705973088741, 0.002279643202200532, -0.006574096158146858, -0.009610158391296864, 0.016852395609021187, -0.0291628148406744, 0.013778253458440304, -0.006553324870765209, -0.022709885612130165, -0.03173844888806343, 0.01989884488284588, -0.016367733478546143, 0.012490436434745789, -0.007456873543560505, 0.009873260743916035, 0.010787195526063442, 0.014761424623429775, -0.027653438970446587, -0.0002302145294379443, 0.014539864845573902, -0.015647664666175842, -0.018029432743787766, 0.057273220270872116, -0.010531016625463963, 0.02963363006711006, 0.021435916423797607, 0.033510927110910416, 0.0036765087861567736, 0.0060998196713626385, -0.013639777898788452, -0.02093740552663803, 0.002916627796366811, 0.0002674296556506306, 0.004168094601482153, -0.018029432743787766, -0.014262915588915348, -0.011223391629755497, 0.005985578056424856, 0.009443989023566246, -0.0006538612069562078, -0.0005590924411080778, -0.00812847726047039, -0.004486586898565292, 0.014262915588915348, -0.0016045779921114445, -0.02349919080734253, -0.010801042430102825, -0.053008195012807846, -0.008654681965708733, -0.019206469878554344, 0.011348018422722816, 0.003195308381691575, -0.008917784318327904, 0.008730842731893063, 0.006664104759693146, -0.02581172250211239, -0.016963176429271698, 0.03857911005616188, -0.04794001206755638, -0.027999626472592354, 0.00588864553719759, -0.03132302314043045, 0.018597180023789406, 0.025950197130441666, -0.0202034879475832, -0.002400808734819293, 0.004431196954101324, -0.0014609103091061115, 0.0018192140851169825, -0.02546553499996662, -0.005715551786124706, 0.011077992618083954, -0.011105687357485294, -0.020134251564741135, 0.019871149212121964, 0.03256929665803909, 0.04123782739043236, 0.015675360336899757, -0.022972986102104187, 0.00018769840244203806, 0.017835568636655807, 0.01866641826927662, -0.028553524985909462, 0.028221186250448227, 0.001203866209834814, 0.03060295432806015, 0.012054240331053734, 0.006508320569992065, 0.015259934589266777, 0.03268007934093475, 0.012026545591652393, 0.010219448246061802, 0.014235220849514008, -0.019926538690924644, -0.011638815514743328, -0.012435046955943108, 0.01722627878189087, -0.020494285970926285, 0.0010627949377521873, 0.0033441688865423203, 0.006757575552910566, -0.011348018422722816, -0.016783159226179123, -0.0022952216677367687, -0.005106262397021055, 0.01830638200044632, -0.0025894807185977697, 0.0018746040295809507, -0.025368602946400642, -0.02946745976805687, 0.005123571492731571, -0.0007975288899615407, 0.016852395609021187, 0.009263970889151096, -0.012940480373799801, 0.020535828545689583, -0.02222522161900997, 0.01856948435306549, 0.025133196264505386, -0.027030302211642265, 0.0006283298716880381, 0.017614008858799934, 0.01836177334189415, 0.016215411946177483, 0.013286666944622993, 0.0017421874217689037, 0.01177729107439518, -0.007103762589395046, 0.037360530346632004, 0.003136456711217761, -0.004715070594102144, 0.007477644830942154, -0.0011908841552212834, -0.02880278043448925, 0.025063958019018173, -0.009457835927605629, 0.011174924671649933, -0.018860282376408577, -0.026933370158076286, 0.0015093764523044229, -0.012206562794744968, 0.018375620245933533, -0.02201751060783863, 0.005701704416424036, 0.03292933106422424, -0.008585444651544094, 0.006227909121662378, 0.003596885595470667, 0.006027120165526867, 0.030270613729953766, 0.004375807009637356, 0.01339744683355093, 0.025285517796874046, 0.006885664537549019, 0.022183680906891823, -0.028692001476883888, 0.002972017740830779, -0.015204545110464096, -0.021532848477363586, -0.013120497576892376, -0.025894807651638985, -0.003596885595470667, 0.019414182752370834, -0.026254842057824135, 0.018444858491420746, 0.008066163398325443, 0.015176849439740181, 0.019400333985686302, 0.030741428956389427, 0.016215411946177483, -0.028221186250448227, -0.05203887075185776, 0.009603234939277172, 0.001980191096663475, -0.007809984963387251, -0.0032957028597593307, -0.05450372397899628, 0.011915765702724457, -0.003003174439072609, -0.014207525178790092, -0.014062127098441124, 0.018029432743787766, -0.0021567465737462044, 0.009762480854988098, -0.0026760275941342115, 0.018804892897605896, 0.027196472510695457, -0.004424273036420345, -0.012594292871654034, -0.0013709015911445022, 0.012026545591652393, -0.002528897952288389, -0.012850471772253513, -0.03179384022951126, 0.01568920724093914, -0.018583333119750023, -0.03522801771759987, -0.010177905671298504, 0.010240219533443451, -0.015619969926774502, 0.03148919343948364, 0.0005110589554533362, 0.010558711364865303, -0.017101651057600975, 0.0026033283211290836, 0.016755463555455208, -0.01676931045949459, 0.022336002439260483, 0.03514493256807327, 0.012684301473200321, 0.015966156497597694, -0.026088671758770943, -0.007713052444159985, 0.018126364797353745, 0.04007463902235031, 0.04057314619421959, -0.012649682350456715, -0.012476589530706406, -0.028359660878777504, 0.0021844415459781885, -0.030879903584718704, -0.017198583111166954, 0.012573521584272385, 0.021020490676164627, 0.01273276749998331, -0.023028377443552017, -0.010918746702373028, -0.0007533900206908584, 0.026711810380220413, 0.007415331434458494, -0.002338494872674346, -0.013383599929511547, -0.014318305067718029, -0.010524093173444271, 0.007387636229395866, -0.020909711718559265, 0.013639777898788452, -0.005788251291960478, 0.011348018422722816, -0.025340907275676727, 0.025797875598073006, 0.020494285970926285, -0.013210506178438663, -0.028068862855434418, -0.026254842057824135, -0.01237273309379816, -0.005968268495053053, 0.0052585843950510025, 0.005338207818567753, 0.005251660943031311, 0.001980191096663475, -0.014664492569863796, 0.012400427833199501, 0.016935480758547783, 0.010801042430102825, 0.011645739898085594, 0.0037457463331520557, -0.013168963603675365, -0.01789095811545849, -0.023236088454723358, 0.0031710753683000803, 0.02351303957402706, -0.008571596816182137, -0.03154458478093147, 0.013826719485223293, 0.020023470744490623, -0.024773159995675087, -0.0005435140337795019, 0.021989814937114716, 0.0025392835959792137, 0.0026967988815158606, 0.002516781445592642, -0.01938648708164692, -0.01964958943426609, -0.006494473200291395, -0.03611425682902336, 0.005556305404752493, -0.0026829512789845467, -0.011562654748558998, 0.0006806906894780695, -0.021560542285442352, -0.010503321886062622, -0.03478489816188812, 0.012926632538437843, -0.010704110376536846, -0.012504284270107746, -0.04877086356282234, -0.010634873062372208, 0.015301477164030075, 0.014719882979989052, 0.00547668244689703, -0.0026414089370518923, 0.0012886821059510112, 0.0034324468579143286, -0.014622949995100498, -0.02524397522211075, -0.0038184456061571836, -0.03085220977663994, 0.01919262297451496, 0.006404464598745108, -0.022862207144498825, -0.014512170106172562, -0.008530054241418839, 0.009499378502368927, -0.0007213677163235843, -0.01092567015439272, -0.007768442388623953, -0.008952402509748936, 0.015869224444031715, 0.019940387457609177, 0.014747577719390392, -0.003894606838002801, 0.007636891212314367, -0.027598049491643906, -0.02129743993282318, -0.027030302211642265, -0.006771422922611237, -0.0388837531208992, 0.012628911063075066, 0.005743246525526047, 0.020632760599255562, 0.012739690952003002, 0.02000962384045124, -0.023623818531632423, 0.013951347209513187, -0.0033649401739239693, 0.006664104759693146, -0.007318398915231228, 0.00843312218785286, 0.0006664104876108468, 0.005092414561659098, 0.022336002439260483, 0.01396519411355257, -0.017115497961640358, 0.0015102420002222061, -0.01784941554069519, 0.025036262348294258, 0.015924613922834396, -0.008273875340819359, 0.03148919343948364, 0.008336189202964306, 0.0019490342820063233, -0.04439505562186241, 0.009859412908554077, -0.014048279263079166, 0.013134344480931759, -0.0011960769770666957, -0.008073086850345135, -0.026877978816628456, -0.013162040151655674, -0.014733729884028435, 0.004496972542256117, -0.01355669368058443, 0.001865949365310371, -0.0036626611836254597, -0.006397540681064129, -0.007616119924932718, 0.005864412523806095, 0.0029339371249079704, -0.02597789280116558, -0.011431103572249413, 0.003718051128089428, 0.0212420504540205, 0.022917596623301506, -0.005223965737968683, 0.01629849709570408, -0.011888070963323116, -0.027071844786405563, -0.01273276749998331, -0.041902508586645126, -0.004573133774101734, 0.010807966813445091, 0.02849813550710678, 0.017032412812113762, 0.03533879667520523, -0.011230315081775188, 0.0022000200115144253, 0.039936162531375885, -0.011043373495340347, -0.006310993805527687, -0.010918746702373028, -0.015523036941885948, -0.02577017992734909, -0.0011830950388684869, -0.00727685634046793, -0.002025195397436619, 0.009859412908554077, -0.00188672065269202, -0.010690262541174889, 0.014235220849514008, -0.008626986294984818, 0.008516206406056881, -0.014387542381882668, -0.015190697275102139, -0.025700943544507027, 0.020397353917360306, -0.02169901877641678, 0.00239907787181437, -0.025188585743308067, -0.008987021632492542, 0.03154458478093147, 0.0042304084636271, -0.0031208782456815243, 0.014131364412605762, 0.042511794716119766, -0.01656159944832325, -0.028553524985909462, -0.0010653913486748934, 0.02067430317401886, 0.004026157781481743, -0.009734786115586758, -0.05339592322707176, 0.016880091279745102, 0.02201751060783863, 0.002222522161900997, 0.028830476105213165, -0.004742765333503485, 0.002565247705206275, -0.0014565829187631607, 0.01270507276058197, 0.004573133774101734, -0.009319361299276352, -0.00295124645344913, -0.0158830713480711, -0.009810946881771088, -0.004912397358566523, 0.011708053760230541, -0.0026569871697574854, -0.003480912884697318, 0.01974652148783207, -0.011354942806065083, 0.021283593028783798, -0.0338432677090168, -0.02180979773402214, 0.027445726096630096, 0.009900955483317375, 0.03198770433664322, 0.01183268055319786, 0.01384056732058525, 0.020646609365940094, 0.020480439066886902, -0.006373307667672634, 0.006487549282610416, 0.006906435824930668, -0.004188865888863802, 0.010399465449154377, 0.005933649837970734, 0.013265895657241344, -0.01861102692782879, 0.015107612125575542, 0.011008755303919315, -0.039991553872823715, -0.038496024906635284, 0.01789095811545849, 0.014747577719390392, 0.022876054048538208, -0.0309629887342453, -0.007796137128025293, -0.025687094777822495, -0.0002211271203123033, -0.011098763905465603, 0.02012040466070175, 0.007996926084160805, -0.009866337291896343, 0.006653719116002321, 0.030104445293545723, 0.005597847979515791, -0.0026050591841340065, -0.006414850242435932, 0.005452449433505535, 0.021214354783296585, 0.015661511570215225, -0.0072214663960039616, 0.013072031550109386, -0.008322342298924923, 0.027653438970446587, 0.005805560387670994, 0.005511301103979349, -0.0003154631413053721, -0.006369845476001501, 0.020549675449728966, 3.1670726457377896e-05, -0.0005153862875886261, 0.03071373514831066, -0.03226465359330177, -0.014318305067718029, 0.018389467149972916, 0.010108668357133865, -0.005521686747670174, -0.03287394344806671, -0.006584481801837683, 0.0018330615712329745, 0.006207137834280729, 0.024676227942109108, 0.019469572231173515, 0.0024890864733606577, -0.016187716275453568, 0.009298590011894703, -0.004188865888863802, -0.025950197130441666, -0.013535922393202782, -0.015439951792359352, 0.009741709567606449, -0.04669373854994774, 0.008183866739273071, 0.001936917775310576, -0.014034431427717209, 0.023734599351882935, 0.008440045639872551, -0.020854320377111435, 0.017614008858799934, 0.03866219520568848, -0.0071383812464773655, -0.012123477645218372, 0.00040092814015224576, -0.013591311872005463, -0.02577017992734909, 0.010884127579629421, -0.018167907372117043, -0.009194733574986458, 0.029328985139727592, -0.025645552203059196, -0.020715845748782158, -0.011015678755939007, -0.01743399165570736, -0.004441582597792149, 0.0028629687149077654, 0.004375807009637356, -0.001000481192022562, -0.007976154796779156, 0.0014513900969177485, -0.001772478804923594, 0.00632830336689949, -0.00912549626082182, 0.01422137301415205, 0.0014738922473043203, 0.018680265173316002, -0.014996832236647606, -0.002665641950443387, -0.006736804265528917, 0.01574459671974182, -0.021020490676164627, -0.060762789100408554, -0.009083953686058521, -0.016990870237350464, 0.0075884247198700905, 0.00235234247520566, 1.6362757378374226e-05, -0.016367733478546143, 0.0043619596399366856, -0.016367733478546143, 0.007713052444159985, 0.0032195416279137135, 0.012483512982726097, 0.001995769562199712, -0.005719013512134552, -0.004064238630235195, 0.010454855859279633, -0.03697279840707779, -0.02012040466070175, -0.014103669673204422, 0.0021307826973497868, -0.025590162724256516, -0.01398596540093422, -0.011375713162124157, 0.056026946753263474, -0.002451005857437849, -0.012033469043672085, -0.00936090387403965, -0.013189734891057014, -0.045807499438524246, -0.016644684597849846, -0.02309761382639408, 0.02247447706758976, -0.005213580094277859, 0.03337245434522629, -0.0003338543465360999, 0.01974652148783207, 0.0033458999823778868, 0.002044235821813345, -0.013535922393202782, 0.0007629101746715605, -0.00786537490785122, -0.004784307908266783, 0.0027210318949073553, 0.0031416495330631733, -0.037886735051870346, -0.00519280880689621, 0.007962306961417198, -0.020688151940703392, 0.003273200709372759, 0.023983852937817574, 0.0014929325552657247, 0.004351573996245861, 0.001603712560608983, 0.01545379962772131, 0.016783159226179123, 0.01809867098927498, 0.0255624670535326, -0.03547726944088936, -0.006082510109990835, 0.005736323073506355, -0.018223296850919724, -0.01881873980164528, -0.02313915640115738, 0.017600160092115402, 0.013951347209513187, 0.003544957609847188, -0.0012055971892550588, -0.007089915219694376, 0.008737766183912754, -0.016935480758547783, 0.03090759925544262, 0.0012384849833324552, -0.007581501267850399, 0.017170889303088188, -0.0005037024966441095, -0.019261859357357025, -0.024454668164253235, 0.008834699168801308, -0.022460630163550377, 0.014622949995100498, 0.0316830575466156, -0.046832215040922165, -0.010738728567957878, 0.0065914057195186615, 0.002613713964819908, -0.027431879192590714, -0.011278781108558178, -0.003316473914310336, -0.0019317249534651637, -0.019884996116161346, 0.02849813550710678, 0.04334264621138573, -0.02268218994140625, 0.01123723853379488, -0.004285798408091068, 0.015536884777247906, -0.0020338501781225204, -0.015384562313556671, 0.006989520974457264, -0.021837493404746056, 0.014027507975697517, 0.02309761382639408, -0.023623818531632423, -0.020757388323545456, 0.017835568636655807, 0.010115591809153557, -0.03924378752708435, 0.020591218024492264, 0.19907152652740479, -0.022723732516169548, 0.010177905671298504, 0.030159834772348404, 0.029965970665216446, 0.005580538883805275, 0.011015678755939007, 0.01417290698736906, 0.001807097578421235, -0.02012040466070175, 0.006210599560290575, -0.016478514298796654, -0.0026985297445207834, -0.0021896343678236008, 0.019981928169727325, -0.026725657284259796, -0.032015398144721985, -0.05519609898328781, -0.013438989408314228, 0.008924707770347595, -0.0032005012035369873, -0.023069920018315315, -0.00465621892362833, -0.022654494270682335, 0.02880278043448925, 0.0020044243428856134, -0.0024769699666649103, -0.006854508072137833, 0.018735654652118683, 0.013660549186170101, -0.01147264614701271, -0.005403983406722546, 0.013418218120932579, 0.0036834324710071087, -0.017558617517352104, 0.0033130121883004904, 0.008218485862016678, 0.006653719116002321, 0.005919802468270063, 0.010434084571897984, 0.031184548512101173, 0.0052585843950510025, 0.03683432564139366, -0.02669796161353588, 0.0045904433354735374, 0.018957214429974556, -0.01093259360641241, -0.0021567465737462044, -0.005275893956422806, -0.004929706454277039, -0.0331231988966465, 0.01928955502808094, 0.024676227942109108, 0.021685170009732246, -0.0248423982411623, 0.004406963940709829, -0.024980872869491577, -0.0076507385820150375, 0.02180979773402214, 0.0006690068985335529, -0.033870961517095566, -0.016727767884731293, -0.015066069550812244, 0.028830476105213165, -0.01246966514736414, 0.022142138332128525, -0.015163002535700798, -0.022252917289733887, 0.013002793304622173, -0.0012133864220231771, -0.02401154860854149, 0.010067125782370567, -0.0212420504540205, 0.00044831252307631075, -0.027071844786405563, -0.05004483088850975, 0.022432934492826462, 0.02010655589401722, 0.022308306768536568, 0.04298261180520058, 0.0025600548833608627, -0.030270613729953766, 0.003210886847227812, -0.020923558622598648, -0.023236088454723358, -0.03550496697425842, 0.017627855762839317, -0.011590349487960339, -0.008093858137726784, -0.011964231729507446, -0.017572466284036636, -0.006130976602435112, -0.005538996309041977, -0.0044208113104105, -0.004805079195648432, 0.004621599800884724, 0.0067125712521374226, 0.00435849791392684, 0.012331190519034863, -0.033040113747119904, -0.014678340405225754, 0.08125707507133484, 0.015439951792359352, 0.00959631148725748, 0.01396519411355257, 0.011112610809504986, 0.002101356629282236, 0.01542610488831997, -0.009187810122966766, -0.011451874859631062, 0.000480767572298646, -0.022959139198064804, 0.011368789710104465, 0.00542821642011404, -0.0005474085919559002, 0.01218579150736332, 0.004642371088266373, -0.0145814074203372, 0.025964045897126198, -0.017447838559746742, 0.014429084956645966, -0.012940480373799801, -0.017032412812113762, 0.002695067785680294, -0.016852395609021187, -0.047912318259477615, 0.004053852986544371, 0.010994907468557358, -0.030021360144019127, 0.009250123985111713, -0.01093259360641241, -0.047136858105659485, 0.015980003401637077, 0.011431103572249413, -0.005975192412734032, -0.003846140578389168, 0.006269451230764389, -0.0032628150656819344, -0.02041120082139969, 0.018486399203538895, 0.01188114657998085, 0.024814702570438385, -0.013792100362479687, -0.00456274813041091, 0.005338207818567753, -0.013508226722478867, 0.011846528388559818, 0.021020490676164627, 0.0026033283211290836, -0.004396578297019005, -0.00010656076483428478, -0.019123384729027748, 0.00933320913463831, 0.003981153480708599, 0.04500434547662735, 0.008668528869748116, -0.019428029656410217, -0.011306475847959518, 0.020480439066886902, 0.0005383212119340897, -0.04342573136091232, -0.0017058377852663398, 0.017101651057600975, -0.009374750778079033, -0.0065671722404658794, -0.007616119924932718, -0.17680476605892181, -0.005719013512134552, 0.007443026173859835, -0.03749900311231613, 0.016187716275453568, -0.009180886670947075, 0.011389560997486115, -0.00500240596011281, -0.014802967198193073, -0.008093858137726784, 0.01390980463474989, -0.004424273036420345, 0.0021203970536589622, -0.0024129252415150404, -0.010801042430102825, -0.003690356155857444, -0.017309363931417465, 0.0043792687356472015, 0.01300971768796444, 0.0080800112336874, 0.03902222961187363, -0.006882202811539173, 0.010641796514391899, -0.023637667298316956, -0.005400521215051413, 0.0023488805163651705, -0.0006850180798210204, 0.032375432550907135, 0.000562987057492137, -0.04409040883183479, -0.002925282344222069, -0.0037630556616932154, 0.021283593028783798, -0.001390807330608368, 0.016492361202836037, -0.0004976441850885749, 0.019511114805936813, -0.0022121365182101727, -0.028608916327357292, -0.002264064736664295, 0.018015585839748383, 0.02313915640115738, 0.01036484632641077, -0.017627855762839317, -0.01629849709570408, -0.01908184215426445, 0.01455371268093586, -0.0161600224673748, 0.015910767018795013, -0.013868262059986591, 0.022460630163550377, 0.00023086363216862082, 0.014401390217244625, 0.0033424380235373974, 0.011451874859631062, 0.011147229932248592, 0.011348018422722816, -0.01120954379439354, -0.01182575710117817, -0.00043078677845187485, -0.0037838267162442207, -0.007131457794457674, 0.006352536380290985, 0.022197527810931206, -0.01933109760284424, -0.021366678178310394, -0.013508226722478867, 0.01985730230808258, -0.02849813550710678, 0.01084950938820839, -0.01177729107439518, -0.0004712474183179438, -0.008571596816182137, -0.019995776936411858, 0.03049217350780964, 0.009547844529151917, -0.014082898385822773, 0.026670267805457115, 0.03226465359330177, -0.00340821361169219, -0.005082028917968273, 0.016603142023086548, 0.007346093654632568, 0.001196942524984479, 0.005348593462258577, -0.0011891532922163606, 0.016575446352362633, 0.022765275090932846, 1.2427582078089472e-05, -0.002502933843061328, 0.03821907564997673, -0.02118666097521782, -0.0017698823940008879, 0.010427160188555717, 0.021311288699507713, 0.016436971724033356, 0.002916627796366811, -0.015758443623781204, 0.01671392098069191, -0.009097801521420479, 0.0056290049105882645, 0.004424273036420345, 0.0024769699666649103, -0.009021639823913574, 0.04619522765278816, 0.012476589530706406, -0.003558805212378502, 0.025700943544507027, 0.042096372693777084, 0.012642758898437023, -0.033870961517095566, -0.013826719485223293, -0.003880759235471487, 0.00410231901332736, 0.014276762492954731, 0.015190697275102139, -0.012926632538437843, -0.024870093911886215, 0.010711033828556538, -0.004555824212729931, 0.03949304297566414, -0.02273757942020893, -0.013951347209513187, 0.020632760599255562, -0.003465334651991725, -0.03791442885994911, -0.12185791879892349, -0.006310993805527687, 0.020480439066886902, 0.017157040536403656, -0.0101017439737916, 0.03132302314043045, -0.0006240025395527482, 0.03976999223232269, -0.013805948197841644, 0.045225903391838074, -0.009215504862368107, -0.006719494704157114, 0.016326190903782845, -0.0057120900601148605, 0.004157708957791328, -0.01491374708712101, 0.025534773245453835, -0.010060202330350876, -0.014152135699987411, 0.04370268061757088, 0.0034826439805328846, -0.012441970407962799, 0.02448236383497715, -0.02277912199497223, -0.01033715158700943, -0.005663623567670584, -0.031156854704022408, 0.025784026831388474, 0.004715070594102144, 0.007166076451539993, 0.024703923612833023, -0.03575422242283821, 0.0031191473826766014, -0.02988288551568985, 0.01881873980164528, 0.008675453253090382, 0.00048552764928899705, -0.0255624670535326, 0.03353862091898918, 0.008827775716781616, -0.011382637545466423, -0.00394653482362628, 0.008516206406056881, -0.00784460362046957, -0.005542458035051823, -0.020494285970926285, -0.017087804153561592, 0.026614876464009285, 0.01578613929450512, -0.010870279744267464, -0.029910579323768616, 0.012213487178087234, -0.04337034001946449, 0.016063088551163673, 0.0070206779055297375, 0.023208394646644592, 0.02689182758331299, 0.017060108482837677, 0.015273782424628735, 0.01062102522701025, -0.024288497865200043, -0.019261859357357025, -0.005293203517794609, 0.012511207722127438, -0.01021252479404211, 0.005815946031361818, -0.0031970394775271416, -0.019580351188778877, 0.012054240331053734, -0.01595230959355831, -0.008523130789399147, 0.015343019738793373, -0.004614676348865032, -0.003480912884697318, -0.008924707770347595, -0.00794153567403555, -0.022502172738313675, -0.006349074654281139, -0.00010526256664888933, 0.010177905671298504, -0.031156854704022408, -0.014387542381882668, 0.016797006130218506, -0.0058090221136808395, -0.016949327662587166, 0.008356960490345955, 0.006972211413085461, -0.017475532367825508, 0.012808929197490215, -0.03749900311231613, -0.008086934685707092, 0.05134649574756622, -0.0006819888949394226, -0.015564579516649246, 0.0011294359574094415, -0.00407462427392602, -0.016672378405928612, -0.0014133094809949398, -0.003686894429847598, 0.01488605234771967, -0.011874223127961159, -0.009478607214987278, -0.06657873839139938, 0.02721031941473484, 0.02118666097521782, -0.021588237956166267, 0.015827681869268417, 0.022349849343299866, 0.016880091279745102, 0.007387636229395866, -0.021103575825691223, -0.00435849791392684, -0.025964045897126198, -0.005497453734278679, -0.02514704316854477, -0.008599291555583477, -0.008959326893091202, -0.021518999710679054, 0.004839697852730751, -0.013459760695695877, 0.007830755785107613, 0.00756072998046875, 0.0038149836473166943, -0.010344075970351696, 0.03589269518852234, 0.02175440825521946, -0.004535053391009569, -0.010773347690701485, -0.05358978733420372, 0.026268688961863518, -0.010794118978083134, -0.0194418765604496, -0.010967212729156017, -0.012116554193198681, -0.006861431524157524, -0.003766517387703061, 0.002546207280829549, 0.009797099977731705, 0.006037505809217691, 0.024551600217819214, 0.006283299066126347, 0.04242870956659317, -0.013868262059986591, -0.04325956106185913, -0.004060776438564062, -0.03378787636756897, 0.0007018946926109493, 0.0002853881160262972, 0.0037907506339251995, -0.01682470180094242, 0.019054146483540535, 0.00974863301962614, 0.023845378309488297, 0.008246180601418018, -0.006134438328444958, -0.03539418429136276, -0.01183268055319786, -0.013930575922131538, 0.010759499855339527, 0.010641796514391899, 0.014830662868916988, 0.005407445132732391, 0.04157016798853874, -0.0023973467759788036, 0.013335133902728558, -0.0019455724395811558, -0.0019628817681223154, 0.0019819221924990416, -0.01815406046807766, -0.0015543808694928885, 0.009714014828205109, -0.0015907305059954524, 0.012324267067015171, -0.0038184456061571836, 0.02093740552663803, 0.021879035979509354, 0.024703923612833023, -0.02762574329972267, 0.0010731805814430118, 0.005639390554279089, -0.012975098565220833, 0.02787499874830246, 0.023776141926646233, 0.01661698892712593, -0.013480531983077526, 0.01002558320760727, 0.018527941778302193, -0.003396097104996443, 0.0047116088680922985, 0.023831531405448914, -0.021062033250927925, 0.0006871817167848349, 0.018541790544986725, 0.006961825769394636, -0.0039292252622544765, 0.005757094360888004, -0.011749595403671265, 0.012220410630106926, -0.005037024617195129, 0.004123090300709009, 0.025493230670690536, 0.02036965824663639, 0.01917877420783043, -0.005088952835649252, -0.0011675165733322501, 0.0013033950235694647, -0.0324862115085125, 0.011313400231301785, -0.05414368957281113, -0.03436947241425514, 0.00931243784725666, 0.011583426035940647, 0.014117516577243805, 0.007602272555232048, 0.005971730221062899, 0.0026466017588973045, -0.02360997162759304, 0.025784026831388474, 0.010794118978083134, -0.02088201604783535, -0.030326005071401596, 0.026988759636878967, 0.026808742433786392, 0.010434084571897984, 0.04938015341758728, -0.016478514298796654, 0.038080599159002304, 0.0003704203700181097, 0.036751240491867065, -0.023679209873080254, 0.028304271399974823, 0.02365151420235634, -0.007934612222015858, -0.011998850852251053, 0.0034861057065427303, -0.02041120082139969, -0.005857488606125116, -0.010475627146661282, -0.016880091279745102, 0.02406693808734417, 0.02705799601972103, 0.08452508598566055, 0.013058183714747429, 0.0010281761642545462, -0.008440045639872551, 0.0034428322687745094, -0.0039499965496361256, 0.009180886670947075, 0.023997701704502106, -0.007519187405705452, 0.003160689724609256, 0.007117610424757004, -0.0048500834964215755, -0.010821813717484474, -0.020328115671873093, 0.006203675642609596, -0.006335226818919182, -0.0005863546975888312, -0.0010740460129454732, -0.018846435472369194, -0.007117610424757004, 0.04890933632850647, -0.013058183714747429, 0.02201751060783863, 0.004015772137790918, -0.028096558526158333, 0.0032489674631506205, 0.036031171679496765, 0.011361866258084774, -0.017309363931417465, -0.02448236383497715, 0.03932687267661095, 0.00787229835987091, -0.031239939853549004, -0.021117422729730606, -0.014899900183081627, -0.012892013415694237, 0.005781327374279499, -0.015869224444031715, 0.002992789028212428, 0.012012697756290436, 0.007436102256178856, -0.009575540199875832, -0.010441008023917675, 0.004213098902255297, 0.00915319100022316, -0.0004184538556728512, -0.014733729884028435, -0.0151214599609375, -0.01928955502808094], 'text': 'performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.', 'SimilarityScore': 0.8395394297757695}), 0.8395394297757695)\n", - "(Document(page_content='2 GPT-4 Observed Safety Challenges\\nGPT-4 demonstrates increased performance in areas such as reasoning, knowledge retention, and\\ncoding, compared to earlier models such as GPT-2[ 22] and GPT-3.[ 10] Many of these improvements\\nalso present new safety challenges, which we highlight in this section.\\nWe conducted a range of qualitative and quantitative evaluations of GPT-4. These evaluations\\nhelped us gain an understanding of GPT-4’s capabilities, limitations, and risks; prioritize our\\nmitigation efforts; and iteratively test and build safer versions of the model. Some of the specific\\nrisks we explored are:6\\n•Hallucinations\\n•Harmful content\\n•Harms of representation, allocation, and quality of service\\n•Disinformation and influence operations\\n•Proliferation of conventional and unconventional weapons\\n•Privacy\\n•Cybersecurity\\n•Potential for risky emergent behaviors\\n•Interactions with other systems\\n•Economic impacts\\n•Acceleration\\n•Overreliance', metadata={'id': '4a3bcc96-5927-477b-bc8c-f499ffceb310', 'embedding': [3.201593426638283e-05, -0.01149508636444807, -0.006550004705786705, -0.034101177006959915, -0.004657018929719925, 0.020493626594543457, -0.018833834677934647, 0.004362097475677729, -0.02326451987028122, -0.022660959511995316, 0.025678761303424835, 0.029574472457170486, -0.047050297260284424, -0.020493626594543457, 0.01711917482316494, 0.03264714404940605, 0.038161493837833405, -0.008086340501904488, -0.0072015756741166115, -0.0050582499243319035, 0.0031618347857147455, 0.01932765729725361, -0.02500661462545395, -0.015720011666417122, 0.0072015756741166115, 0.010905243456363678, 0.03187897428870201, -0.030479812994599342, 0.002822331851348281, 0.011693987064063549, 0.009903881698846817, -0.019492264837026596, -0.02755803056061268, -0.004879924934357405, -0.018532054498791695, -0.019890066236257553, 0.0030623844359070063, -0.020960014313459396, 0.00266629783436656, -0.000522542919497937, 0.0035699240397661924, 0.01788734272122383, 0.019135616719722748, 0.008662466891109943, 0.006735187955200672, 0.024869441986083984, 0.001978718675673008, -0.026789862662553787, -0.02666640654206276, 0.018230274319648743, 0.01909446343779564, 0.02075425535440445, -0.003881992306560278, 0.0019667160231620073, -0.003103536320850253, -0.012537600472569466, -0.001624641241505742, 0.00995189230889082, 0.016556765884160995, 0.01788734272122383, 0.01244157925248146, -0.005867569707334042, -0.017914777621626854, 0.040191650390625, -0.020493626594543457, -0.015665141865611076, -0.021357815712690353, 0.02067195251584053, -0.007908016443252563, -0.012222102843225002, 0.0011471081525087357, 0.020617082715034485, 0.00737990066409111, -0.01684482954442501, 0.03253740444779396, -0.01746210642158985, 0.0022444911301136017, -0.019231636077165604, -0.008545869961380959, 0.012777653522789478, 0.024704834446310997, -0.021975094452500343, 0.003437895094975829, 0.023895515128970146, 0.005078825633972883, 0.009300320409238338, 0.006419690325856209, 0.012084930203855038, -0.00858016312122345, -0.0016589344013482332, 0.009807860478758812, -0.007908016443252563, 0.013257757760584354, 0.02700933814048767, -0.008086340501904488, 0.02126179449260235, -0.008751628920435905, 0.01816168799996376, -0.0294098649173975, -0.021700749173760414, -0.011467652395367622, 0.002376520074903965, -0.035143692046403885, -0.012661056593060493, -0.013484093360602856, -0.008916236460208893, 0.01684482954442501, 0.007064403034746647, 0.006632308475673199, 0.015006712637841702, -0.004924505949020386, 0.025719914585351944, -0.008875085040926933, 0.005288014188408852, -0.02932756021618843, 0.0038134059868752956, 0.016104094684123993, 0.007798277772963047, -0.0022119125351309776, -0.016323572024703026, 0.02396410144865513, -0.01573372818529606, 0.018641794100403786, -0.0032338504679501057, 0.009732414968311787, -0.006159062031656504, 0.0047496105544269085, -0.015198754146695137, -0.005613799672573805, -0.015623990446329117, 0.002990368753671646, 0.008573304861783981, 0.0280930045992136, -0.019848914816975594, -0.044690921902656555, 0.033991437405347824, -0.04027395695447922, -0.01603550836443901, -0.0275854654610157, -0.01839488185942173, 0.002345656044781208, 0.02233174443244934, -0.006316810846328735, 0.008998540230095387, 0.003665945027023554, 0.01750325784087181, 0.0195882860571146, 0.05094600468873978, -0.0031223974656313658, 0.002926926128566265, -0.0031258268281817436, -0.014073937200009823, -0.01336749643087387, 0.01735236868262291, -0.022551219910383224, -0.016680222004652023, -0.008683042600750923, 0.01866922900080681, -0.02032901905477047, -0.03923144191503525, -0.00860073883086443, -0.007503356318920851, -0.0090602682903409, 0.010157651267945766, 0.0017309501999989152, 0.01777760498225689, 0.016131529584527016, 0.00958152487874031, 0.007791419047862291, -0.003028091276064515, -0.015116450376808643, 0.026693841442465782, -0.015212471596896648, 0.029382430016994476, 0.005860710982233286, 0.012709066271781921, 0.012084930203855038, -0.0072975968942046165, -0.016392158344388008, -0.010150793008506298, -0.008929953910410404, -0.0005645520868711174, 0.022894153371453285, -0.0003515054995659739, -0.006738617550581694, -0.03218075633049011, 0.03404630720615387, 0.014540324918925762, -0.028257612138986588, -0.006896366365253925, 0.017420955002307892, -0.00716728251427412, -0.03829866647720337, -0.008223514072597027, -0.6140955090522766, -0.019698023796081543, 0.01573372818529606, -0.02086399495601654, 0.010754353366792202, 0.025953108444809914, 0.0176678653806448, 0.018340013921260834, -0.004807909019291401, 0.05146726220846176, -0.006111051421612501, -0.009560949169099331, -0.01312058512121439, -0.025322113186120987, -0.0028789157513529062, -0.028778869658708572, 0.0029114943463355303, -0.030232900753617287, 0.01784619130194187, 0.023936666548252106, -0.013154878281056881, 0.014581476338207722, -0.0018792683258652687, -0.007338748779147863, -0.00879278127104044, 0.02569247968494892, 0.00022912156418897212, -0.010576028376817703, -0.011371631175279617, 0.011014982126653194, -0.00948550458997488, 0.0228804349899292, 0.002378234639763832, 0.017009437084197998, 0.03972526267170906, -0.028079288080334663, -0.024320749565958977, 0.015445665456354618, 0.01072691846638918, 0.04038369283080101, -0.024663683027029037, -0.01021937932819128, 0.023209650069475174, 0.03289405629038811, -0.0037448194343596697, 0.031247980892658234, 0.02364860288798809, -0.0008749057305976748, -0.020342737436294556, -0.0012954262783750892, -0.00013492237485479563, 0.006508852820843458, 0.015486817806959152, 0.009163147769868374, -0.006248224526643753, 0.006683748215436935, 0.031741801649332047, -0.011508803814649582, 0.0011573961237445474, -0.015212471596896648, -0.00807262398302555, 0.020685669034719467, -0.029848817735910416, -0.05152213200926781, -0.028449654579162598, 0.03393656760454178, -0.013168595731258392, 0.004015735816210508, 0.003223562613129616, -0.022578654810786247, 0.007715974003076553, 0.02886117249727249, -0.016200115904211998, -0.01028110645711422, -0.0003193556040059775, 0.030589550733566284, 0.03187897428870201, -0.0359392911195755, 0.008010895922780037, 0.02709164284169674, 0.01565142534673214, -0.019451113417744637, -0.0045095584355294704, 0.015706293284893036, 0.025761066004633904, -0.005685815587639809, -0.0009061982855200768, 0.019533416256308556, 0.024128708988428116, -0.008751628920435905, 0.015829749405384064, 0.017983363941311836, -0.0310285035520792, -0.032948922365903854, 0.02017812989652157, -0.006865502335131168, -0.011934040114283562, -0.007503356318920851, 0.0008311818819493055, -0.015788597986102104, -0.008628173731267452, -0.020575931295752525, 0.02303132601082325, 0.03245510160923004, 0.02422473020851612, 0.008244089782238007, -0.00490735936909914, -0.009691263549029827, -0.0028737715911120176, -0.01452660746872425, -0.017640432342886925, -0.002530839527025819, -0.007434769533574581, -0.014499172568321228, 0.009519797749817371, -0.03687206655740738, 0.02307247743010521, -0.01333320327103138, -0.0372561514377594, -0.02743457444012165, 0.030424943193793297, -0.0029012062586843967, 0.01189974695444107, -0.028422219678759575, 0.011145295575261116, 0.02315478026866913, 0.0253083948045969, -0.01956085115671158, 0.018710380420088768, 0.02233174443244934, -0.009732414968311787, -0.022784413769841194, 0.011028698645532131, 0.00556578952819109, 0.014855822548270226, 0.0019444255158305168, 0.019848914816975594, -0.0017369515262544155, 0.02315478026866913, -0.00811377540230751, -0.02628232166171074, -0.0018038232810795307, -0.0034121752250939608, 0.004790762439370155, -0.019067028537392616, -0.011069850996136665, -0.0069306595250964165, 0.002038731938228011, 0.0008067479357123375, -0.015898335725069046, -0.017901059240102768, -0.020918862894177437, 0.002997227245941758, 0.029492167755961418, -0.0040500289760529995, -0.021275512874126434, 0.008751628920435905, -0.031001068651676178, -0.01242100354284048, -0.03152232617139816, 0.0005645520868711174, 0.022976456210017204, 0.006213931366801262, 0.0006511425017379224, -0.013106867671012878, -0.00978042557835579, -0.016789959743618965, 0.030315205454826355, -0.0346224345266819, -0.023662321269512177, -0.009958750568330288, -0.04471835494041443, 0.025335829704999924, 0.028778869658708572, -0.00606647040694952, 0.018257709220051765, 0.0005752687575295568, -0.0007424482028000057, 0.005493773613125086, 0.001172828022390604, 0.007331890054047108, 0.0055452133528888226, -0.01141964178532362, -0.007421052549034357, 0.022743262350559235, 0.023401692509651184, 0.02673499286174774, 0.02128922939300537, -0.006858643610030413, 0.02864169515669346, 0.02071310393512249, 0.017187761142849922, -0.018340013921260834, 0.013264616951346397, -0.004262647125869989, 0.006508852820843458, 0.0033538767602294683, 0.007434769533574581, 0.009218016639351845, 0.0385730117559433, 0.029492167755961418, 0.0019804334733635187, 0.0024382479023188353, -0.02148127183318138, 0.01595320552587509, -0.004314086865633726, -0.008710477501153946, -0.012928543612360954, -0.0003549348039086908, 0.009320897050201893, -0.009355190210044384, -0.01588461920619011, -0.01743467152118683, -0.022660959511995316, 0.004739322699606419, 0.005706391762942076, -0.018655510619282722, 0.014581476338207722, -0.016858546063303947, -0.016858546063303947, 0.01688598096370697, 0.018998442217707634, 0.018463468179106712, 0.01769530028104782, -0.018230274319648743, 0.02886117249727249, -0.0011719707399606705, 0.01147451065480709, 0.02098744921386242, -0.023895515128970146, -0.010555452667176723, 0.01573372818529606, 0.02098744921386242, 0.014924408867955208, 0.01854577288031578, 0.006834638305008411, 0.020109543576836586, 0.0063511040061712265, 0.02241404727101326, -0.0044786944054067135, -0.0017643860774114728, -0.007558225188404322, -0.010390845127403736, -0.020644517615437508, 0.020589647814631462, -0.006498564966022968, 0.019300222396850586, -0.00402945326641202, -0.03602159768342972, 0.013394931331276894, 0.004969087429344654, 0.0009953606640920043, -0.03212588652968407, -0.00782571267336607, 0.004684453830122948, -0.001239699893631041, -0.005219427868723869, 0.00395057862624526, -0.0063065229915082455, 0.023909231647849083, 0.009746132418513298, 0.006762622855603695, 0.021152056753635406, 0.018998442217707634, 0.01526734046638012, -0.020150694996118546, -0.00954723171889782, -0.02171446569263935, -0.011282469145953655, -0.0024691116996109486, -0.012962836772203445, -0.01673508994281292, 0.01335377898067236, -0.021001167595386505, 0.005610370542854071, 0.012338699772953987, -0.0033401595428586006, 0.014718648977577686, 0.01310000941157341, 0.009519797749817371, -0.01023995503783226, -0.04400505870580673, 0.009375765919685364, 0.020466193556785583, -0.006361391860991716, 0.003991730511188507, -0.04436170682311058, -0.0034121752250939608, 0.005137124098837376, 0.011124719865620136, -0.017475824803113937, -0.0008011753088794649, -0.0039300029166042805, 0.015171320177614689, 0.0056069414131343365, -0.008381262421607971, 0.014499172568321228, -0.022043680772185326, 0.005017098039388657, -0.0195882860571146, -0.003290434367954731, 0.0027125936467200518, -0.022743262350559235, -0.0009859299752861261, 0.008902519941329956, -0.004910788964480162, -0.013806450180709362, -0.014101371169090271, -0.0017849620198830962, -0.03648798540234566, 0.035610079765319824, 0.0012782796984538436, 0.0014317418681457639, -0.016049226745963097, -0.0032887195702642202, 0.01170770451426506, -0.022057397291064262, -0.011714563705027103, 0.024677399545907974, 0.014005349949002266, 0.0004903930239379406, -0.013456658460199833, -0.00666660163551569, 0.02847708761692047, 0.050644226372241974, 0.03473217040300369, -0.015816032886505127, 0.01456775888800621, -0.003919715061783791, -0.0035939293447881937, -0.022043680772185326, -0.030424943193793297, 0.016090378165245056, 0.015185036696493626, 0.02179677039384842, -0.016584200784564018, 0.007078120484948158, -0.010137075558304787, 0.02128922939300537, 0.009211158379912376, -0.0002038303209701553, -0.029272690415382385, -0.012626763433218002, -0.024759704247117043, 0.020809125155210495, -0.014073937200009823, 0.014581476338207722, 0.00853901170194149, -3.365021984791383e-05, -0.005874428432434797, 0.02399153634905815, 0.025719914585351944, -0.017558127641677856, -0.014608911238610744, -0.013065716251730919, 0.005617229267954826, 0.00605618255212903, -0.0072221518494188786, -0.009259168989956379, 0.011769432574510574, 0.014444303698837757, -0.021769335493445396, 9.082344331545755e-05, 0.0008234659326262772, 0.0074484869837760925, 0.016680222004652023, 0.017366085201501846, -0.016940850764513016, 0.0034996229223906994, -0.0013194315833970904, 0.016392158344388008, 0.026831014081835747, -0.024211011826992035, -0.026337191462516785, 0.0019615720957517624, 0.01592577062547207, -0.030123163014650345, 0.005401181988418102, 0.02032901905477047, -0.0008003179682418704, -0.009142572060227394, 0.008943671360611916, -0.02696818672120571, -0.0058469935320317745, -0.01286681555211544, -0.02275697886943817, 0.00827152468264103, -0.008621315471827984, -0.02543185092508793, 0.01886126957833767, -0.015596555545926094, -0.014499172568321228, -0.03733845800161362, 0.01696828380227089, -0.0035664946772158146, -0.0042214952409267426, -0.021357815712690353, -0.009128854610025883, 0.01149508636444807, 0.005997884087264538, 0.011104144155979156, -0.0033933138474822044, 0.0036865209694951773, -0.005901862867176533, 0.002184478100389242, -0.035994160920381546, -0.009437493979930878, -0.02743457444012165, 0.028888607397675514, -0.01805195026099682, -0.008648749440908432, -0.01973917707800865, -0.00881335698068142, 0.011159013025462627, 0.009163147769868374, -0.012043777853250504, -0.006766051985323429, 0.004464976955205202, 0.012722783721983433, 0.022715827450156212, 0.009163147769868374, 0.01684482954442501, 0.015198754146695137, -0.016172681003808975, -0.004852490499615669, -0.030123163014650345, -0.016748808324337006, -0.04702286049723625, 0.01688598096370697, 0.007592518348246813, 0.002968078013509512, 0.01288739126175642, 0.02275697886943817, -0.024293316528201103, 0.0267761442810297, -0.011975191533565521, 0.005538354627788067, -0.0049313646741211414, 0.0036350812297314405, 0.0077571258880198, 0.0077502671629190445, 0.015939487144351006, -0.006111051421612501, -0.02422473020851612, -0.026954470202326775, -0.026981905102729797, 0.031741801649332047, 0.0028857742436230183, 0.006940947379916906, 0.024197295308113098, 0.025088919326663017, -0.009677546098828316, -0.06606245785951614, 0.015308492816984653, -0.019423678517341614, 0.02364860288798809, -0.0019272788194939494, -0.014924408867955208, 0.005836705677211285, -0.01045257318764925, -0.023621167987585068, 0.0118448780849576, -0.006429978646337986, 0.007798277772963047, -0.0316869355738163, 0.003923144191503525, 0.004269505850970745, -0.0009687833953648806, 0.034649867564439774, -0.03209845349192619, -0.017914777621626854, 0.011838018894195557, -0.0034258924424648285, 0.015377079136669636, 0.009842153638601303, 0.013161737471818924, -0.030342640355229378, -0.012853098101913929, -0.01626870222389698, -0.023936666548252106, 0.011344196274876595, 0.0021776193752884865, 0.0280930045992136, 0.019135616719722748, 0.03275688365101814, -0.016954567283391953, 0.0051336949691176414, 0.03473217040300369, 0.0012405571760609746, 0.001189117319881916, 0.011762574315071106, -0.007475921418517828, -0.023401692509651184, 0.006570580415427685, -0.008854509331285954, -0.015569121576845646, 0.015445665456354618, 0.01427969615906477, 0.0005894146743230522, 0.01785990782082081, -0.01408765371888876, 0.01004791259765625, -0.03706410899758339, -0.04115186259150505, -0.03547290340065956, 0.01939624361693859, -0.013051998801529408, -0.011172730475664139, -0.032976359128952026, -0.003926573321223259, 0.020274151116609573, 0.00077631272142753, 0.004423825070261955, -0.0011179589200764894, 0.028202742338180542, -0.0017275208374485373, -0.015171320177614689, 0.008497859351336956, 0.019190484657883644, -0.010624038986861706, -0.028202742338180542, -0.045239612460136414, -0.014951842837035656, 0.02477342076599598, 0.0067489054054021835, 0.037777408957481384, -0.0018947003409266472, -0.000686293060425669, -0.024197295308113098, 0.017750170081853867, -0.006622020620852709, -0.00037379609420895576, 0.011543096974492073, -0.026309756562113762, 0.013196030631661415, -0.01669393852353096, -0.0011393921449780464, 0.006364821456372738, -0.009807860478758812, 0.005737255327403545, -0.02283928357064724, 0.03838096931576729, -0.02875143475830555, -0.019876349717378616, 0.0077571258880198, 0.015610272996127605, 0.018710380420088768, 0.01641959324479103, 0.013086291961371899, 0.023278236389160156, 0.006128198001533747, 0.0008804783574305475, 0.015431948006153107, 0.030671855434775352, -0.013209748081862926, 0.007393618114292622, 0.011405924335122108, -0.01116587221622467, -0.010267389938235283, 0.0037928300444036722, 0.008888802491128445, -0.03478704020380974, -0.04757155105471611, 0.01595320552587509, 0.009204300120472908, 0.002568562049418688, -0.024992898106575012, -0.014115088619291782, -0.0322904959321022, -0.0038957095239311457, 0.0024399624671787024, 0.024951744824647903, 0.01565142534673214, -0.00286005437374115, -0.008854509331285954, 0.023950383067131042, -0.0022393472027033567, -0.002558274194598198, -0.008086340501904488, -0.005792124662548304, 0.006241365801542997, 0.011625400744378567, -0.0034653297625482082, -0.0006575724692083895, 0.0033778820652514696, 0.025747347623109818, -0.0019512841245159507, -0.012709066271781921, 0.019121898338198662, -0.020493626594543457, 0.025171222165226936, 4.68584694317542e-05, -0.01975289359688759, 0.030205465853214264, -0.012667914852499962, -0.017805039882659912, -0.006495135370641947, -0.011364772915840149, -0.0010605177376419306, -0.02562389336526394, -0.005850423127412796, 0.00723586929962039, 0.007818853482604027, 0.010877808555960655, 0.01839488185942173, 0.03700924292206764, -0.016872262582182884, 0.011522521264851093, 0.0006867217016406357, -0.033333007246255875, -3.190876668668352e-05, -0.00453013414517045, 0.034018874168395996, -0.04238641634583473, 0.012331841513514519, 0.00321670388802886, -0.01584346778690815, 0.011152154766023159, 0.010836657136678696, -0.016172681003808975, 0.008196079172194004, 0.023676037788391113, -0.007064403034746647, 0.00877220556139946, 0.003696809057146311, 0.011488228105008602, -0.0228804349899292, 0.00440667849034071, -0.019121898338198662, -0.012537600472569466, 0.016789959743618965, -0.01850462146103382, -0.01843603327870369, -0.030013425275683403, -0.020301586017012596, 0.021769335493445396, 0.011591107584536076, 0.0015226189279928803, 0.027023056522011757, -0.020809125155210495, -0.0018158259335905313, 0.006474559661000967, -0.009108278900384903, 0.006409402471035719, -0.003587070619687438, 0.021467555314302444, 0.01603550836443901, -0.007798277772963047, -0.0019444255158305168, -0.0228804349899292, 0.019080746918916702, -0.012962836772203445, -0.05212569236755371, -0.00667688949033618, 0.0021741900127381086, -0.003070957725867629, -0.008799639530479908, -0.015610272996127605, -0.006851784884929657, 0.0021124621853232384, -0.013264616951346397, 0.020932581275701523, 0.030150597915053368, 0.0018158259335905313, 0.0038922803942114115, -0.015445665456354618, 0.002902920823544264, 0.005366888828575611, -0.017132891342043877, -0.026762427762150764, -0.005706391762942076, -0.0099724680185318, -0.018490903079509735, -0.007839430123567581, 0.010397703386843204, 0.055280666798353195, 0.002047305228188634, -0.014361999928951263, -0.022304309532046318, -0.01956085115671158, -0.03223562613129616, -0.017338650301098824, -0.023278236389160156, 0.025335829704999924, 0.016817394644021988, 0.023950383067131042, 0.0029183528386056423, 0.017297498881816864, 0.016954567283391953, 0.0031978425104171038, -0.02979394793510437, 0.0030640990007668734, 0.002451965119689703, -0.021316664293408394, 0.005123406648635864, -0.0014025926357135177, -0.046967990696430206, -0.005654951557517052, 0.012613045983016491, -0.01172828022390604, -0.008888802491128445, 0.0181205365806818, -6.628235860262066e-05, -0.012770794332027435, -0.006916942074894905, -0.0063956850208342075, 0.022304309532046318, 0.000931918213609606, 0.008360686711966991, -0.054814279079437256, 0.010548594407737255, 0.006364821456372738, -0.010233096778392792, -0.019876349717378616, -0.013456658460199833, 0.015912054106593132, 0.0035973587073385715, -0.004194060806185007, 0.001626355922780931, -0.016488179564476013, 0.00641283206641674, -0.014608911238610744, 0.03053468093276024, 0.019313940778374672, 0.010630897246301174, 0.02218085341155529, 0.0029372142162173986, 0.00737990066409111, -0.016474461182951927, 0.012263255193829536, 0.01336749643087387, -0.0053737470880150795, 0.04060317203402519, -0.045075006783008575, 0.012338699772953987, 0.005970449186861515, 0.004735893569886684, -0.019766611978411674, -0.017009437084197998, 0.015898335725069046, -0.031083373352885246, -0.03423834964632988, 0.03357991948723793, 0.014211109839379787, -0.017997080460190773, 0.007496497593820095, -0.016488179564476013, 0.00225992314517498, 0.015980640426278114, -0.020548496395349503, -0.006063040811568499, -0.021206926554441452, 0.006584297865629196, -0.0025754207745194435, 0.006618591025471687, 0.01618639938533306, 0.00175581278745085, 0.015102732926607132, -0.01565142534673214, 0.01703687012195587, 0.1922615021467209, -0.009142572060227394, 0.0035699240397661924, 0.025953108444809914, 0.036268506199121475, 0.007791419047862291, 0.026254888623952866, 0.004108327440917492, -0.004958799574524164, -0.01650189608335495, 0.008518435060977936, -0.0028309051413089037, -0.0009953606640920043, 0.00021476128313224763, 0.013381213881075382, -0.011200165376067162, -0.0316869355738163, -0.022935304790735245, -0.00882707443088293, 0.01234555896371603, -0.013484093360602856, -0.017475824803113937, -0.009348331019282341, -0.016954567283391953, 0.021165775135159492, 0.004835343919694424, -0.016213834285736084, -0.0070506855845451355, 0.012318124063313007, 0.012578752823174, -0.018298860639333725, -0.01023995503783226, 0.003693379694595933, 0.00033907420584000647, -0.005819559097290039, -0.013593832030892372, 0.009087703190743923, -0.006625449750572443, -0.0029852245934307575, 0.0043278043158352375, 0.030013425275683403, 0.019300222396850586, 0.014115088619291782, -0.02318221516907215, -0.008484141901135445, 0.018696662038564682, 0.0032801462803035975, 0.003281861077994108, -0.030287770554423332, 0.009142572060227394, -0.0404934324324131, 0.012757076881825924, 0.03322327136993408, 0.009615818038582802, -0.026254888623952866, -0.00534288352355361, -0.020767973735928535, 0.0005221142782829702, 0.007263303734362125, -0.005850423127412796, -0.015623990446329117, -0.01928650587797165, -0.019108181819319725, 0.0330037921667099, -0.03179667145013809, 0.026364626362919807, 0.009766708128154278, 0.010562310926616192, 0.020932581275701523, -0.009046550840139389, -0.011810583993792534, 0.013731004670262337, -0.039780132472515106, 0.009266027249395847, -0.03772253915667534, -0.03980756923556328, 0.02001352235674858, 0.01924535445868969, 0.0004861063789576292, 0.0294098649173975, 0.004187202081084251, -0.011049275286495686, -0.001172828022390604, -0.019505983218550682, -0.011556814424693584, -0.03956065699458122, 0.026831014081835747, -0.013477235101163387, -0.016817394644021988, 0.017942212522029877, -0.030699288472533226, 0.00021090329391881824, -0.00607675826177001, 0.0029132089111953974, 0.00790115725249052, 0.02886117249727249, 0.03714641556143761, 0.007270162459462881, 0.005888145416975021, -0.04587060958147049, -0.021398968994617462, 0.09300320595502853, 0.0031686932779848576, 0.0008174646063707769, -0.0017360941274091601, 0.006772910710424185, 0.0033727381378412247, 0.02469111792743206, -1.2839863302360754e-05, -0.024595096707344055, -0.005438904277980328, -0.021206926554441452, 0.0020232999231666327, 0.01916304975748062, 0.013230323791503906, 0.020452475175261497, 0.0007613094639964402, 0.007537649478763342, 0.01236613467335701, -0.015870900824666023, 0.011234458535909653, -0.010555452667176723, 0.008840791881084442, 0.020274151116609573, -0.01735236868262291, -0.03818892687559128, -0.001005648635327816, 0.004430683795362711, -0.04331919178366661, -0.023744624108076096, 0.006604874040931463, -0.050068099051713943, 0.011618542484939098, 0.017571846023201942, -0.005826417822390795, -0.007352466229349375, -0.0034550416748970747, -0.00514055322855711, -0.007722832728177309, 0.018874987959861755, 0.011597966775298119, 0.007832570932805538, 0.004759898874908686, -0.008765346370637417, 0.007565083913505077, -0.015171320177614689, 0.006004742346704006, 0.008436131291091442, 0.0024399624671787024, -0.006896366365253925, -0.012228962033987045, -0.0030143738258630037, -0.008422414772212505, 0.01646074466407299, 0.029547037556767464, 0.017709018662571907, -0.018600640818476677, -0.02160472795367241, 0.022043680772185326, -0.0010605177376419306, -0.05421071872115135, 0.017763886600732803, 0.005702962167561054, -0.005123406648635864, -0.011035557836294174, -0.010562310926616192, -0.173935204744339, -0.01000676117837429, 0.00786686409264803, -0.04334662854671478, 0.017132891342043877, -0.010137075558304787, 0.035225994884967804, 0.000820465269498527, -0.03396400436758995, 0.007393618114292622, 0.03497908264398575, -0.006237936206161976, 0.0005619801231659949, -0.015610272996127605, -0.010768070816993713, -0.005068537779152393, -0.024320749565958977, 0.0065019940957427025, 0.02836734987795353, -0.003981442656368017, 0.03621364012360573, -0.022619806230068207, 0.012715925462543964, -0.00025677046505734324, 0.004694741684943438, -0.005788695067167282, -0.021632162854075432, 0.027064207941293716, -0.009752991609275341, -0.0385730117559433, -0.0011882600374519825, 0.005082254763692617, 0.03212588652968407, 0.002820617286488414, -0.0129079669713974, 0.008298958651721478, 0.01314116083085537, -0.004873066209256649, -0.04041112959384918, 0.006714612245559692, 0.02728368528187275, 0.03621364012360573, 0.011062991805374622, -0.009156289510428905, -0.03426578268408775, -0.012256396003067493, 0.008758488111197948, -0.03481447696685791, 0.006639167200773954, -0.00977356731891632, 0.026638971641659737, -0.016556765884160995, 0.01187917124480009, -0.0027588894590735435, 0.0027691773138940334, 0.009142572060227394, 0.004818197339773178, -0.018696662038564682, 0.0022444911301136017, -0.003628222504630685, -0.0055452133528888226, -0.015075298957526684, 0.020960014313459396, -0.007914874702692032, -0.004211207386106253, -0.023826928809285164, -0.004759898874908686, -0.003741390071809292, -0.028422219678759575, 0.016282420605421066, -0.025678761303424835, -0.003779112594202161, -0.014361999928951263, -0.017366085201501846, 0.03423834964632988, 0.02875143475830555, -0.01314116083085537, 0.006193355191498995, -0.0008761917124502361, 0.013196030631661415, 0.01835373044013977, 0.002527410164475441, -0.01615896448493004, -0.01580231450498104, -0.008010895922780037, -0.0038614163640886545, 0.004622725769877434, 0.033717092126607895, -0.007997178472578526, -0.02164587937295437, 0.02828504703938961, -0.02179677039384842, 0.013017705641686916, 0.00877220556139946, 0.021110905334353447, 0.008216654881834984, 0.020960014313459396, -0.009464927949011326, 0.013003988191485405, -0.02075425535440445, 0.017955929040908813, 0.0042523592710494995, -0.016858546063303947, -0.002601140644401312, 0.03308609873056412, -0.005613799672573805, 0.010054771788418293, 0.013724146410822868, 0.03193384408950806, -0.0009199155610986054, -0.01654304936528206, -0.011433359235525131, 0.024992898106575012, 0.005504061467945576, -0.01051430031657219, 0.012715925462543964, -0.020315302535891533, -0.020877711474895477, 0.010370269417762756, -0.008785923011600971, 0.06008172035217285, -0.023593734949827194, -0.02454022690653801, 0.011323620565235615, -0.00908084399998188, -0.03975269943475723, -0.1201634407043457, -0.03550034016370773, 0.029300125315785408, 0.022002529352903366, -0.005658381152898073, 0.022125983610749245, -0.011303044855594635, 0.032866619527339935, -0.01805195026099682, 0.04348380118608475, 0.012084930203855038, -0.024238446727395058, 0.022853000089526176, -0.012036919593811035, 0.005411469843238592, -0.005984166637063026, 0.03094620071351528, -0.023045042529702187, -0.015829749405384064, 0.04822998121380806, 0.01503414660692215, -0.000814035243820399, 0.01611781306564808, -0.01522618904709816, -0.01758556254208088, -0.015555404126644135, -0.0322904959321022, 0.012571893632411957, -0.016049226745963097, 0.013998491689562798, 0.016639068722724915, -0.0035322015173733234, -0.0018895562971010804, -0.019382527098059654, -0.012194668874144554, 0.011275609955191612, 0.007359324488788843, -0.0176678653806448, 0.017050588503479958, -0.005798983387649059, 0.004770186729729176, 0.0018449751660227776, 0.017420955002307892, 0.0034173191525042057, -0.029464732855558395, -0.015212471596896648, -0.02388179674744606, 0.023854361847043037, 0.010095923207700253, -0.008998540230095387, -0.029931120574474335, -0.006632308475673199, -0.027544312179088593, 0.0007137276115827262, 0.012256396003067493, 0.011906605213880539, 0.02311362884938717, 0.016666503623127937, 0.0012199812335893512, -0.007098696194589138, -0.018916139379143715, -0.016872262582182884, -0.0043723853304982185, -0.002184478100389242, -0.008785923011600971, 0.007914874702692032, -0.010178226977586746, -0.012276972644031048, 0.017023153603076935, -0.012084930203855038, 1.7628857676754706e-05, 0.008161786012351513, 0.0002308362309122458, 0.02079540677368641, -0.024828290566802025, -0.006172779481858015, 0.011124719865620136, -0.00475304014980793, 0.021069753915071487, -0.002633719239383936, -0.02585708722472191, -0.006680319085717201, 0.02071310393512249, -0.026145149022340775, 0.00303323520347476, 0.00860073883086443, 0.0030949630308896303, -0.019176768139004707, 0.012496449053287506, -0.018532054498791695, 0.0020627370104193687, 0.030589550733566284, -0.015788597986102104, -0.018367446959018707, 0.0036762331146746874, 0.004588432610034943, -0.01588461920619011, -0.024211011826992035, 0.008504718542098999, 0.013305768370628357, -0.014595193788409233, -0.007619953248649836, -0.04628212749958038, 0.024279598146677017, 0.007558225188404322, -0.010644614696502686, -0.004266076255589724, 0.020850276574492455, 0.02477342076599598, 0.001428312505595386, -0.023593734949827194, 0.011426500044763088, -0.01854577288031578, 0.004938223399221897, -0.0330037921667099, -0.029656775295734406, -0.01769530028104782, -0.02283928357064724, 0.0011359628988429904, 0.003254426410421729, 0.006450554355978966, 0.015980640426278114, -0.0022050540428608656, -0.009622677229344845, 0.006364821456372738, 0.023017607629299164, 0.024992898106575012, 0.015322210267186165, -0.036734893918037415, 0.022345460951328278, -0.023868080228567123, -0.019259070977568626, 0.02847708761692047, -0.0028806303162127733, -0.004722176119685173, 0.00042180658783763647, -0.010390845127403736, 0.006817491725087166, 0.00526400888338685, 0.012791370041668415, 0.026145149022340775, 0.0425235889852047, -0.015143885277211666, -0.03160462900996208, 0.01362126599997282, -0.037201281636953354, 0.015774879604578018, -0.02755803056061268, -0.009855871088802814, -0.004060316830873489, 0.02124807797372341, 0.012743360362946987, 0.026721276342868805, 0.0020627370104193687, -0.026309756562113762, -0.02670755796134472, -0.008881943300366402, -0.0010536591289564967, 0.007366183213889599, -0.0024588238447904587, 0.01603550836443901, -0.019300222396850586, 0.03942348435521126, 0.008449848741292953, 0.006529428996145725, 0.010925819166004658, 0.016940850764513016, 0.0008003179682418704, -0.02705049142241478, -0.016858546063303947, 0.024046404287219048, -0.00725644500926137, 0.001021937932819128, -0.012091788463294506, 0.021206926554441452, 0.02689960040152073, 0.008353828452527523, -0.022386612370610237, 0.008724194951355457, 0.024334467947483063, -0.026831014081835747, 0.003081245580688119, 0.025911955162882805, -0.0013185743009671569, -0.014608911238610744, -0.006237936206161976, 0.013148020021617413, 0.0006575724692083895, -0.022290591150522232, 0.0025857086293399334, -0.014348282478749752, 0.0007900299970060587, 0.017914777621626854, -0.0019375667907297611, -0.014691215008497238, 0.008189220912754536, 0.012064354494214058, 0.031165676191449165, -0.024142425507307053, 0.00556578952819109, 0.02360745146870613, 0.016515614464879036, 0.0010785217164084315, -0.004060316830873489, -0.013463517650961876, 0.002316506812348962, -0.01920420303940773, 0.018833834677934647, -0.04625469446182251, -0.038545578718185425, 0.03204358369112015, 0.03840840607881546, 0.016789959743618965, 0.01901216059923172, -0.007770843338221312, -0.001498613622970879, -0.009903881698846817, 0.006680319085717201, -0.0034190339501947165, -0.019190484657883644, -0.01935509219765663, 0.029931120574474335, 0.016817394644021988, 0.016872262582182884, 0.03525342792272568, -0.009183723479509354, 0.04208463802933693, -0.01522618904709816, 0.06430664658546448, -0.013847601599991322, 0.00716728251427412, 0.02507520094513893, -0.0009507794748060405, 0.005634375847876072, 0.009554090909659863, -0.004036311991512775, 0.0129079669713974, -0.040712907910346985, -0.022619806230068207, 0.011721421964466572, 0.020383888855576515, 0.0711652860045433, 0.029437297955155373, -0.0014077365631237626, -0.002091886242851615, -0.006841497030109167, -0.01375843957066536, 0.003038379130885005, 0.016104094684123993, 0.0008217512513510883, -0.026529233902692795, 0.02063080109655857, -0.015143885277211666, 0.01526734046638012, -0.02681729756295681, 0.008045189082622528, -0.022551219910383224, 0.011405924335122108, 0.016748808324337006, 0.00807948224246502, 0.0012379852123558521, 0.04812024533748627, -0.015706293284893036, 0.048092808574438095, 0.011906605213880539, -0.01595320552587509, -0.010205661877989769, 0.040795210748910904, -0.006131627596914768, -0.014039644040167332, -0.021879073232412338, 0.0051474119536578655, 0.026021694764494896, -0.049217626452445984, -0.02388179674744606, -0.015006712637841702, -0.014540324918925762, -0.0049210768193006516, -0.018888704478740692, 0.011392206884920597, -0.007811995223164558, -0.0013100008945912123, -0.0036419397220015526, 0.004557568579912186, -0.009855871088802814, -0.0027623188216239214, -0.007215293124318123, -0.026405777782201767, -0.006587727461010218, -0.005150841549038887], 'text': '2 GPT-4 Observed Safety Challenges\\nGPT-4 demonstrates increased performance in areas such as reasoning, knowledge retention, and\\ncoding, compared to earlier models such as GPT-2[ 22] and GPT-3.[ 10] Many of these improvements\\nalso present new safety challenges, which we highlight in this section.\\nWe conducted a range of qualitative and quantitative evaluations of GPT-4. These evaluations\\nhelped us gain an understanding of GPT-4’s capabilities, limitations, and risks; prioritize our\\nmitigation efforts; and iteratively test and build safer versions of the model. Some of the specific\\nrisks we explored are:6\\n•Hallucinations\\n•Harmful content\\n•Harms of representation, allocation, and quality of service\\n•Disinformation and influence operations\\n•Proliferation of conventional and unconventional weapons\\n•Privacy\\n•Cybersecurity\\n•Potential for risky emergent behaviors\\n•Interactions with other systems\\n•Economic impacts\\n•Acceleration\\n•Overreliance', 'SimilarityScore': 0.8297811051676229}), 0.8297811051676229)\n", - "(Document(page_content='GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction', metadata={'id': '89def1ae-3e87-4a7b-8e00-3d5a340b83ae', 'embedding': [-0.007440360262989998, -0.016012800857424736, -0.015371742658317089, -0.010420596227049828, 0.025601385161280632, 0.012255111709237099, -0.01030466053634882, 0.014921639114618301, -0.016231033951044083, -0.01673569530248642, 0.031043555587530136, 0.037372294813394547, -0.03641752898693085, -0.01844063587486744, -0.0032530263997614384, 0.02834293060004711, 0.022627970203757286, -0.006424215622246265, -0.001998190302401781, -0.01643562503159046, 0.0033928314223885536, -0.00016473987489007413, -0.028670279309153557, -0.014471534639596939, -0.00011455068306531757, 0.002762003568932414, 0.027483640238642693, -0.037590526044368744, -0.0028267912566661835, -0.0037338195834308863, 0.01684481091797352, -0.005680861417204142, -0.0324893444776535, -0.0021345855202525854, -0.0017279571620747447, -0.010481974110007286, -0.0002007567381951958, -0.005595614667981863, 0.016612939536571503, -0.007269866298884153, 0.0028711196500808, 0.01304620411247015, 0.022764364257454872, -0.004381697159260511, -0.009397631511092186, 0.0026034440379589796, 0.020568402484059334, -0.040045641362667084, -0.009111201390624046, 0.01823604293167591, 0.021714121103286743, 0.030770765617489815, -0.004443074576556683, -0.0011968682520091534, -0.009281695820391178, -0.0032751907128840685, -0.01091843843460083, 0.027019895613193512, 0.009097562171518803, 0.00019191236060578376, -0.00019169924780726433, -0.025410432368516922, -0.01333263423293829, 0.03570827096700668, -0.009377172216773033, -0.007297145202755928, -0.04918412119150162, 0.03480806574225426, -0.004098676610738039, 0.000926635111682117, 0.021332215517759323, 0.013230337761342525, -0.0004607601440511644, -0.013298535719513893, 0.03268029913306236, -0.001023816759698093, -0.005772928241640329, -0.009629503823816776, -0.030170626938343048, 0.0024210154078900814, 0.024714816361665726, -0.007051633670926094, 0.016026441007852554, 0.029652323573827744, 0.00949992798268795, 0.001252278801985085, 0.01105483341962099, 0.003166074398905039, -0.022546133026480675, -0.007788167800754309, 0.01157313585281372, 0.005380792077630758, 0.02295531891286373, 0.03319859877228737, -0.006734514608979225, 0.007897283881902695, -0.00340306106954813, 0.036908552050590515, 0.0036724417004734278, -0.03554459661245346, -0.0012258521746844053, -0.0042828102596104145, -0.015426301397383213, -0.01597188226878643, -0.006315099541097879, 0.0072357673197984695, 0.012036879546940327, 0.010747944936156273, 0.014171465300023556, 0.017349474132061005, -0.017526788637042046, 0.03243478760123253, -0.005691091064363718, -0.013925953768193722, -0.020527483895421028, 0.0027091503143310547, 0.02459206059575081, -0.013428110629320145, -0.006642447784543037, 0.002199373207986355, 0.019613634794950485, -4.1344806959386915e-05, 0.01182546652853489, 0.0027227899990975857, 0.023978281766176224, 0.012296030297875404, -0.014144185930490494, -0.02005009911954403, 0.0013017220189794898, -0.004016839899122715, 0.018972577527165413, 0.014430616050958633, 0.011157129891216755, -0.004957966972142458, -0.03434431925415993, 0.020159216597676277, -0.02770187333226204, 0.007774528581649065, -0.039363663643598557, -0.01994098350405693, 0.019340844824910164, 0.028097419068217278, -0.022450655698776245, -0.0032240424770861864, -0.009765898808836937, 0.009459009394049644, 0.0211685411632061, 0.013530407100915909, 0.007310784421861172, 0.009718160144984722, 0.01261655893176794, -0.009349893778562546, -0.023978281766176224, 0.0222324226051569, 0.014116906560957432, 0.0002813151804730296, 0.005527416709810495, 0.016503823921084404, -0.02665162831544876, -0.02287348173558712, -0.009227138012647629, 0.017131241038441658, 0.002131175482645631, 0.004453304223716259, 0.013203058391809464, 0.029734160751104355, 0.02277800440788269, 0.019354483112692833, -0.004886359442025423, -0.015835486352443695, -0.021250378340482712, 0.026406116783618927, -0.05153011903166771, 0.009738619439303875, -0.0009445369942113757, 0.013012105599045753, 0.00963632296770811, -0.02097758650779724, -0.0024039659183472395, -0.007378982380032539, 0.007706330623477697, 0.02607876993715763, 0.0008524702279828489, 0.01078204344958067, -0.008183714002370834, -0.03508085384964943, 0.01957271620631218, -0.00013927232066635042, -0.004763603676110506, -0.0274154432117939, 0.028752116486430168, 0.0117231709882617, -0.003522407030686736, -0.015017115511000156, -0.6149788498878479, -0.020759355276823044, -0.01157995592802763, -0.02249157428741455, 0.02457842230796814, 0.002790987491607666, 0.0006905008922331035, 0.006212803069502115, -0.017076684162020683, 0.013394012115895748, -0.008770213462412357, -0.01852247305214405, -0.021141260862350464, -0.005217117723077536, 0.0054830885492265224, -0.03731773793697357, -0.0173085555434227, -0.016381068155169487, 0.010147805325686932, 0.012677936814725399, -0.018686147406697273, 0.012527902610599995, -0.007365342695266008, -0.001481593237258494, -0.021345853805541992, -0.004613569006323814, -0.00874293502420187, -0.01939540170133114, -0.005660402122884989, 0.0076995110139250755, -0.03358050808310509, 0.015589975751936436, 0.009322614409029484, 0.0012019829591736197, 0.040427546948194504, -0.008020039647817612, -0.034644391387701035, 0.016012800857424736, -0.006158244796097279, 0.02636519819498062, -0.01852247305214405, -0.023869166150689125, 0.0031814188696444035, 0.0056297131814062595, -0.011668612249195576, 0.029543207958340645, 0.013257617130875587, -0.0056058443151414394, -0.007508557755500078, -0.011259426362812519, -0.03224383294582367, 0.007774528581649065, 0.02371913194656372, 0.0023408832494169474, 0.003108106553554535, 0.01285525131970644, 0.04190061613917351, 0.0167220551520586, 0.0037883776240050793, -0.024987606331706047, 0.002731314627453685, 0.01185956597328186, -0.008108696900308132, -0.03120722994208336, -0.021673202514648438, 0.008592899888753891, 0.007338063791394234, 0.011395822279155254, -0.0007403703639283776, -0.045501451939344406, 0.002144815167412162, 0.024973968043923378, 0.010563811287283897, -0.008013220503926277, 0.0034132907167077065, 0.010502433404326439, 0.023773688822984695, -0.02022741362452507, 0.021427690982818604, 0.02024105377495289, 0.027483640238642693, -0.022737085819244385, 0.002279505366459489, 0.004726095125079155, 0.00765859242528677, 0.016762973740696907, -0.010413776151835918, 0.011975501663982868, 0.019218089058995247, 0.003897493937984109, 0.017431311309337616, 0.01775866001844406, -0.029243137687444687, -0.03576283156871796, 0.014717046171426773, 0.009684061631560326, -0.02787918597459793, -0.0021379953250288963, 0.0010357513092458248, -0.00949992798268795, -0.012125536799430847, -0.022450655698776245, 0.042009733617305756, 0.03404425084590912, 0.040700338780879974, 0.02365093305706978, -0.018304239958524704, 0.0019351074006408453, -0.00340306106954813, -0.01512623205780983, -0.01635378785431385, -0.001277852919884026, -0.019231727346777916, -0.013380372896790504, 0.020841192454099655, -0.022136947140097618, 0.008449684828519821, -0.0061343759298324585, -0.020541122183203697, -0.010461514815688133, 0.024182874709367752, -0.0016256606904789805, 0.007938202470541, 0.003774738172069192, 0.010897979140281677, 0.012602919712662697, 0.008626998402178288, -0.03221655637025833, -0.001570250140503049, 0.00802685972303152, -0.006809532176703215, -0.01665385812520981, 0.05739511549472809, -0.0061070965602993965, 0.02059568092226982, 0.02249157428741455, 0.047601938247680664, 0.006997075397521257, 0.010250101797282696, 0.0005685976357199252, -0.022928038612008095, 0.018386077135801315, -0.003399651264771819, 0.014048709534108639, -0.02731996588408947, -0.009738619439303875, -0.007460819557309151, 0.0009939803276211023, 0.006795892491936684, -0.017840497195720673, -0.0076995110139250755, 0.005162559449672699, -0.005813846830278635, 0.010536531917750835, 0.001232671900652349, -0.038436178117990494, -0.012930268421769142, -0.047220028936862946, -0.005524007137864828, -0.025083083659410477, 0.003877034643664956, 0.010393316857516766, -0.0017884825356304646, 0.0010391612304374576, 0.0011363427620381117, -0.013966872356832027, -0.006860680412501097, 0.04056394472718239, -0.04061850160360336, -0.032271113246679306, 0.0029018085915595293, -0.04634710028767586, 0.0069902557879686356, 0.00817007478326559, -0.0292158592492342, 0.0071130115538835526, 0.005878634750843048, -0.008238271810114384, 0.008415586315095425, -0.029461370781064034, -0.012664297595620155, 0.029815997928380966, -0.013769099488854408, -0.020868470892310143, 0.013994150795042515, 0.029843278229236603, 0.041464149951934814, 0.021032145246863365, -0.013059844262897968, 0.0018600900657474995, 0.0016120212385430932, 0.015044394880533218, -0.023951003327965736, 0.035871945321559906, -0.0013307060580700636, 0.023023515939712524, 0.011477659456431866, 0.0009487993665970862, 0.013741820119321346, 0.040427546948194504, 0.006298049818724394, 0.019627274945378304, 0.006267360877245665, -0.020159216597676277, -0.002550591016188264, -0.008981626480817795, 0.009895474649965763, -0.019231727346777916, 0.0006973206182010472, 0.003111516358330846, 0.008190534077584743, -0.011654973030090332, -0.024810293689370155, -0.005513777490705252, 0.010045508854091167, 0.014962557703256607, -0.008770213462412357, 0.013203058391809464, -0.01635378785431385, -0.023214468732476234, 0.005486498586833477, 0.004272580612450838, 0.0034439796581864357, 0.0033161090686917305, -0.015508138574659824, 0.03213471919298172, -0.014580650255084038, 0.008838411420583725, 0.023187190294265747, -0.020445646718144417, 0.007863185368478298, 0.01702212542295456, 0.016749335452914238, 0.011464019306004047, 0.02269616723060608, -0.00546262925490737, 0.019995542243123055, -0.002306784503161907, 0.03194376453757286, -0.010877519845962524, -0.011361722834408283, 0.0055205971002578735, -0.004125955980271101, -0.03750868886709213, 0.022764364257454872, -0.011920943856239319, 0.023555457592010498, -0.007440360262989998, -0.029925115406513214, -0.010366037487983704, -0.005762698594480753, 0.01332581415772438, -0.025219479575753212, 0.00029154482763260603, 0.016476543620228767, -0.006550381425768137, -0.006250311620533466, -0.001981140812858939, -0.009595404379069805, 0.0259832926094532, 0.010897979140281677, 0.016203753650188446, 0.01627195253968239, -0.005476268939673901, 0.013966872356832027, -0.02126401662826538, 0.0016572021413594484, -0.023282667621970177, -0.01015462540090084, -0.004095267038792372, -0.013346273452043533, -0.005813846830278635, 0.011873205192387104, -0.04222796484827995, 0.020827552303671837, 0.022164225578308105, 0.009118021465837955, 0.02040472812950611, 0.015071673318743706, 0.015726370736956596, -0.015617254190146923, -0.047792889177799225, 0.020254692062735558, 0.01777229830622673, -0.002690396038815379, -0.0022437016014009714, -0.04162782430648804, 0.0085997199639678, -0.005909323692321777, -0.010516072623431683, -0.013107581995427608, 0.009895474649965763, -0.00439874641597271, 0.008572440594434738, -0.0008916838560253382, 0.015999160706996918, 0.02967960387468338, -0.016694776713848114, -0.006887959316372871, -0.0002951678179670125, 0.01882254332304001, 0.008633818477392197, -0.011416281573474407, -0.012384687550365925, 0.01994098350405693, -0.022532492876052856, -0.029188580811023712, -0.003074007574468851, -0.007910924032330513, -0.03101627714931965, 0.025451350957155228, -0.0032445017714053392, 0.0043612378649413586, -0.010134166106581688, -0.0020118297543376684, 0.020350169390439987, -0.01853611320257187, -0.001953861676156521, 0.02298259735107422, 0.014757964760065079, 0.012684756889939308, -0.005762698594480753, -0.014116906560957432, 0.02618788555264473, 0.05717688426375389, 0.0466744489967823, -0.015194429084658623, -0.004797702189534903, -0.019259007647633553, 0.01063200831413269, -0.022900760173797607, -0.026215163990855217, -0.004221432376652956, 0.025178560987114906, 0.009718160144984722, -0.023814607411623, -0.009486288763582706, -0.0010604729177430272, 0.03169825300574303, 0.006298049818724394, -0.010113706812262535, -0.01627195253968239, -0.008449684828519821, -0.02676074579358101, -0.008401946164667606, -0.01684481091797352, 0.005595614667981863, 0.007378982380032539, 0.009465829469263554, -0.022818922996520996, 0.016872091218829155, 0.031916484236717224, -0.027279047295451164, -0.030116068199276924, -0.012746134772896767, -0.008265551179647446, -0.003445684676989913, 0.0037951974663883448, 0.0014219203731045127, -0.0048249815590679646, 0.008777033537626266, -0.017635904252529144, 0.012241472490131855, 0.01786777563393116, 0.0067583839409053326, 0.008592899888753891, -0.002182323718443513, -0.017349474132061005, 0.001904418459162116, 0.0017151701031252742, 0.0005801059887744486, 0.020472925156354904, -0.0044055660255253315, -0.029543207958340645, -0.0054251207038760185, 0.005977521184831858, -0.04026387259364128, -0.0008089942275546491, 0.019900064915418625, 0.021100342273712158, 0.0073448834009468555, 8.51404620334506e-05, -0.01834515854716301, -0.015371742658317089, -0.022246062755584717, -0.03685399144887924, -0.0011960157426074147, -0.013796377927064896, -0.007297145202755928, 0.0031848286744207144, -0.00963632296770811, -0.018099647015333176, -0.02534223534166813, 0.008947527036070824, -0.00936353299766779, -0.017731381580233574, -0.03944550082087517, -0.014812522567808628, 0.007644952740520239, 0.006686776410788298, 0.010747944936156273, -0.0077131506986916065, 0.015617254190146923, -0.006976616103202105, -0.005639942828565836, -0.022655248641967773, 0.002530131721869111, -0.030034231022000313, 0.019804587587714195, -0.00524439662694931, -0.012793873436748981, -0.022914400324225426, -0.006959566846489906, 0.00958858523517847, -0.0070925522595644, -0.00602866942062974, -0.0053023649379611015, -0.009029364213347435, 0.013168959878385067, 0.00864745769649744, 0.018099647015333176, -0.006502642761915922, 0.012316489592194557, -0.027006257325410843, -0.02504216507077217, -0.029052184894680977, -0.003339978400617838, -0.04501042887568474, 0.00822463259100914, 0.010509252548217773, 0.022832563146948814, 0.016312871128320694, 0.008572440594434738, -0.03505357727408409, 0.0180450901389122, -0.011689071543514729, -0.007767708506435156, -0.01512623205780983, -0.0009470944060012698, -0.004047528840601444, 0.0033791919704526663, 0.01844063587486744, -0.017417671158909798, -0.0292158592492342, -0.00808141753077507, -0.02230062149465084, 0.03879080340266228, 0.018167845904827118, -0.0005916143418289721, 0.029843278229236603, -0.0005204330664128065, -0.0005651877727359533, -0.048583984375, 0.00907710287719965, -0.0007812889525666833, 0.01426694169640541, -0.007815446704626083, -0.008415586315095425, -0.015385382808744907, -0.017076684162020683, -0.0015370037872344255, -0.009056643582880497, -0.018481554463505745, -0.010052328929305077, -0.026869861409068108, -0.005527416709810495, -0.01220055390149355, 0.008374667726457119, 0.011409461498260498, -0.025587746873497963, -0.011927763000130653, 0.014212383888661861, 0.012330129742622375, 0.01614919677376747, 0.010004590265452862, 0.012698396109044552, -0.009465829469263554, -0.018767984583973885, -0.014867080375552177, -0.032189276069402695, 0.009036184288561344, 0.0027500689029693604, 0.027961023151874542, 0.019886424764990807, 0.027865545824170113, -0.006642447784543037, 0.0029563666321337223, 0.04563784599304199, -0.0010042099747806787, 0.004531731829047203, -0.0045931097120046616, -0.01346902921795845, -0.03208015859127045, -0.003145615104585886, -0.013455389998853207, 0.0003631523286458105, 0.021332215517759323, 0.000726304657291621, -0.009227138012647629, 0.006703825667500496, -0.00802685972303152, 0.011982321739196777, -0.013107581995427608, -0.029434092342853546, -0.02391008473932743, 0.024619340896606445, -0.021291296929121017, -0.0049340976402163506, -0.030934439972043037, -0.01775866001844406, 0.03148002177476883, 0.006448084954172373, 0.014226023107767105, 0.014989836141467094, 0.04405566304922104, -0.01643562503159046, -0.01869978755712509, 0.007078912574797869, 0.026897139847278595, -0.0016827762592583895, 0.0050704930908977985, -0.0504935160279274, 0.006608349271118641, 0.029897835105657578, 0.001049390877597034, 0.02459206059575081, -0.016121916472911835, -0.0004187760059721768, 0.0024892131332308054, 0.00907028280198574, 0.007419900968670845, 0.008729294873774052, -0.002511377213522792, -0.010625189170241356, -0.009002085775136948, -0.01347584929317236, -0.0034951278939843178, -0.015399022027850151, -0.016490183770656586, 0.027715511620044708, -0.008551981300115585, 0.03829978406429291, -0.02088211104273796, -0.036253854632377625, 0.0146079296246171, 0.022573411464691162, 0.03543548285961151, 0.015399022027850151, 0.006509462837129831, 0.030388858169317245, 0.02883395366370678, 0.0012676232727244496, 0.016899369657039642, 0.005350103136152029, -0.007147110532969236, 0.01432149950414896, 0.011689071543514729, 0.007256226614117622, -0.0259832926094532, 0.017322195693850517, 0.0221233069896698, -0.026392478495836258, -0.05092998221516609, 0.026160607114434242, 0.016599299386143684, 0.018113287165760994, -0.016572020947933197, -0.0034013562835752964, -0.03742685168981552, 0.005796797573566437, 0.0026187885086983442, 0.009043004363775253, 0.0027381344698369503, -0.00873611494898796, -0.007754069287329912, 0.01586276665329933, 0.008006400428712368, -0.003917952999472618, 0.00030965980840846896, -0.00524439662694931, 0.02365093305706978, 0.008961167186498642, -0.01040695607662201, 0.006478773895651102, -0.0078086270950734615, 0.027865545824170113, -0.003576965071260929, 0.011273066513240337, 0.00949992798268795, -0.015958242118358612, 0.01395323220640421, -0.0028489555697888136, -0.0022573410533368587, 0.030743485316634178, -0.03944550082087517, -0.02571050263941288, -0.0034081758931279182, -0.008020039647817612, 0.002867709845304489, -0.024905769154429436, -0.0008831591694615781, 0.005568335298448801, -0.012507443316280842, 0.019627274945378304, 0.01824968308210373, 0.011313985101878643, -0.01700848527252674, 0.013373552821576595, -0.009158940054476261, -0.025096723809838295, -0.010679746977984905, -0.004235072061419487, 0.014075987972319126, -0.05136644467711449, -0.004371467512100935, 0.005558105651289225, -0.015371742658317089, 0.006877729669213295, -0.0006427625776268542, -0.017431311309337616, 0.007058453280478716, 0.04149143025279045, -0.016421986743807793, -0.014935278333723545, 0.014089628122746944, -0.004057758487761021, -0.020172854885458946, 0.0171994399279356, -0.0077131506986916065, 0.004074807744473219, 0.03428976237773895, -0.022737085819244385, -0.01664021797478199, 4.3875577830476686e-05, -0.0203228909522295, 0.006533331703394651, 0.005101182032376528, 0.006915238685905933, -0.01413054671138525, -0.007058453280478716, 0.003883854253217578, -0.006110506597906351, 0.004306679591536522, -0.014089628122746944, 0.0074676391668617725, 0.005295544862747192, 0.024933049455285072, -0.018004171550273895, 0.0047738333232700825, 0.0002124782040482387, 0.008333749137818813, -0.013093942776322365, -0.060068462044000626, -0.013162139803171158, -0.006714055314660072, 0.008981626480817795, 0.0022437016014009714, -0.004794292617589235, -0.014089628122746944, 0.007140290457755327, -0.02088211104273796, 0.008340568281710148, 0.017158521339297295, 0.00446694390848279, -0.0019709111656993628, 0.0006299755186773837, -0.002175504108890891, 0.0042998599819839, -0.033416833728551865, -0.028479324653744698, -0.015412661246955395, 0.009888654574751854, -0.009472649544477463, -0.02872483618557453, -0.01778593845665455, 0.05046623572707176, 0.0078086270950734615, -0.019927343353629112, -0.019436320289969444, -0.02099122665822506, -0.042691707611083984, -0.014594290405511856, -0.030361579731106758, 0.02636519819498062, 0.0044328453950583935, 0.03338955342769623, 0.0006005652830936015, 0.03311676159501076, 0.010291020385921001, 0.009752259589731693, -0.00685045076534152, 0.006714055314660072, -0.006621988490223885, 0.0032376819290220737, -0.014853441156446934, -0.0050704930908977985, -0.018454276025295258, -0.007474458776414394, 0.004975016228854656, -0.018590670078992844, -0.002608558861538768, 0.03025246411561966, 0.011989140883088112, -0.006318509113043547, 0.004657897166907787, 0.025110362097620964, 0.003128565615043044, 0.010045508854091167, 0.012752954848110676, -0.04479219391942024, -0.0021857337560504675, 0.00987501535564661, -0.0180450901389122, -0.013087122701108456, -0.0141851045191288, 0.006482183467596769, 0.015330824069678783, 0.00500911520794034, 0.006567430682480335, -0.01497619692236185, -0.0002487081801518798, -0.0035462761297822, 0.03458983078598976, -0.00645149452611804, -0.0026716417632997036, 0.012268751859664917, -0.005213707685470581, -0.01086388062685728, -0.014457895420491695, 0.02193235419690609, -0.010386496782302856, 0.008142795413732529, 0.032571181654930115, -0.04370103403925896, -0.004098676610738039, 0.00047226849710568786, 0.003921363037079573, -0.02695169858634472, -0.01664021797478199, 0.010475154034793377, -0.007781348191201687, -0.022750725969672203, 0.03641752898693085, 0.029461370781064034, -0.010536531917750835, 0.0032376819290220737, -0.007399441674351692, 0.01721307821571827, -0.008899789303541183, -0.013578145764768124, -0.0045283217914402485, -0.021004866808652878, -0.0016887434758245945, 0.02288712002336979, -0.011273066513240337, -0.011968681588768959, 0.01739039272069931, 0.028615720570087433, -0.032189276069402695, 0.01700848527252674, 0.20568402111530304, -0.032380230724811554, 0.005333053879439831, 0.021018505096435547, 0.023214468732476234, -0.007835905998945236, 0.01413054671138525, 0.01318941917270422, -0.004289630334824324, -0.012152815237641335, 0.003195058321580291, -0.002613673685118556, -0.004289630334824324, -0.004906818736344576, 0.014526092447340488, -0.021523168310523033, -0.030307020992040634, -0.03718134015798569, -0.005837716162204742, 0.006533331703394651, 0.0050568534061312675, -0.011873205192387104, -0.009561305865645409, -0.011845925822854042, 0.016599299386143684, 0.0014534617075696588, -0.010932078585028648, -0.007037993986159563, 0.02024105377495289, 0.01729491539299488, -0.02459206059575081, 0.0008797492482699454, 0.013509947806596756, 0.011539037339389324, -0.015371742658317089, -0.006386707071214914, 0.014389697462320328, 0.006918648257851601, 0.00571837043389678, 0.012084618210792542, 0.03374418243765831, 0.00831328984349966, 0.019722750410437584, -0.036335691809654236, 0.005749059375375509, 0.022559771314263344, -0.010822962038218975, -0.001524216728284955, -0.003002400044351816, 0.0023255387786775827, -0.02078663371503353, 0.0171994399279356, 0.02760639600455761, 0.021236738190054893, -0.022518852725625038, 0.007317604497075081, -0.019122611731290817, 0.014389697462320328, 0.007324424106627703, 0.006127555854618549, -0.023296305909752846, -0.01234376896172762, -0.016967566683888435, 0.03865440934896469, -0.016108278185129166, 0.021223098039627075, -0.011320804245769978, -0.0106388283893466, 0.016026441007852554, -0.005551286041736603, -0.01568545214831829, 0.0019709111656993628, -0.02326902747154236, -0.010079608298838139, -0.05324869975447655, -0.04367375373840332, 0.03423520550131798, 0.027456361800432205, 0.023978281766176224, 0.04100040718913078, 0.001798712182790041, -0.026719827204942703, 0.014676127582788467, -0.024605700746178627, -0.024714816361665726, -0.04124591872096062, -0.0023204239550977945, -0.006809532176703215, -0.011041194200515747, -0.020091017708182335, -0.007065273355692625, -0.008995265699923038, -0.0005084985168650746, 0.01072748564183712, 0.0003663490933831781, 0.008436045609414577, 0.005991160869598389, -0.0016120212385430932, 0.009383992291986942, -0.02967960387468338, -0.01483980193734169, 0.06885231286287308, 0.004262350965291262, 0.006311689503490925, 0.002550591016188264, 0.009254416450858116, -0.010243282653391361, 0.011252607218921185, -0.0027176751755177975, -0.0204320065677166, 0.0012377867242321372, -0.02153680846095085, 0.012486984021961689, 0.014171465300023556, 0.0035189969930797815, 0.022914400324225426, -0.0026989206671714783, 0.00522052776068449, 0.020636599510908127, -0.013005285523831844, -0.00921349786221981, -0.020281972363591194, -0.009029364213347435, 0.009145300835371017, -0.015221708454191685, -0.04631982371211052, -0.01213917601853609, 0.009479468688368797, -0.030934439972043037, 0.02062295936048031, 0.01597188226878643, -0.0423370823264122, 0.001209655194543302, 0.009206678718328476, -0.010543351992964745, -0.0031609595753252506, -0.01395323220640421, 0.0094453701749444, -0.016899369657039642, 0.017458589747548103, 0.009595404379069805, 0.024619340896606445, -0.013284895569086075, -0.010331938974559307, 0.011048014275729656, -0.02314627170562744, 0.01749950833618641, 0.006734514608979225, -0.004599929321557283, -0.017608625814318657, -0.024742096662521362, -0.009738619439303875, 0.014798883348703384, 0.02013193629682064, 0.029352255165576935, 0.0056433528661727905, -0.019218089058995247, -0.012541541829705238, 0.03172553330659866, -0.00808141753077507, -0.047629214823246, 0.00936353299766779, 0.01796325296163559, -0.008047319017350674, -0.017826857045292854, 0.0010766698978841305, -0.17305827140808105, -0.015712730586528778, 0.0016546447295695543, -0.035299088805913925, 0.021223098039627075, -0.0018975987331941724, 0.023964643478393555, 0.004057758487761021, -0.010263741947710514, 0.009950032457709312, 0.013114402070641518, -0.008626998402178288, -0.007263046223670244, 0.002284620190039277, -0.0060559483245015144, 0.006905009038746357, -0.010031869634985924, -0.00958858523517847, -0.004351008217781782, 0.004275990650057793, 0.03565371409058571, -0.03210743889212608, 0.0171994399279356, -0.0047226850874722, -0.0076858713291585445, -0.0038702148012816906, -0.01356450654566288, 0.045037705451250076, 0.006969796493649483, -0.0372631773352623, -0.007522197440266609, -0.007644952740520239, 0.030688928440213203, -0.008858870714902878, 0.004293039906769991, -0.013482669368386269, 0.028151975944638252, -0.009561305865645409, -0.03434431925415993, 0.01214599609375, 0.03156185895204544, 0.01882254332304001, 0.018836181610822678, -0.017922334372997284, -0.03186192736029625, -0.018208764493465424, 0.023882806301116943, -0.02022741362452507, 0.008824772201478481, -0.021304935216903687, 0.028206534683704376, -0.010509252548217773, 0.008558801375329494, -0.0013110991567373276, 0.009227138012647629, 0.004003200214356184, 0.010079608298838139, 0.007310784421861172, -0.0005749911651946604, -0.0040645780973136425, -0.008156435564160347, -0.010857060551643372, 0.010400136932730675, 0.019231727346777916, -0.01807236857712269, -0.021509528160095215, -0.022477934136986732, 0.013646343722939491, -0.023282667621970177, 0.014307860285043716, -0.015521777793765068, -0.0014594290405511856, -0.01441697683185339, -0.010761584155261517, 0.025642303749918938, 0.021113982424139977, -0.024442026391625404, 0.022477934136986732, 0.020582040771842003, -0.008476964198052883, -0.014512453228235245, 0.020650237798690796, -0.007140290457755327, 0.0028233814518898726, -0.004524911753833294, -0.008381486870348454, 0.011593595147132874, 0.016858451068401337, -0.009950032457709312, -0.0063662477768957615, 0.022668888792395592, -0.01483980193734169, -0.010250101797282696, 0.010291020385921001, 0.015467219986021519, 0.012514262460172176, 0.019436320289969444, -0.01634014956653118, 0.03194376453757286, -0.019136251881718636, 0.008674737066030502, 0.016694776713848114, -0.002419310389086604, -0.011607234366238117, 0.04397382587194443, 0.02618788555264473, -0.01794961281120777, 0.013503128662705421, 0.04075489565730095, 0.006253721658140421, -0.02711537294089794, -0.02185051701962948, 0.0010110297007486224, 0.02298259735107422, 0.0003407749754842371, 0.023569095879793167, -0.020854830741882324, -0.029652323573827744, 0.022464295849204063, -0.0025574106257408857, 0.03846345469355583, -0.009404451586306095, -0.03243478760123253, 0.023882806301116943, -0.00851106271147728, -0.028806673362851143, -0.11129850894212723, -0.012077798135578632, 0.023473620414733887, 0.033034924417734146, -0.004559010732918978, 0.015303545631468296, -0.0029819407500326633, 0.038217946887016296, -0.021141260862350464, 0.03639024868607521, -0.004337368533015251, -0.01994098350405693, 0.030225183814764023, -0.010038689710199833, 0.012609739787876606, -0.009138480760157108, 0.0171994399279356, -0.02333722449839115, -0.0055342367850244045, 0.037590526044368744, 0.004804522264748812, 0.003123450791463256, 0.030034231022000313, -0.02041836641728878, -0.01020918320864439, -0.010052328929305077, -0.03769964352250099, 0.024251073598861694, 0.003655392210930586, 0.009997771121561527, 0.017172159627079964, -0.0370449461042881, 0.015562696382403374, -0.02751091867685318, 0.02646067552268505, 0.02240973711013794, -0.011920943856239319, -0.03243478760123253, 0.019531797617673874, -0.0150307547301054, -0.010291020385921001, -0.01149129867553711, 0.019231727346777916, -0.017704101279377937, -0.005309184547513723, -0.017335833981633186, -0.010202364064753056, 0.023896444588899612, 0.003208698006346822, -0.023814607411623, -0.029379533603787422, 0.01531718485057354, -0.03221655637025833, 0.03063436970114708, 0.01920444890856743, 0.010086427442729473, 0.012309670448303223, 0.009649963118135929, 0.01995462365448475, 0.00035931621096096933, -0.024155596271157265, -0.010229642502963543, 0.003914543427526951, 0.021591365337371826, 0.0031149261631071568, 0.014444255270063877, -0.005650172475725412, -0.014935278333723545, 0.015930963680148125, -0.01441697683185339, 0.007747249212116003, 0.015767289325594902, 0.0003220206417609006, 0.007324424106627703, -0.0141851045191288, -0.01947723887860775, -0.008667916990816593, -0.012377867475152016, 0.007563116028904915, 0.013946413062512875, -0.027920104563236237, -0.03205288201570511, 0.018208764493465424, -0.012575640343129635, -0.019981902092695236, -0.0009189628763124347, 0.008981626480817795, -0.012793873436748981, 0.0167220551520586, -0.03268029913306236, 0.0070925522595644, 0.05215753987431526, -0.011777728796005249, -0.005145510192960501, -0.004340778570622206, -0.012111896649003029, -0.014457895420491695, 0.002236881759017706, -0.014689766801893711, 0.010318299755454063, -0.014062348753213882, -0.0012735904892906547, -0.07392621785402298, 0.02760639600455761, 0.020950308069586754, -0.02345998026430607, 0.01044105552136898, 0.008981626480817795, 0.014785243198275566, 0.0004692848597187549, -0.018972577527165413, -0.0053739724680781364, -0.024428386241197586, -0.002359637524932623, -0.0203228909522295, -0.014403336681425571, -0.0015318889636546373, -0.013721360825002193, 0.005595614667981863, -0.007433540187776089, 0.006659497506916523, -0.0012931972742080688, -0.0014415271580219269, 0.00017976465460378677, 0.025956014171242714, 0.011143490672111511, -0.011893664486706257, -0.004214612767100334, -0.04691996052861214, 0.028943069279193878, -0.00963632296770811, -0.01361224427819252, 0.005728600081056356, -0.013646343722939491, -0.02153680846095085, 0.014485173858702183, -0.005067083053290844, 0.01006596814841032, 0.0047022257931530476, 0.023814607411623, 0.003863394958898425, 0.0360901802778244, -0.013523587957024574, -0.03944550082087517, 0.005684271454811096, -0.022068748250603676, 0.014717046171426773, 0.003314404282718897, 0.00864745769649744, -0.02807013876736164, 0.02174140140414238, 0.020472925156354904, 0.024987606331706047, 0.01384411659091711, -0.017444951459765434, -0.028752116486430168, -0.0074539994820952415, -0.017799578607082367, 0.0229689572006464, 0.010536531917750835, 0.006506052799522877, -0.0022351769730448723, 0.04198245331645012, -0.0075903949327766895, 0.006417396012693644, 0.006437855307012796, -0.010952537879347801, 0.0022709807381033897, -0.01778593845665455, -0.005793387535959482, 0.011511757969856262, -0.005336463451385498, 0.006652677431702614, -0.015508138574659824, 0.019531797617673874, 0.02561502531170845, 0.020963948220014572, -0.007617673836648464, -0.005087542347609997, 0.010325118899345398, -0.017622264102101326, 0.037481412291526794, 0.019968261942267418, -0.0033808969892561436, -0.007269866298884153, -0.005489908158779144, 0.01681753247976303, -0.003326338715851307, 0.0012821152340620756, 0.01389185432344675, -0.028397487476468086, -2.7465524908620864e-05, 0.012589280493557453, 0.0041736941784620285, -0.004630618263036013, 0.023596376180648804, 0.004828391131013632, -0.0008916838560253382, -0.004490813240408897, 0.001849860418587923, 0.03393513336777687, 0.01361224427819252, 0.008531522005796432, -0.01526262704282999, -0.006103686988353729, -0.002124355873093009, -0.03308948501944542, 0.013312174938619137, -0.045501451939344406, -0.03731773793697357, 0.010904799215495586, 0.01681753247976303, 0.01749950833618641, 0.01318941917270422, 0.008374667726457119, 0.007010715082287788, -0.024237433448433876, 0.03287125006318092, -0.0011423100950196385, -0.022082388401031494, -0.0413823127746582, 0.022177865728735924, 0.029461370781064034, 0.01234376896172762, 0.0567404180765152, -0.012637018226087093, 0.02648795396089554, 0.00765859242528677, 0.031343623995780945, -0.016967566683888435, 0.030088789761066437, 0.02062295936048031, -0.014253302477300167, -0.007883644662797451, 0.0028830543160438538, -0.01842699572443962, -0.009043004363775253, -0.011443560011684895, -0.02135949395596981, 0.015821848064661026, 0.023391783237457275, 0.08963894844055176, 0.01739039272069931, -0.0019606815185397863, 0.008572440594434738, -0.003188238712027669, -0.012521082535386086, 0.004385106731206179, 0.02742908149957657, -0.005800207611173391, 0.006253721658140421, 0.009568125940859318, -0.0036758515052497387, -0.003542866325005889, -0.017035765573382378, 0.013946413062512875, -0.008981626480817795, 0.005316004157066345, -0.006724284961819649, -0.00874293502420187, -0.004630618263036013, 0.03958189859986305, -0.012943907640874386, 0.02685622125864029, 0.020200135186314583, -0.021400412544608116, -0.007972301915287971, 0.029925115406513214, 0.009602224454283714, -0.006980026140809059, -0.028506604954600334, 0.02940681204199791, 0.0019265826558694243, -0.043619196861982346, -0.022164225578308105, -0.007508557755500078, -0.0032155176158994436, -0.006188933737576008, -0.027292687445878983, 0.010106886737048626, -0.0016188409645110369, 0.014171465300023556, 0.007419900968670845, -0.0039691012352705, -0.0023204239550977945, -0.0018737295176833868, -0.0061343759298324585, -0.008163254708051682, -0.018304239958524704, -0.02231425978243351], 'text': 'GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction', 'SimilarityScore': 0.8287368247866265}), 0.8287368247866265)\n", - "(Document(page_content='plan to refine these methods and register performance predictions across various capabilities before\\nlarge model training begins, and we hope this becomes a common goal in the field.\\n4 Capabilities\\nWe tested GPT-4 on a diverse set of benchmarks, including simulating exams that were originally\\ndesigned for humans.4We did no specific training for these exams. A minority of the problems in the\\nexams were seen by the model during training; for each exam we run a variant with these questions\\nremoved and report the lower score of the two. We believe the results to be representative. For further\\ndetails on contamination (methodology and per-exam statistics), see Appendix C.\\nExams were sourced from publicly-available materials. Exam questions included both multiple-\\nchoice and free-response questions; we designed separate prompts for each format, and images were\\nincluded in the input for questions which required it. The evaluation setup was designed based', metadata={'id': '4b369374-f24d-452c-a830-d9af481bc157', 'embedding': [-0.012286471202969551, -0.006482758093625307, 0.006040671374648809, -0.025973472744226456, -0.005934570450335741, 0.021644560620188713, 0.013184790499508381, -0.002574712270870805, -0.0237948689609766, -0.01914058066904545, 0.020838193595409393, 0.037319183349609375, -0.02171529456973076, -0.007844384759664536, 0.002850574441254139, 0.02864721231162548, 0.015151191502809525, -0.013708221726119518, -0.0005751546705141664, -0.019352782517671585, 0.008389035239815712, 0.007950485683977604, -0.032622452825307846, -0.016608309000730515, 0.002132625784724951, -0.0037241375539451838, 0.029566751793026924, -0.036045972257852554, 0.008466842584311962, 0.016438547521829605, 0.010617151856422424, -0.012229884043335915, -0.03287709504365921, -0.014330679550766945, -0.007893898524343967, -0.00890539214015007, -0.005895666778087616, -0.013177717104554176, 0.0314624197781086, -0.003886825405061245, 0.003653403604403138, 0.010702032595872879, 0.0025481872726231813, -0.005984084215015173, 0.014436780475080013, 0.010553491301834583, 0.015165338292717934, -0.008155613206326962, -0.01621219888329506, 0.006857647560536861, 0.023243146017193794, 0.018362509086728096, -0.006242262665182352, 0.008912465535104275, -0.0018302384996786714, 0.017372235655784607, -0.00907515361905098, 0.013531386852264404, 0.007278513628989458, -0.008693191222846508, -0.007794870994985104, 0.006033597979694605, -0.015335099771618843, 0.03151900693774223, -0.015094605274498463, -0.020272323861718178, -0.007462421897798777, 0.02462952956557274, 0.009867372922599316, -0.01019982248544693, 0.00740583473816514, 0.007596815936267376, 0.006924844346940517, -0.016084879636764526, 0.03644208237528801, -0.027147654443979263, -0.019678158685564995, -0.011388150975108147, -0.02397877722978592, 0.018857646733522415, 0.020852340385317802, -0.01222281064838171, -0.024954903870821, 0.021474799141287804, 0.006578248459845781, 0.009110520593822002, 0.02021573670208454, -0.007045092061161995, 0.006274092942476273, -0.009662244468927383, 0.025167105719447136, 0.016778070479631424, 0.02008841559290886, 0.02088063396513462, -0.010709105990827084, 0.014295312575995922, -0.011748893186450005, 0.035225458443164825, -0.009747125208377838, -0.02948187105357647, -0.017598584294319153, 0.012491598725318909, -0.028236955404281616, -0.010051281191408634, -0.016792217269539833, 0.007073385640978813, -0.0014217505231499672, -0.022111402824521065, -0.0008023871923796833, 0.004102563485503197, -0.0060265245847404, 0.026963746175169945, -0.014344826340675354, -0.015419980511069298, -0.013644561171531677, -0.004541113507002592, -0.0034854107070714235, -0.011338637210428715, 0.00981078576296568, 0.004282935056835413, 0.021106982603669167, -0.006351900286972523, 0.04538284242153168, -0.003989389631897211, 0.00740583473816514, -0.0024845267180353403, -0.025351013988256454, -0.016933685168623924, -0.0010353667894378304, -0.0030397875234484673, -0.0030698494520038366, 0.009209548123180866, 0.015363393351435661, -0.008954905904829502, -0.025365160778164864, 0.01864544488489628, -0.01747126318514347, 0.0075614494271576405, -0.04051635414361954, -0.02982139401137829, 0.009633950889110565, 0.017527850344777107, -0.0003656056069303304, 0.004024756606668234, -0.0160424392670393, 0.031038016080856323, 0.027487175539135933, 0.02041379176080227, 0.015023871324956417, -0.006454464513808489, -0.005527850706130266, -0.02719009481370449, -0.02021573670208454, 0.01993280090391636, -0.0017170642968267202, -0.02818036824464798, 0.006033597979694605, 0.01957913115620613, -0.017909811809659004, -0.019847920164465904, 0.0014606540789827704, 0.007172413170337677, 0.012088416144251823, 0.007497788872569799, -0.006613615434616804, 0.013892129063606262, 0.026482755318284035, 0.01261891983449459, -0.003257294185459614, -0.013114056549966335, -0.021800175309181213, 0.015900971367955208, -0.048636600375175476, 0.020852340385317802, 0.001419982174411416, 0.004576480481773615, 0.016735630109906197, -0.0003050397499464452, -0.011812553741037846, -0.012350130826234818, 0.0127886813133955, 0.01875861920416355, 0.012965516187250614, -0.005152961239218712, -0.002604774199426174, -0.02365340106189251, 0.010015914216637611, 0.006139698904007673, -0.011317417025566101, -0.0020088416058570147, 0.017032712697982788, 0.004403182305395603, -0.01716003380715847, 0.00921662151813507, -0.6220052242279053, -0.0234836395829916, -0.006302386522293091, -0.025832004845142365, 0.011232536286115646, 0.011692306026816368, 0.008488062769174576, 0.004831122234463692, -0.021474799141287804, 0.047052159905433655, -0.01748540997505188, -0.014217505231499672, -0.005220158491283655, -0.03584791719913483, -0.008792218752205372, -0.029340403154492378, -0.004551723599433899, -0.01537754014134407, 0.007370467763394117, 0.010574711486697197, -0.01636781357228756, 0.011685232631862164, 0.0008390803704969585, -0.00449159974232316, -0.016580015420913696, 0.018051279708743095, -0.005064544267952442, -0.009945180267095566, 0.013559680432081223, 0.0025977008044719696, -0.007087532430887222, 0.03355614095926285, 0.01765516959130764, -0.002748010214418173, 0.04306277260184288, -0.005923960357904434, -0.03632890805602074, 0.013630414381623268, -0.008551723323762417, 0.042496901005506516, -0.0336410216987133, -0.008035366423428059, -0.003405835246667266, 0.03969584032893181, -0.005828469526022673, 0.03853580355644226, 0.0025835540145635605, 0.0018320068484172225, -0.013170643709599972, -0.01280282810330391, -0.004505746532231569, -0.005874446593225002, 0.014528733678162098, 0.01045446377247572, 0.016551721841096878, -0.00915296096354723, 0.021927494555711746, -0.003453580429777503, -0.012045975774526596, -0.01714588701725006, 0.008488062769174576, 0.009938106872141361, 0.005874446593225002, -0.0589071549475193, -0.029057467356324196, 0.03870556503534317, -0.012696727178990841, 0.015561448410153389, 0.0012864719610661268, -0.03448982909321785, 0.0016259945696219802, 0.0077170636504888535, -0.01304332260042429, -0.02090892754495144, 0.008763925172388554, 0.014528733678162098, 0.024403180927038193, -0.02351193316280842, 0.002196286339312792, 0.01875861920416355, 0.011798406951129436, -0.008374888449907303, 0.008042439818382263, -0.005241378676146269, 0.006663129199296236, -0.001750662922859192, -0.01061007846146822, 0.01134571060538292, -0.0038337749429047108, 0.01700441911816597, 0.020343057811260223, 0.01959327794611454, -0.015193631872534752, -0.042808130383491516, 0.0007625994039699435, 0.010475683957338333, -0.02153138630092144, -0.009435896761715412, 0.004993810318410397, -0.012088416144251823, -0.020965514704585075, -0.0079434122890234, 0.03372590243816376, 0.01622634567320347, 0.04775949940085411, 0.012746240943670273, 0.0020548184402287006, -0.002703801728785038, 0.0005322722718119621, -0.01084350049495697, 0.0016648981254547834, -0.0003770998737309128, -0.015448274090886116, 0.006953137926757336, 0.010093721561133862, -0.030698493123054504, 0.02362510748207569, -0.001715296064503491, -0.02993456833064556, -0.03126436471939087, 0.0110061876475811, -0.017259061336517334, 0.01498143095523119, -0.018008839339017868, 0.016961978748440742, 0.01327674463391304, 0.004969053436070681, -0.022748008370399475, 0.0035508396103978157, 0.014797522686421871, -0.016891244798898697, -0.023738281801342964, 0.03415030613541603, -0.016099026426672935, 0.020159149542450905, 0.029425283893942833, 0.010093721561133862, -0.014351899735629559, 0.009174181148409843, -0.006786913610994816, -0.017046859487891197, 0.004272324964404106, -0.005998231004923582, -0.0031158262863755226, -0.023540226742625237, 0.011946948245167732, -0.00907515361905098, 0.0019717062823474407, 0.006341290194541216, -0.003900972194969654, -0.000811228936072439, 0.0009593279100954533, 0.0059274970553815365, 0.0056834653951227665, -0.010263482108712196, -0.02024403028190136, -0.006090185139328241, -0.04263836890459061, -0.010235188528895378, -0.03536692634224892, 0.007299733813852072, 0.016792217269539833, 0.0019434127025306225, -0.0013801943277940154, 0.01407603733241558, -0.011204242706298828, -0.013715295121073723, 0.03400883823633194, -0.015519008040428162, -0.034065425395965576, 0.011904507875442505, -0.03980901464819908, 0.020965514704585075, 0.028095487505197525, -0.024827582761645317, -0.00429354514926672, -0.011784260161221027, 0.00412378367036581, 0.0034252870827913284, -0.026921305805444717, 0.00623872596770525, 0.009428823366761208, -0.014684348367154598, -0.012081342749297619, 0.03347126021981239, 0.03949778527021408, 0.022762155160307884, 0.012442084960639477, -0.01432360615581274, 0.018716178834438324, 0.011748893186450005, -0.002923076506704092, -0.01813616044819355, 0.025308573618531227, 0.0012166223023086786, 0.020145002752542496, 0.013460652902722359, -0.011494251899421215, 0.012951369397342205, 0.034546416252851486, 0.020767459645867348, 0.007045092061161995, 0.009414676576852798, -0.025068078190088272, 0.007197170052677393, -0.0018231651047244668, 0.010914234444499016, -0.014839963056147099, -0.01166401244699955, -0.0005875331116840243, -0.004919539671391249, -0.01909814029932022, -0.022083109244704247, -0.013545533642172813, -0.011140582151710987, 0.010249335318803787, 0.015038018114864826, 0.011218389496207237, -0.023285584524273872, -0.00835366826504469, 0.015971705317497253, 0.0007953137974254787, 0.01636781357228756, 0.0013545533875003457, -0.02008841559290886, 0.019480103626847267, -0.01077983994036913, 0.013340405188500881, 0.0333297923207283, -0.009336869232356548, -0.0047356318682432175, 0.015632182359695435, 0.01779663749039173, 0.018687885254621506, 0.034885939210653305, -0.002988505410030484, 0.025167105719447136, -0.007147656287997961, 0.039412904530763626, 0.010984967462718487, 0.016396107152104378, -0.0039045088924467564, -0.008785145357251167, -0.014726788736879826, 0.010334216058254242, -0.007264366839081049, 0.023257290944457054, -0.0001488726556999609, -0.028576478362083435, 0.01450044009834528, -0.0007015914306975901, 0.003366931574419141, -0.009492483921349049, 0.008162686601281166, 0.01914058066904545, -0.008834658190608025, 0.012031828984618187, 0.0026967283338308334, 0.005046860780566931, 0.021149422973394394, 0.015745356678962708, 0.008410255424678326, 0.004017683211714029, -0.003766577923670411, 0.022776301950216293, -0.02864721231162548, -0.01059593167155981, -0.01990450732409954, -0.03550839424133301, -0.016240492463111877, -0.018093720078468323, -0.023200705647468567, -0.0008514588116668165, -0.0234836395829916, -0.0034005302004516125, 0.015462420880794525, -0.0002816091582644731, -0.0030309457797557116, 0.02545004151761532, 0.010327142663300037, -0.0286755058914423, -0.029595045372843742, 0.0042015910148620605, 0.008679044432938099, -0.008714411407709122, -0.006896550767123699, -0.026638370007276535, 0.0018673738231882453, -0.004350132308900356, 0.0024650748819112778, -0.007837311364710331, 0.008205126971006393, -0.030981428921222687, 0.0055809011682868, -0.009379309602081776, 0.0237948689609766, 0.03284880146384239, 0.0035826698876917362, 0.003717064158990979, -0.007098142523318529, -0.0012926611816510558, 0.006723253056406975, -0.02317241206765175, -0.03199999779462814, 0.019225461408495903, -0.011076921597123146, -0.03307515010237694, -0.026312993839383125, 0.0020265248604118824, -0.0012988504022359848, 0.02717594802379608, 0.022889476269483566, 0.004834658931940794, -0.008459769189357758, 0.005308575928211212, 0.01990450732409954, -0.02202652208507061, 0.008473915979266167, 0.02669495716691017, 0.02880282700061798, -0.009725905023515224, -0.02931210957467556, -0.0027108751237392426, 0.01621219888329506, 0.04422280564904213, 0.03465959057211876, -0.003329796250909567, -0.0006030061049386859, -0.020526964217424393, -0.010871794074773788, -0.02574712410569191, -0.03463129699230194, 0.028944293037056923, 0.01012201514095068, 0.012003535404801369, -0.010022987611591816, -0.0007259062258526683, -0.008403182029724121, 0.01929619535803795, 0.012675506994128227, -0.009832005947828293, -0.01928204856812954, -0.003036250825971365, 0.01496728416532278, 0.01892838068306446, -0.02137577161192894, 0.006008841097354889, 0.02107868902385235, 0.01909814029932022, -0.009096373803913593, 0.019494250416755676, 0.01846153661608696, -0.012102562934160233, -0.02059769816696644, -0.039582666009664536, 0.010397876612842083, -0.020017681643366814, 0.014670201577246189, -0.01327674463391304, 0.005955790635198355, 0.00769584346562624, 0.005425286944955587, 0.027458883821964264, 0.006578248459845781, 0.016466841101646423, -0.006100795231759548, 0.01351016666740179, -0.005170644726604223, -0.004721485078334808, -0.014040670357644558, 0.01416799146682024, 0.0076038893312215805, 0.0024102560710161924, -0.036838192492723465, 0.010037134401500225, -0.002864721231162548, -0.04125198349356651, -0.004862952511757612, 0.026440314948558807, 0.0032855875324457884, 0.01912643387913704, -0.004056586418300867, -0.020979661494493484, -0.011494251899421215, -0.020668432116508484, -0.020003534853458405, -0.0023236072156578302, -0.0042900084517896175, -0.023059237748384476, 0.009987620636820793, -0.036045972257852554, -0.016565868631005287, -0.03678160533308983, 0.009902739897370338, -0.01914058066904545, 0.00016202473489101976, -0.049457110464572906, 0.0034022985491901636, 0.015066311694681644, 0.009938106872141361, 0.015320952981710434, -0.004367815796285868, 0.00725022004917264, 0.008756851777434349, -0.023568520322442055, -0.038960207253694534, 0.0020176833495497704, -0.02655348926782608, 0.02622811309993267, 0.002862952882423997, -0.025860298424959183, -0.020625991746783257, -0.01730150170624256, 0.007264366839081049, 0.0018443852895870805, -0.009598583914339542, -0.0025676388759166002, -0.009534923359751701, 0.014542880468070507, 0.014698495157063007, 0.011925728060305119, 0.012852341867983341, 0.01864544488489628, -0.03468788415193558, 0.001015914953313768, -0.021022101864218712, 0.004463306162506342, -0.03027409128844738, 0.0045977006666362286, 0.00437135249376297, 0.0027108751237392426, 0.00940052978694439, 0.007172413170337677, -0.03630061447620392, 0.03078337386250496, -0.018970821052789688, 0.013906275853514671, 0.0030557026620954275, 0.024162685498595238, 0.0074482751078903675, -0.0037842614110559225, 0.03352784737944603, 0.00938638299703598, -0.023596813902258873, -0.013538460247218609, -0.014118477702140808, 0.036215733736753464, -0.0026472145691514015, -0.02378072217106819, 0.03516887500882149, 0.007879751734435558, -0.0028682579286396503, -0.06750839203596115, 0.00224579987116158, -0.02284703589975834, 0.026595929637551308, -0.005541997496038675, -0.01343235932290554, -0.003519009333103895, -0.015207778662443161, -0.013566753827035427, -0.0079434122890234, -0.013142350129783154, 0.026426168158650398, -0.012010608799755573, 0.0014456232311204076, 0.016113173216581345, 0.015674622729420662, 0.01843324303627014, -0.030641905963420868, -0.016155613586306572, 0.0064509278163313866, 0.004760388284921646, 0.012852341867983341, 0.0008956674719229341, 0.01126790326088667, -0.029425283893942833, -0.0060795750468969345, -0.028251102194190025, -0.04181785508990288, 0.024007070809602737, 0.016240492463111877, 0.023922190070152283, 0.015108752064406872, 0.015787797048687935, -0.020187443122267723, -0.0065004415810108185, 0.019168874248862267, -0.0006710874731652439, -0.008763925172388554, -0.0009389919578097761, -0.01843324303627014, -0.00039168872172012925, -0.008877098560333252, -0.005743589252233505, -0.01638196036219597, 0.009471263736486435, -0.004827585536986589, 0.0006940759485587478, 0.03299026936292648, -0.0015198937617242336, 0.007759504020214081, -0.014054817147552967, -0.033782489597797394, -0.030415557324886322, 0.03584791719913483, 0.014528733678162098, 0.01652342826128006, -0.037630412727594376, -0.003798407968133688, 0.012611846439540386, 0.0009778955718502402, 0.01649513468146324, -0.002838196000084281, 0.02154553309082985, -0.013389918953180313, -0.009301502257585526, -0.0019221925176680088, 0.013701148331165314, 0.0014491599285975099, -0.020286470651626587, -0.05217329040169716, -0.005128204356878996, 0.013248451054096222, -0.02042793668806553, 0.033273205161094666, -0.0015278513310477138, 0.017089299857616425, -0.02767108380794525, 0.005365163087844849, -0.003605658421292901, 0.003932802472263575, 0.00032051277230493724, -0.019211314618587494, -0.001948717748746276, -0.015207778662443161, 0.015900971367955208, 0.025704683735966682, -0.036187440156936646, 0.014316532760858536, -0.012519892305135727, 0.024827582761645317, -0.04043147340416908, -0.012413791380822659, 0.018362509086728096, 0.004010609816759825, 0.03151900693774223, 0.004438549280166626, 0.021616267040371895, 0.01799469254910946, 0.009259061887860298, -0.003305039368569851, 0.023851456120610237, 0.009089300408959389, 0.0064120241440832615, 0.011402297765016556, 0.028576478362083435, -0.013312111608684063, -0.02349778637290001, -0.009980547241866589, 0.026525195688009262, -0.038648977875709534, -0.05019274353981018, 0.02170114777982235, -0.002760388655588031, 0.022719714790582657, -0.009556143544614315, -0.013870908878743649, -0.04326082766056061, 0.007126436103135347, -0.00808488018810749, 0.009513703174889088, 0.0036675503943115473, -0.018503976985812187, -0.0051989383064210415, 0.02638372778892517, 0.012781607918441296, 0.020767459645867348, -0.008360741659998894, -0.01827762834727764, 0.012972589582204819, 0.016466841101646423, -0.007119362708181143, 0.0059522539377212524, 0.0037559678312391043, 0.022479219362139702, -0.0007639256655238569, 0.013312111608684063, 0.02525198645889759, -0.009902739897370338, 0.006022987887263298, 0.007504862267524004, -0.002233421429991722, 0.03216975927352905, -0.018037132918834686, -0.015915118157863617, -0.0015022102743387222, -0.007809017784893513, 0.0019168874714523554, -0.03171706199645996, -0.008113172836601734, 0.007766577415168285, 0.0017807248514145613, 0.011763039976358414, 0.038167987018823624, 0.008622457273304462, -0.03788505494594574, 0.005106984172016382, -0.0066560558043420315, -0.01351016666740179, -0.01214500330388546, -0.02351193316280842, 0.00017860298976302147, -0.04195932298898697, -0.008679044432938099, 0.010645445436239243, -0.004063659813255072, 0.019480103626847267, 0.006320070009678602, -0.026963746175169945, 0.006022987887263298, 0.02528028003871441, -0.007412908133119345, -0.010249335318803787, 0.0023854991886764765, 0.008792218752205372, -0.03836604207754135, 0.02041379176080227, -0.023059237748384476, -0.004799291957169771, 0.021743588149547577, -0.027232535183429718, -0.025520775467157364, -0.01832006871700287, 0.003503094194456935, 0.007349247578531504, 0.009669317863881588, -0.004024756606668234, 0.009365162812173367, -0.002728558611124754, 0.008091953583061695, -0.0012298849178478122, -0.006627762224525213, -0.0009000883437693119, 0.003897435497492552, 0.008290007710456848, 0.027006186544895172, -0.022125549614429474, 0.013913349248468876, 0.009973473846912384, -0.0006653403397649527, 0.00921662151813507, -0.045722365379333496, -0.013142350129783154, -0.01977718621492386, 0.017075153067708015, -0.016735630109906197, -0.01684880442917347, -0.006132625509053469, -0.0005884172860532999, -0.005170644726604223, 0.007342174183577299, 0.012717947363853455, 0.021163569763302803, 0.000557471183128655, -0.02523783966898918, 0.005775419529527426, 0.024049511179327965, -0.03389566391706467, -0.006702032871544361, -0.006302386522293091, -0.016749776899814606, -0.0233280248939991, 0.004286471754312515, -0.01092838030308485, 0.03129265829920769, 0.005623341538012028, -0.013241377659142017, -0.03485764563083649, -0.02996286191046238, -0.030330676585435867, -0.01036250963807106, -0.017740050330758095, 0.027473028749227524, 0.013057469390332699, 0.027572056278586388, -0.009442970156669617, 0.01271087396889925, -0.002749778563156724, 0.01764102280139923, -0.01084350049495697, 0.004587090574204922, -0.00818390678614378, -0.008721484802663326, 0.0025676388759166002, 0.008749778382480145, -0.02088063396513462, -0.001087533077225089, 0.008686117827892303, -0.009110520593822002, 0.010305922478437424, 0.019239608198404312, 0.025945179164409637, -0.004265251569449902, -0.004767461679875851, 0.011444738134741783, 0.013800174929201603, 0.011183022521436214, 0.01568876951932907, -0.033782489597797394, 0.018560564145445824, 0.015250219032168388, -0.023936336860060692, -0.023243146017193794, -0.02429000660777092, 0.017244914546608925, -0.0028063657227903605, 0.0005185675690881908, -0.0028717946261167526, -0.01360212080180645, -0.0007683465373702347, -0.02738814987242222, 0.025068078190088272, -0.008537576533854008, -0.00970468483865261, 0.023228999227285385, 0.003978779539465904, 0.002887709764763713, -0.017414676025509834, 0.008381961844861507, -0.004254641477018595, 0.012017682194709778, 0.025803711265325546, -0.0404597669839859, 0.002044208347797394, 0.0030167989898473024, 0.008749778382480145, -0.02329973131418228, -0.021290890872478485, 0.0022493365686386824, -0.013396992348134518, 0.0070769223384559155, 0.02655348926782608, 0.025336867198348045, -0.0005592395318672061, 0.016438547521829605, -0.009605657309293747, 0.017782490700483322, -0.009492483921349049, -0.015830237418413162, 0.005743589252233505, -0.012809901498258114, 0.008162686601281166, -0.006001767702400684, -0.014910697005689144, -0.007554376032203436, -0.00956321693956852, 0.025662243366241455, -0.0058390796184539795, 0.014472146518528461, 0.2013368457555771, -0.011678159236907959, 0.005421750247478485, 0.02755790948867798, 0.023752428591251373, 0.0006348363822326064, 0.01051812432706356, 0.0218567606061697, 0.005899203475564718, 0.006146772298961878, 0.007731210440397263, -0.011876214295625687, -0.021771881729364395, -0.0026949599850922823, 0.0015287355054169893, -0.022097256034612656, -0.03369760885834694, -0.03567815572023392, -0.016721483319997787, 0.0076675498858094215, 0.009527849964797497, -0.017740050330758095, 0.011748893186450005, -0.003320954507216811, 0.03092484176158905, 0.0067621567286551, -0.0027957556303590536, -0.002887709764763713, 0.02022988349199295, 0.024742702022194862, -0.01587267778813839, -0.006132625509053469, 0.004689654801040888, -0.011939874850213528, -0.0018779839156195521, 0.0016772765666246414, 0.011465958319604397, 0.014019450172781944, 0.005609194748103619, 0.005598584655672312, 0.03966754674911499, 0.013743587769567966, 0.013057469390332699, -0.016141466796398163, -0.006924844346940517, 0.015052164904773235, -0.01432360615581274, -0.016622455790638924, 0.0036215733271092176, -0.004190980922430754, -0.03188682347536087, 0.014953137375414371, 0.03194341063499451, 0.02641202136874199, -0.026426168158650398, 0.004176834132522345, -0.012831121683120728, -0.0023112287744879723, 0.019183021038770676, 0.009725905023515224, -0.016792217269539833, -0.009902739897370338, 0.0007466842653229833, 0.03369760885834694, -0.023115824908018112, 0.007126436103135347, -0.00996640045195818, -0.01621219888329506, 0.024162685498595238, 0.0015517239226028323, -0.022238723933696747, -0.01358797401189804, -0.014429707080125809, 0.016778070479631424, -0.027572056278586388, -0.03723430261015892, 0.02250751294195652, 0.03307515010237694, 0.02264898084104061, 0.041534919291734695, 0.006472148001194, -0.011536692269146442, -0.010801060125231743, -0.012286471202969551, -0.03811139985918999, -0.05005127564072609, 0.012937222607433796, -0.005520777311176062, -0.0117418197914958, -0.005977010820060968, -0.004353669006377459, -0.0030450925696641207, -0.018235187977552414, 0.0004434128641150892, 0.00858709029853344, 0.006850574165582657, 0.0011715295258909464, 0.00498320022597909, 0.011437664739787579, -0.05041909217834473, -0.019706452265381813, 0.07661890983581543, 0.0016525196842849255, -0.002231653081253171, 0.022903623059391975, 0.0018726788694038987, -0.012307691387832165, 0.0160424392670393, -0.0009442970040254295, -0.009987620636820793, 0.0023430590517818928, -0.03870556503534317, 0.009421749971807003, 0.013411139138042927, 0.010638372041285038, 0.01601414568722248, -0.0015862067230045795, -0.012739167548716068, 0.028562331572175026, -0.00198585307225585, 0.009053933434188366, -0.00036427934537641704, -0.005206011701375246, 0.012406717985868454, 0.02346949279308319, -0.03924314305186272, -0.020145002752542496, 0.005460653454065323, -0.012095489539206028, -0.009273208677768707, 0.0036357201170176268, -0.03777188062667847, 0.012088416144251823, -0.004562333691865206, 0.0030910694040358067, -0.008601237088441849, 0.004396109376102686, -0.01796639896929264, -0.021135276183485985, -0.008410255424678326, 0.023073384538292885, 0.012569406069815159, 0.004548186901956797, 0.018489830195903778, 0.0009098142036236823, 0.00485941581428051, 0.010015914216637611, 0.0241060983389616, 0.0033138811122626066, -0.02397877722978592, -0.03451812267303467, -0.010468610562384129, 0.0071794865652918816, 0.01384261529892683, 0.03777188062667847, 0.0004924844834022224, -0.0030786911956965923, -0.021630413830280304, 0.021474799141287804, -0.006546418182551861, -0.04855171963572502, -0.008452695794403553, 0.018079573288559914, 0.0035260827280580997, -0.024742702022194862, -0.002427939558401704, -0.18164454400539398, 0.007069848943501711, 0.021573826670646667, -0.04286471754312515, -0.002887709764763713, -0.010730326175689697, 0.0421573780477047, -0.01812201365828514, -0.0218567606061697, -0.0029832003638148308, 0.01650928147137165, -0.008148539811372757, -0.021149422973394394, -0.003964632749557495, -0.006129088811576366, -0.011352784000337124, -0.011939874850213528, -0.0022617150098085403, 0.010277628898620605, 6.855796073068632e-07, 0.021969934925436974, -0.03946949169039726, 0.006822280585765839, 0.0053510162979364395, -0.011296196840703487, -0.0033404063433408737, -0.0022245796862989664, 0.02380901575088501, -0.00018810784968081862, -0.05338991433382034, -0.004484526347368956, 0.015193631872534752, 0.026525195688009262, 0.007547302637249231, -0.02445976808667183, 0.0019274975638836622, 0.0007997346692718565, -0.011062774807214737, -0.04003536328673363, -0.0001731874217512086, 0.030076036229729652, 0.023243146017193794, 0.010164455510675907, -0.017527850344777107, -0.017570290714502335, -0.013658707961440086, 0.01864544488489628, -0.020838193595409393, 0.013941642828285694, -0.03451812267303467, 0.020442083477973938, -0.014118477702140808, 0.011770113371312618, 0.004244031384587288, 0.0038797520101070404, -0.006603005342185497, 0.007469495292752981, -0.0034376652911305428, 0.01312112994492054, 0.007299733813852072, -0.0029053932521492243, -0.006376657169312239, 0.02653934247791767, 0.016311226412653923, -0.02025817707180977, -0.039752427488565445, -0.00921662151813507, 0.017273208126425743, -0.020130855962634087, 0.01029884908348322, -0.012640140019357204, -0.015900971367955208, 0.004212201107293367, -0.01745711639523506, 0.04343058913946152, 0.01717418059706688, 0.006139698904007673, -0.003664013696834445, 0.021969934925436974, 0.0002628204820211977, 0.002095490461215377, 0.031207777559757233, -0.005283819045871496, -0.015900971367955208, -0.0012502209283411503, -0.01482581626623869, 0.009846152737736702, 0.01929619535803795, -0.018263481557369232, -0.013736514374613762, 0.0333297923207283, -0.015957558527588844, -0.01749955676496029, 0.0001873341971077025, 0.028944293037056923, 0.011381077580153942, 0.00533686950802803, -0.012689653784036636, 0.029538458213210106, -0.01568876951932907, 0.017273208126425743, 0.0058709098957479, -0.017372235655784607, 0.0019221925176680088, 0.0234836395829916, 0.013545533642172813, 0.00036980543518438935, 0.021290890872478485, 0.04453403502702713, -0.0009372236090712249, -0.027331562712788582, -0.013913349248468876, 0.017230767756700516, -0.008657824248075485, 0.010977894067764282, 0.02591688558459282, 0.006221042480319738, -0.017131740227341652, 0.006178602110594511, -0.0030309457797557116, 0.03822457417845726, -0.01523607224225998, -0.0033881517592817545, 0.021955788135528564, -0.009938106872141361, -0.024700261652469635, -0.13490360975265503, -0.023285584524273872, 0.0027533152606338263, 0.019635718315839767, 0.00034173292806372046, 0.02816622145473957, 0.003421750385314226, 0.045920420438051224, -0.0327073335647583, 0.04886294901371002, -0.013135276734828949, -0.013757734559476376, 0.025223692879080772, -0.007801944389939308, 0.006355436984449625, -0.00200176821090281, 0.01880105957388878, -0.02591688558459282, -0.021503092721104622, 0.03881873935461044, 0.0031034478452056646, -0.012901855632662773, 0.035876210778951645, -0.015646329149603844, -0.003025640733540058, -0.004452696070075035, -0.036045972257852554, 0.021404065191745758, 0.0048417323268949986, 0.007964632473886013, 0.0027656937018036842, -0.030387263745069504, -0.000163572040037252, -0.020767459645867348, 0.010277628898620605, 0.006369583774358034, -0.0064509278163313866, -0.03202829137444496, 0.039752427488565445, 0.007059238851070404, 0.001048629404976964, 0.00808488018810749, 0.00921662151813507, 0.000598585233092308, -0.017414676025509834, -0.0161980539560318, -0.018532270565629005, 0.02948187105357647, -0.004067196510732174, -0.030217504128813744, -0.016806364059448242, 0.010921306908130646, -0.028576478362083435, 0.009259061887860298, 0.002762157004326582, 0.006051281467080116, 0.018192747607827187, 0.0251105185598135, 0.02444562129676342, 0.001795755815692246, -0.028236955404281616, -0.020809900015592575, 0.00627055624499917, -0.0055950479581952095, -0.01440141350030899, 0.010907161049544811, -0.02250751294195652, -0.012965516187250614, 0.01076569315046072, -0.015066311694681644, 0.01587267778813839, 0.012951369397342205, -0.00594871724024415, 0.010277628898620605, -0.01666489616036415, 0.0020300615578889847, -0.008410255424678326, -0.013828468509018421, 0.0026083108969032764, -0.00761803612112999, -0.017358088865876198, -0.01911228708922863, 0.022465072572231293, -0.027232535183429718, -0.01084350049495697, 0.0006450043292716146, -0.0007679044501855969, -0.024190979078412056, -0.0026861182413995266, -0.03686648607254028, -0.001740052830427885, 0.0336410216987133, 0.011918654665350914, -0.023073384538292885, -0.014373119920492172, 0.005004420410841703, -0.009004419669508934, -0.004357205703854561, -0.011614499613642693, 0.02690715901553631, 0.0010893014259636402, -0.002603005850687623, -0.04855171963572502, 0.029085760936141014, 0.0032749774400144815, -0.01653757505118847, -2.2325373720377684e-05, 0.00466136122122407, 0.01973474584519863, -0.003343943040817976, -0.024643676355481148, -0.008608310483396053, -0.027289122343063354, 0.01798054575920105, -0.018659591674804688, -0.00874270498752594, -0.010206895880401134, -0.031320951879024506, 0.012251104228198528, -0.0016215736977756023, 0.003476568963378668, 0.026793984696269035, -0.01223695743829012, -0.005450043827295303, 0.03321661800146103, 0.01181962713599205, -0.017414676025509834, -0.006076038349419832, -0.034065425395965576, 0.02917064167559147, -0.007108752615749836, -0.029510164633393288, -0.0016171528259292245, 0.004017683211714029, -0.01407603733241558, 0.009145887568593025, -0.0017241376917809248, 0.003798407968133688, 0.0020282932091504335, 0.011183022521436214, 0.019635718315839767, 0.030076036229729652, -0.011069848202168941, -0.01686295121908188, -0.004053049720823765, -0.041365157812833786, 0.005294429138302803, -0.002877099672332406, -0.009117593988776207, -0.016735630109906197, 0.0037842614110559225, 0.008862951770424843, 0.0005822280654683709, 0.009690538048744202, -0.00575066264718771, -0.03468788415193558, -0.018221041187644005, -0.01465605478733778, -0.0074482751078903675, 0.009351016022264957, 0.033131737262010574, -0.00769584346562624, 0.042355433106422424, -0.010489830747246742, 0.006486294791102409, -0.0076038893312215805, 0.009513703174889088, 0.017612729221582413, -0.0026755081489682198, 0.00016014586435630918, -0.0032254639081656933, -0.01554730162024498, -0.004488063044846058, 0.0014022986870259047, 0.017103446647524834, 0.02090892754495144, 0.00850928295403719, -0.025195399299263954, 0.0039717061445117, 0.014882403425872326, -0.012470378540456295, 0.024360740557312965, 0.02670910395681858, 0.0036357201170176268, -0.022946063429117203, 0.0270486269146204, 0.02055525779724121, 0.0033492480870336294, 0.005301502533257008, 0.013418212532997131, -0.028081340715289116, 0.003106984542682767, 0.009711758233606815, 0.019069846719503403, -0.008954905904829502, 0.012159150093793869, 0.0019098140764981508, 0.019678158685564995, -0.006245799362659454, 0.007172413170337677, 0.047872673720121384, 0.023837309330701828, 0.001839080243371427, -0.0061043319292366505, -0.007218390237540007, -0.012335984036326408, -0.03429177403450012, -0.007879751734435558, -0.046401411294937134, -0.02948187105357647, -0.006956674624234438, 0.026751544326543808, 0.013340405188500881, 0.014783375896513462, 0.007950485683977604, 0.0008377541089430451, -0.0016268787439912558, 0.024502208456397057, 0.00315826665610075, -0.001591511769220233, -0.02786913886666298, 0.017541997134685516, 0.030415557324886322, 0.02253580652177334, 0.04538284242153168, 0.0008965516462922096, 0.034065425395965576, -0.010553491301834583, 0.02266312763094902, -0.02252165973186493, 0.006206895690411329, 0.033754196017980576, -0.007787797600030899, -0.00792926549911499, 0.01989036053419113, -0.017117593437433243, -0.015745356678962708, -0.030698493123054504, -0.02153138630092144, 0.011784260161221027, 0.02154553309082985, 0.07910874485969543, 0.028717946261167526, -0.004668434616178274, 0.006702032871544361, -0.011593279428780079, -0.003653403604403138, 0.00013118919741827995, 0.01164986565709114, -0.01599999889731407, -0.015306806191802025, 0.0016357203712686896, 0.008155613206326962, 0.02021573670208454, -0.026808131486177444, -0.006691422779113054, 0.009414676576852798, 0.007724137045443058, -0.000503536663018167, 0.002532272133976221, -0.0136799281463027, 0.0404597669839859, -0.027232535183429718, 0.028562331572175026, 0.015278512611985207, -0.013573827221989632, -0.01004420779645443, 0.025266133248806, 0.004431475885212421, -0.0035738281439989805, -0.0001597037771716714, 0.03075508028268814, 0.01523607224225998, -0.03129265829920769, -0.01183377392590046, -0.0022245796862989664, -0.0010778071591630578, 0.008601237088441849, -0.008304154500365257, 0.018334215506911278, -0.004916002973914146, 0.024205125868320465, -0.020343057811260223, -0.01697612553834915, -0.0028983198571950197, -0.011197169311344624, 0.003738284343853593, 0.008035366423428059, -0.020116709172725677, -0.017513703554868698], 'text': 'plan to refine these methods and register performance predictions across various capabilities before\\nlarge model training begins, and we hope this becomes a common goal in the field.\\n4 Capabilities\\nWe tested GPT-4 on a diverse set of benchmarks, including simulating exams that were originally\\ndesigned for humans.4We did no specific training for these exams. A minority of the problems in the\\nexams were seen by the model during training; for each exam we run a variant with these questions\\nremoved and report the lower score of the two. We believe the results to be representative. For further\\ndetails on contamination (methodology and per-exam statistics), see Appendix C.\\nExams were sourced from publicly-available materials. Exam questions included both multiple-\\nchoice and free-response questions; we designed separate prompts for each format, and images were\\nincluded in the input for questions which required it. The evaluation setup was designed based', 'SimilarityScore': 0.8285566330929187}), 0.8285566330929187)\n", - "(Document(page_content='overall GPT-4 training budget. When mixing in data from these math benchmarks, a portion of the\\ntraining data was held back, so each individual training example may or may not have been seen by\\nGPT-4 during training.\\nWe conducted contamination checking to verify the test set for GSM-8K is not included in the training\\nset (see Appendix D). We recommend interpreting the performance results reported for GPT-4\\nGSM-8K in Table 2 as something in-between true few-shot transfer and full benchmark-specific\\ntuning.\\nF Multilingual MMLU\\nWe translated all questions and answers from MMLU [ 49] using Azure Translate. We used an\\nexternal model to perform the translation, instead of relying on GPT-4 itself, in case the model had\\nunrepresentative performance for its own translations. We selected a range of languages that cover\\ndifferent geographic regions and scripts, we show an example question taken from the astronomy', metadata={'id': 'a7e86563-592a-4304-962b-25315f0d3f46', 'embedding': [-0.005677336826920509, -0.007514122407883406, 0.013824593275785446, -0.029277246445417404, -0.005625155754387379, 0.020705582574009895, 0.00037940230686217546, -0.006303513888269663, -0.042524367570877075, -0.03512156382203102, 0.025394950062036514, 0.05716298893094063, -0.031086202710866928, -0.008022021502256393, 0.0013036654563620687, 0.03437015414237976, 0.015584846027195454, 0.013984616845846176, 0.0026055914349853992, -0.02005157433450222, -0.01544569618999958, 0.008731688372790813, -0.029945168644189835, -0.005879105068743229, 0.007695017848163843, 0.0021742251701653004, 0.034147512167692184, -0.02949988842010498, -0.019981998950242996, 0.01817304454743862, 0.008279449306428432, -0.0009810103802010417, -0.037681933492422104, 0.006557463202625513, -0.01828436367213726, -0.017171161249279976, 0.0107215391471982, -0.01224523689597845, 0.03431449085474014, -0.0044841221533715725, 0.021777039393782616, 0.030780071392655373, 0.023029394447803497, -0.002196837216615677, 0.024727027863264084, 0.023432929068803787, 0.009830976836383343, -0.0059834676794707775, -0.009253502823412418, 0.015390035696327686, 6.017168198013678e-05, 0.020093319937586784, -0.016336258500814438, -0.0015419605188071728, -0.012878370471298695, 0.0027969232760369778, 0.007423674687743187, 0.0176721028983593, -0.009190885350108147, 0.006741837598383427, -0.007152331527322531, -0.0024316534399986267, -0.02152656950056553, 0.029945168644189835, -0.012537452392280102, -0.03918475657701492, -0.03659655898809433, 0.0030056489631533623, -0.00324220466427505, 0.008703858591616154, 0.0041466825641691685, 0.02422608807682991, 0.001679371576756239, -0.030112149193882942, 0.048285193741321564, -0.01651715487241745, 0.000572690914850682, -0.01476385910063982, -0.018534835427999496, 0.011431206949055195, 0.03228289633989334, -0.013748060911893845, -0.006275683641433716, 0.030724411830306053, 0.0056634219363331795, 0.020274216309189796, 0.014262917451560497, 0.001909839455038309, 0.004052755888551474, 0.007896785624325275, 0.016350174322724342, -0.00249601062387228, 0.023057224228978157, 0.01571008190512657, -0.0030352184548974037, -0.00639744009822607, -0.009858806617558002, 0.03192110359668732, -0.014833434484899044, -0.036095619201660156, -0.020260300487279892, 0.0032439441420137882, -0.019258417189121246, -0.008091596886515617, -0.038405515253543854, 0.017143331468105316, 0.018562665209174156, -0.010930265299975872, -0.012175661511719227, 0.0022629336453974247, -0.021610058844089508, 0.03219940513372421, -0.003892732784152031, -0.023168543353676796, -0.013087096624076366, 0.0011740815825760365, 0.018590494990348816, -0.005792135838419199, -0.01088156271725893, 0.005381642375141382, 0.017630357295274734, -0.012356556951999664, 0.026619473472237587, -0.00556253781542182, 0.03373005986213684, 0.02518622577190399, -0.006220023613423109, -0.0250609889626503, -0.03598429635167122, -0.004616315010935068, 0.0015637028263881803, 0.017922572791576385, 0.012760093435645103, 0.0013723709853366017, -0.02883196622133255, 0.02083081752061844, -0.024615708738565445, -0.007347141858190298, -0.025840232148766518, -0.008968244306743145, 0.0033309131395071745, 0.024504387751221657, 0.014200299978256226, -0.009601378813385963, 0.002789965830743313, 0.029444226995110512, 0.04068757966160774, 0.009156097657978535, 0.0056912521831691265, -0.02080298773944378, -0.008870839141309261, -0.029889509081840515, -0.004188427701592445, 0.01545961108058691, 0.015807487070560455, -0.02300156280398369, -0.004264960065484047, 0.01758861169219017, -0.016057956963777542, -0.0038301150780171156, -0.017171161249279976, 0.0189801175147295, 0.02558976039290428, 0.003910126630216837, 0.002181182848289609, 0.029750358313322067, 0.015403950586915016, 0.0005635591805912554, 0.0009149139514192939, -0.016405833885073662, -0.024267831817269325, 0.029138097539544106, -0.026563813909888268, 0.028177959844470024, 0.006439185235649347, 0.011758210137486458, 0.016725879162549973, -0.012711390852928162, -0.0068601155653595924, -0.006226981058716774, -0.0010749369394034147, 0.014680368825793266, 0.009928382001817226, -0.010102320462465286, -0.00798027589917183, -0.01834002509713173, 0.013560207560658455, 0.012808796018362045, -0.0072427792474627495, -0.019133182242512703, 0.017226820811629295, 0.012683560140430927, -0.002816056599840522, 0.009399610571563244, -0.6193863153457642, -0.01657281443476677, 0.014346407726407051, 0.00012599634646903723, 0.015890978276729584, 0.009448313154280186, 0.0013349743094295263, -0.010387578047811985, -0.040381450206041336, 0.07091104984283447, -0.005753869656473398, -0.0004900703788734972, -0.029026776552200317, -0.005350333638489246, 0.009065649472177029, -0.02540886588394642, -0.012725305743515491, -0.013706316240131855, 0.012947945855557919, 0.008564707823097706, -0.023419015109539032, 0.01177908293902874, 0.008738646283745766, 0.03094705194234848, -0.013434972614049911, 0.010443238541483879, -0.0024229565169662237, -0.019314076751470566, 0.008286407217383385, -0.00564950704574585, -0.03175412490963936, 0.011549483984708786, 0.0003996226005256176, 0.0004383237974252552, 0.03924041613936424, -0.017435546964406967, -0.03147582337260246, 0.014249002560973167, 0.010032745078206062, 0.008362939581274986, -0.03019564040005207, -0.012836625799536705, 0.009670954197645187, 0.025603676214814186, -0.007340184412896633, 0.02283458225429058, -0.0010540643706917763, 0.01069370936602354, 0.003986659459769726, -0.0064461431466042995, -0.010095362551510334, 0.010046659968793392, 0.009037819691002369, 0.002309896983206272, 0.00671400735154748, 0.009253502823412418, 0.012120001018047333, -0.009350907988846302, -0.004261481575667858, -0.006087830755859613, -0.014638623222708702, 0.014075064100325108, 0.0032300290185958147, -0.009851849637925625, -0.03150365501642227, -0.01692069135606289, -0.004445855971425772, 0.013720231130719185, 0.00841860007494688, -0.025784572586417198, 0.007354099303483963, 0.017839083448052406, 0.015111735090613365, -0.003367440076544881, -0.0009479622240178287, 0.014262917451560497, 0.015014329925179482, -0.025047075003385544, 0.013261034153401852, 0.01769993267953396, 0.007674145512282848, -0.01335844025015831, 0.015890978276729584, -0.001111463992856443, 0.015195225365459919, -0.013240162283182144, 0.007479334715753794, 0.014986500144004822, 0.0006270465673878789, 0.015626590698957443, 0.00738192955031991, 0.030613090842962265, -0.026925604790449142, -0.026243768632411957, 0.001408028299920261, 0.013845466077327728, -0.025659335777163506, -0.00018937501590698957, 0.02877630665898323, -0.03748712316155434, -0.012739220634102821, 0.000315914920065552, 0.05362857133150101, 0.023683400824666023, 0.02170746400952339, 0.014903009869158268, -0.02634117379784584, -0.014346407726407051, -0.012120001018047333, -0.013233204372227192, -0.0006096527795307338, 0.013365397229790688, 0.0019950689747929573, 0.0038301150780171156, 0.014012446627020836, -0.03656872734427452, 0.011166821233928204, 0.013233204372227192, -0.016016213223338127, -0.005604282952845097, -0.002468180377036333, -0.011751252226531506, 0.04344275966286659, -0.012419174425303936, 0.011639932170510292, 0.014429898001253605, 0.00999795738607645, -0.021262183785438538, 0.019773274660110474, 0.024991415441036224, -0.03320128843188286, -0.00814029946923256, 0.049537546932697296, -0.0231824591755867, 0.04912009462714195, 0.009336993098258972, 0.02572891116142273, -0.0052911946550011635, 0.013490633107721806, -0.023780805990099907, -0.04099371284246445, -0.0021011712960898876, 0.006348737515509129, -0.00981010403484106, -0.016030127182602882, -0.01753295212984085, -0.01657281443476677, -0.003750103758648038, 0.010575431399047375, -0.0035570324398577213, 0.0006470494554378092, -0.0028369291685521603, -0.007159288972616196, -0.0007114065228961408, -0.0013723709853366017, -0.023224204778671265, -0.010603261180222034, -0.03384138271212578, -0.022124916315078735, -0.03386921063065529, 0.007764593232423067, 0.01109724584966898, 0.011514697223901749, 0.017574697732925415, 0.009684869088232517, -6.560724432347342e-05, -0.011445121839642525, 0.016057956963777542, -0.011883445084095001, -0.025854147970676422, 0.0002359034406254068, -0.037765420973300934, 0.0075906552374362946, 0.011486866511404514, -0.023516420274972916, 1.8535271010478027e-05, -0.007388886995613575, -0.0026942999102175236, -0.019467143341898918, -0.029833849519491196, 0.007166246417909861, 0.01660064421594143, -0.025074904784560204, -0.008342067711055279, 0.04166163504123688, 0.02778833732008934, 0.02433740720152855, -0.0009001292637549341, -0.0068009765818715096, 0.008912583813071251, 0.012947945855557919, -0.005343375727534294, -0.029889509081840515, 0.019898509606719017, -0.0019133181776851416, 0.014652539044618607, 0.022097086533904076, -0.02112303301692009, 0.005232055671513081, 0.027635272592306137, 0.02315462939441204, 0.01657281443476677, 0.01646149344742298, -0.008015063591301441, 0.008056809194386005, -0.015000415034592152, 0.010749369859695435, -0.00250992551445961, 0.01611361838877201, 0.004390195477753878, 0.002487313700839877, -0.015501356683671474, -0.025144480168819427, -0.012377429753541946, -0.0010731975780799985, -0.006181756965816021, -0.003156975144520402, -0.007020138204097748, -0.02077515609562397, -0.0019550633151084185, 0.003146538743749261, 0.0107215391471982, 0.0140541922301054, 0.011528612114489079, -0.007841126061975956, 0.017825167626142502, -0.006567899603396654, 0.011173778213560581, 0.03715316206216812, -0.014172470197081566, -0.0037048798985779285, 0.001341931871138513, 0.029360737651586533, 0.015097820200026035, 0.026355087757110596, -0.006550505757331848, 0.023655571043491364, 0.014040276408195496, 0.029082436114549637, -2.145688449672889e-05, 0.02366948500275612, -0.0011984329903498292, -0.0004974627518095076, -0.02321028895676136, 0.021665720269083977, -0.010798072442412376, 0.023919956758618355, 0.0035413780715316534, -0.03253336623311043, 0.020329875871539116, -0.010700667276978493, 0.0027430024929344654, -0.022820668295025826, -0.005472090095281601, 0.005162480287253857, -0.020566431805491447, -0.0033813551999628544, 0.015334376133978367, 0.00011490780161693692, 0.020886477082967758, 0.013337567448616028, 0.011799954809248447, -0.00599042559042573, -0.012453962117433548, 0.018813136965036392, -0.015056074596941471, -1.521957619843306e-05, -0.004859828390181065, -0.027510037645697594, 0.007500207517296076, -0.011535569094121456, -0.022375386208295822, 0.003913605585694313, -0.030084319412708282, 0.02257019653916359, 0.011278141289949417, 0.0029569463804364204, 0.016503239050507545, 0.03239421546459198, 0.022375386208295822, -0.02958337776362896, -0.030418280512094498, 0.02382255159318447, 0.01003970205783844, -0.01973152905702591, -0.01022755540907383, -0.029221586883068085, -0.0016419749008491635, -0.014346407726407051, -0.007903743535280228, -0.025506271049380302, -0.0021846615709364414, -0.011264226399362087, -0.00025634115445427597, -0.01715724542737007, -0.004306705202907324, 0.02147090993821621, 0.005033766385167837, 0.00604260666295886, -0.00795244611799717, -0.010060574859380722, 0.0006609644624404609, -0.03937956690788269, -0.00884996633976698, 0.009009988978505135, -0.0021255225874483585, -0.022584112361073494, -0.021164778620004654, 0.0034074457362294197, -0.011347716674208641, 0.024212172254920006, -0.007875913754105568, -0.0026542942505329847, 0.008696900680661201, 0.013803721405565739, 0.0027116937562823296, -0.009858806617558002, 0.003529202425852418, 0.03612344712018967, 0.012147830799221992, -0.021512653678655624, -0.01906360685825348, -0.015097820200026035, 0.014304663054645061, 0.0359286367893219, 0.03094705194234848, -0.015376120805740356, -0.00949005875736475, -0.031086202710866928, 0.0032630772329866886, -0.024379152804613113, -0.03192110359668732, -0.005722560919821262, 0.006265247240662575, -0.007263651583343744, -0.0033639613538980484, 0.00930220540612936, -0.007009702268987894, 0.02961120754480362, 0.006623559631407261, -0.010192767716944218, -0.0208586473017931, -0.013977658934891224, 0.008244662545621395, 0.007451504934579134, -0.027106501162052155, 0.021025627851486206, 0.016266683116555214, 0.004758944269269705, -0.003341349307447672, 0.012913158163428307, -0.0015045638429000974, -0.014047234319150448, -0.02880413644015789, -0.02077515609562397, 0.0009749226155690849, 0.014875179156661034, 0.004560654982924461, -0.006091309245675802, 0.007555867545306683, 0.007201034110039473, -0.0072427792474627495, 0.012147830799221992, -0.010276257991790771, 0.019369738176465034, -0.0014863003743812442, -0.003078702837228775, -0.010714582167565823, 0.008801263757050037, -0.014916924759745598, 0.005033766385167837, 0.032060254365205765, 0.002301200060173869, -0.027913574129343033, -0.00046310998732224107, 0.005771263502538204, -0.014485558494925499, -0.011667761951684952, 0.025603676214814186, -0.0041118948720395565, -6.598773325094953e-05, 0.008578622713685036, -0.025478441268205643, -0.028720645233988762, -0.013671528548002243, -0.03286732733249664, 0.02575674094259739, -0.0017532951897010207, -0.03239421546459198, 0.010081447660923004, -0.019842850044369698, -0.011862573213875294, -0.02540886588394642, 0.012760093435645103, -0.030056489631533623, -0.021957935765385628, -0.04848000407218933, -0.013114926405251026, 0.01476385910063982, 0.02401736192405224, 0.03142016381025314, -0.0075906552374362946, -0.007771550677716732, 0.008307280018925667, -0.014165512286126614, -0.016364088281989098, 0.004546739626675844, -0.026090702041983604, -0.014889094047248363, -0.0022351036313921213, -0.009650081396102905, -0.0011653847759589553, 0.0020559474360197783, 0.01817304454743862, -0.013915041461586952, -0.0007492380100302398, -0.008863881230354309, -0.004491079598665237, 0.019870679825544357, 0.001178430044092238, 0.01726856641471386, -0.005162480287253857, 0.025283630937337875, -0.009135224856436253, -0.008989117108285427, -0.03437015414237976, -0.026229852810502052, -0.04283049702644348, 0.017783423885703087, 0.01831219531595707, 0.0044215042144060135, 0.016670219600200653, 0.017087670043110847, -0.00534685468301177, 0.007548910100013018, -0.020260300487279892, 0.0012949685333296657, 0.004602400120347738, 0.012913158163428307, -0.01943931356072426, 0.019842850044369698, 0.020580345764756203, -0.003348306752741337, -0.031253181397914886, -0.000644440355245024, -0.04383238032460213, 0.027370886877179146, 0.016906775534152985, 0.0038649027701467276, 0.033423930406570435, -0.0007918528281152248, -0.0015671815490350127, -0.050344619899988174, 0.003739667357876897, -0.00011577749683056027, 0.03921258822083473, 0.002102910540997982, 0.0016350173391401768, -0.0072149490006268024, -0.00029373783036135137, -0.02436523698270321, 0.005739954765886068, 0.00033570037339814007, 0.004216257482767105, -0.025812402367591858, 0.010839817114174366, 0.0005935634835623205, -0.0070270961150527, 0.007016659714281559, -0.026563813909888268, -0.02002374455332756, 0.011278141289949417, 0.003850987646728754, 0.023766890168190002, 0.017310312017798424, 0.013191459700465202, -0.0338970422744751, -0.021248267963528633, -0.019884593784809113, -0.016016213223338127, 0.004132767207920551, -0.014026361517608166, 0.0253532063215971, 0.02205534093081951, 0.04650406911969185, -0.015097820200026035, -0.009768359363079071, 0.015835316851735115, -0.011723422445356846, 0.0013341045705601573, -0.012746177613735199, 0.008035936392843723, -0.0027951840311288834, -0.010874604806303978, 0.0010714582167565823, -0.01408202201128006, 0.01625276915729046, -0.008195959031581879, -0.003115230007097125, 0.030640920624136925, -0.013448887504637241, 0.010116235353052616, -0.015863146632909775, -0.020524686202406883, -0.013441930525004864, 0.027370886877179146, -0.016266683116555214, -0.008766476064920425, -0.014471643604338169, -0.032060254365205765, 0.01002578716725111, -0.0022420610766857862, -0.0072358218021690845, 0.01342801470309496, 0.02848408930003643, -0.020037660375237465, -0.015988383442163467, -0.00792461633682251, 0.013581080362200737, -0.02277892269194126, -0.011215523816645145, -0.044082850217819214, -0.0011358152842149138, 0.01315667200833559, -0.020329875871539116, 0.018047809600830078, -0.0017445983830839396, -0.0060808733105659485, -0.018075639382004738, 0.005385120864957571, -0.016308428719639778, -0.014200299978256226, -0.012029553763568401, -0.03172629326581955, -0.011333800852298737, -0.012182618491351604, 0.009573548100888729, 0.013448887504637241, -0.003656177083030343, 0.017198991030454636, 0.004525867290794849, 0.02489400841295719, -0.024379152804613113, -0.003551814239472151, 0.024406982585787773, 0.011827785521745682, 0.05351724848151207, 0.005642549134790897, 0.007848083041608334, 0.041745126247406006, 0.0075976126827299595, -0.027370886877179146, 0.012440047226846218, 0.011145948432385921, 0.0028873709961771965, 0.02848408930003643, 0.033479589968919754, -0.026229852810502052, -0.011786039918661118, 0.019564548507332802, 0.01756078191101551, -0.01973152905702591, -0.0385446660220623, 0.02135958895087242, 0.019188841804862022, 0.020677750930190086, -0.01854874938726425, -0.006874030455946922, -0.0365130677819252, 0.01409593690186739, 0.000167632766533643, 0.00490505201742053, 0.007660230156034231, 0.007444547023624182, -0.0008357721962966025, 0.03142016381025314, 0.0012688778806477785, 0.004240608774125576, -0.008961286395788193, -0.0068009765818715096, 0.023405099287629128, -0.00220553413964808, -0.005736475810408592, 0.010763284750282764, 0.0033865731675177813, 0.03990833833813667, -0.011674719862639904, 0.015723997727036476, 0.007917658425867558, -0.012467877008020878, 0.013038394041359425, 0.0009984042262658477, 0.007312354166060686, 0.031308844685554504, -0.0324498750269413, 0.004522388335317373, 0.002577761420980096, -0.005381642375141382, -0.0009410046623088419, 0.002958685625344515, -0.009211757220327854, -0.017254650592803955, -0.012732262723147869, 0.015292630530893803, 0.012746177613735199, 0.003958829212933779, -0.015236970037221909, 0.016614560037851334, -0.001603708486072719, -0.02749612182378769, -0.019661953672766685, -0.013761975802481174, 0.030000830069184303, -0.04377672076225281, -0.014708198606967926, 0.003690964775159955, -0.013532377779483795, 0.025548016652464867, -0.01184865739196539, -0.025019245222210884, 0.0005809529684484005, 0.00982401892542839, -0.016475409269332886, -0.0023603388108313084, -4.606639777193777e-06, -0.008516005240380764, -0.020037660375237465, 0.006553984712809324, -0.016684135422110558, -0.005319024436175823, 0.019773274660110474, -0.028971116989850998, -0.014680368825793266, -0.016044043004512787, 0.005684294272214174, -0.001927233301103115, 0.01477777399122715, -0.015264800749719143, -0.0009870982030406594, -0.014569048769772053, 0.007030574604868889, -0.016711965203285217, 0.006265247240662575, -0.0024368716403841972, -0.007354099303483963, 0.01040149386972189, 0.018840966746211052, -0.011236395686864853, 0.0072358218021690845, 0.0009870982030406594, -0.0022559762001037598, -0.007263651583343744, -0.03100271336734295, 0.0012958382721990347, 0.006077394355088472, 0.017379887402057648, -0.02422608807682991, -0.006195672322064638, -0.015751827508211136, 0.00770893320441246, -0.012857498601078987, 0.007507164962589741, -0.00432409904897213, 0.00841860007494688, 0.022528452798724174, -0.0012123479973524809, 0.02147090993821621, 0.01616927795112133, -0.0486748144030571, -0.001636756700463593, -0.01159122958779335, 0.009051734581589699, -0.007472377270460129, 4.805037679034285e-05, 0.009030861780047417, 0.04146682471036911, -0.007841126061975956, -0.028233619406819344, -0.023419015109539032, -0.017018096521496773, -0.03242204710841179, -0.036874860525131226, -0.013789806514978409, 0.019856764003634453, 0.009830976836383343, 0.048257362097501755, -0.01113899052143097, 0.03573382645845413, 0.018938371911644936, 0.008202916942536831, -0.011069415137171745, 0.0013375834096223116, -0.017435546964406967, -0.008738646283745766, -0.004404110834002495, 0.0029395525343716145, -0.04850783571600914, -0.026675134897232056, 0.0009975344873964787, -0.019522802904248238, 0.01654498465359211, 0.021957935765385628, 0.015807487070560455, -0.02880413644015789, -0.007034053560346365, 0.008321194909512997, 0.013163628987967968, 0.006480930373072624, 0.025923721492290497, -0.032060254365205765, 0.007194076664745808, 0.006289598532021046, -0.013240162283182144, -0.021749209612607956, -0.009246544912457466, 0.02417042665183544, -0.019230587407946587, -0.00036722663207910955, -0.003649219637736678, -0.006515718065202236, 0.029917338863015175, -0.030390450730919838, 0.025979382917284966, 0.011034627445042133, -0.000648353947326541, 0.012940988875925541, 0.01619710773229599, 0.005948680453002453, -0.014304663054645061, 0.009893594309687614, -0.015209140256047249, -0.015014329925179482, 0.04163380339741707, -0.019216671586036682, -0.017602527514100075, -0.02048294059932232, -0.003341349307447672, -0.019828934222459793, -0.012989691458642483, -0.0035239842254668474, 0.0009262199746444821, -0.035483356565237045, 0.03225506469607353, 0.012140873819589615, -0.027816167101264, 0.0075697824358940125, -0.008759519085288048, 0.011236395686864853, -0.001740249921567738, -0.012620942667126656, -0.004000574350357056, -0.03400836139917374, -0.001925493823364377, -0.0020820379722863436, -0.0016915472224354744, -0.008355982601642609, -0.009552676230669022, 0.0009957951260730624, -0.015111735090613365, 0.013504547998309135, 0.1903577744960785, -0.03673570975661278, -0.00273082684725523, 0.03175412490963936, 0.014986500144004822, 0.005465132649987936, 0.015974467620253563, 0.01900794729590416, 0.006814891472458839, -0.0030908784829080105, -0.01587706245481968, -0.015334376133978367, 0.0009236108744516969, -0.0021342195104807615, 0.004407589323818684, -0.019578462466597557, -0.026299428194761276, -0.026939520612359047, -0.0063835252076387405, 0.012092171236872673, 0.0007192337070591748, -0.008474260568618774, 0.004365844186395407, -0.01476385910063982, 0.006668783724308014, 0.003844030201435089, -0.004623272456228733, 0.006021734327077866, 0.014235087670385838, 0.000670531066134572, -0.007361056748777628, -0.006282641086727381, 0.006185235921293497, -0.006094788201153278, -0.0038057637866586447, 0.0018820093246176839, 0.011278141289949417, -0.004585006274282932, 0.020496856421232224, 0.02034378983080387, 0.016823284327983856, 0.0028664986602962017, 0.0199959147721529, -0.0225980281829834, -0.011939105577766895, 0.027857912704348564, -0.010484984144568443, 0.015529186464846134, -0.010324960574507713, -0.004842434544116259, -0.015835316851735115, 0.004369323141872883, 0.026466408744454384, 0.016294512897729874, -0.011883445084095001, 0.0024281747173517942, -0.014847349375486374, -0.010192767716944218, 0.0029291161336004734, 0.004251045174896717, -0.022124916315078735, -0.005148565396666527, -0.02390604093670845, 0.037654101848602295, -0.02685602940618992, 0.015195225365459919, -0.004157118499279022, -0.009524845518171787, 0.017059840261936188, -0.00612957589328289, -0.02176312543451786, -0.002949988702312112, -0.008801263757050037, -0.004348450340330601, -0.03445364162325859, -0.038989946246147156, 0.020246384665369987, 0.029277246445417404, 0.02572891116142273, 0.036930520087480545, -0.011946063488721848, -0.026035042479634285, 0.004553697537630796, -0.028943287208676338, -0.025520186871290207, -0.04792340472340584, 0.001739380182698369, -0.02590980753302574, -0.00046267511788755655, -0.005969552788883448, -0.007124501280486584, -0.022389302030205727, -0.008780390955507755, 0.004177991300821304, -0.002911722520366311, 0.0055764527060091496, 0.025923721492290497, -0.015584846027195454, -0.010088404640555382, -0.04850783571600914, -0.025951553136110306, 0.05468611419200897, 0.00475198682397604, 0.013497590087354183, 0.02222232148051262, 0.011521654203534126, -0.009434398263692856, 0.008856924250721931, -0.00021927061607129872, -0.008175087161362171, 0.0034248395822942257, -0.03954654932022095, 0.018242619931697845, 0.0014471643371507525, -0.00694012688472867, 0.0044388980604708195, -0.0034561483189463615, 0.00884996633976698, 0.034815434366464615, -0.012732262723147869, 0.008286407217383385, -0.01596055179834366, 0.013149714097380638, 0.00692621199414134, 0.001561093726195395, -0.03142016381025314, -0.013476717285811901, 0.0015054335817694664, -0.023572079837322235, -0.01576574146747589, 0.019453227519989014, -0.07085539400577545, 0.0176721028983593, -0.02184661477804184, 0.002920419443398714, 0.012544410303235054, -0.0038822966162115335, -0.006112182047218084, -0.032700348645448685, -0.0031100118067115545, 0.033535249531269073, 0.006734880153089762, -0.0028369291685521603, -0.009023904800415039, 0.017463376745581627, -0.01646149344742298, 0.010686751455068588, 0.021136948838829994, -0.011612102389335632, -0.013191459700465202, -0.02387821115553379, -0.0007431501871906221, -0.006661826279014349, 0.00954571831971407, 0.04102154076099396, -0.002916940487921238, -0.013671528548002243, -0.015501356683671474, 0.018646156415343285, -0.004932882264256477, -0.03790457174181938, -0.0168093703687191, 0.016503239050507545, -0.007068841252475977, -0.012238278985023499, -0.0008657764992676675, -0.177889883518219, -0.001493257936090231, -0.0052911946550011635, -0.02179095521569252, 0.026327257975935936, -0.0009009989444166422, 0.030668752267956734, 0.004132767207920551, -0.02709258534014225, -0.011027670465409756, 0.0072427792474627495, -0.0035013724118471146, -0.0008057678933255374, 0.00022133612947072834, -0.027941403910517693, 0.0017524255672469735, -0.016531068831682205, 0.007778508123010397, 0.019842850044369698, 0.011758210137486458, 0.038934286683797836, -0.020538602024316788, 0.0026508152950555086, -0.007493250072002411, -0.003927520476281643, -0.0064357067458331585, -0.0008957808022387326, 0.029277246445417404, -0.004404110834002495, -0.03857249394059181, -0.011521654203534126, 0.008495132438838482, 0.01090939249843359, 0.009455271065235138, 0.0044215042144060135, 0.016558898612856865, 0.03153148293495178, -0.00039114311221055686, -0.025102734565734863, 0.011521654203534126, 0.034036193042993546, 0.02283458225429058, -0.015000415034592152, 0.004327578004449606, -0.029082436114549637, 0.014214214868843555, 0.0231824591755867, -0.018298279494047165, 0.01871572993695736, 0.013149714097380638, 0.02845625951886177, -0.009907509200274944, 0.010533686727285385, -0.0032717741560190916, -0.004529345780611038, 0.004254524130374193, -0.0013636740623041987, -0.013469760306179523, 0.00910043716430664, -0.011890402995049953, -0.01477777399122715, -0.011438163928687572, 0.016823284327983856, -0.0035083298571407795, -0.005141607951372862, -0.025367120280861855, -0.02051077038049698, 0.020900392904877663, -0.006901860702782869, 0.010290172882378101, -0.023544250056147575, -0.016294512897729874, 0.01003970205783844, -0.0075628249906003475, 0.033368270844221115, 0.017727762460708618, -0.005298152100294828, 0.009615293703973293, 0.024114767089486122, 0.01761644333600998, -0.01721290685236454, 0.03456496447324753, -0.007903743535280228, 0.0006840112619102001, -0.012767050415277481, -0.007034053560346365, 0.007065362296998501, 0.024351323023438454, -0.005127692595124245, -0.008168129250407219, 0.013038394041359425, 0.0028578017372637987, 0.011020712554454803, 0.013789806514978409, 0.04221823439002037, -0.001977675361558795, -0.011145948432385921, -0.012370471842586994, 0.01943931356072426, -0.016350174322724342, -0.003252641065046191, 0.01863224059343338, -0.020427281036973, -0.005259885918349028, 0.03192110359668732, 0.00951788853853941, -0.009497015736997128, 0.014652539044618607, 0.028205789625644684, -0.015835316851735115, -0.01622493751347065, -0.014360322616994381, 0.0029969520401209593, 0.02631334401667118, -0.005712124519050121, 0.028442345559597015, 0.004285832867026329, -0.025422781705856323, 0.021999681368470192, 0.0072358218021690845, 0.03818287327885628, -0.026925604790449142, -0.0026786455418914557, 0.015932722017169, -0.009657038375735283, -0.01657281443476677, -0.12167312204837799, -0.001468036905862391, 0.02379472181200981, 0.03219940513372421, -0.004397152923047543, 0.030362620949745178, -0.0003496154095046222, 0.04569699615240097, -0.012627900578081608, 0.04360973834991455, -0.025047075003385544, -0.017783423885703087, 0.008717773482203484, 0.0014567308826372027, -0.021665720269083977, -0.006313950289040804, 0.04419417306780815, -0.02543669566512108, -0.01335844025015831, 0.03793240338563919, 0.022403215989470482, 0.00021655282762367278, 0.01790865883231163, -0.026953434571623802, -0.006460058037191629, -0.014249002560973167, -0.05148565396666527, 0.028442345559597015, 0.012906201183795929, -0.008293365128338337, 0.01686502993106842, -0.035594675689935684, 0.010311045683920383, -0.031281013041734695, 0.0017072017071768641, -0.010985924862325191, -0.020580345764756203, -0.0037292311899363995, 0.01477777399122715, 0.008968244306743145, -0.005715603474527597, 0.01726856641471386, -0.006672262214124203, -0.02471311390399933, -0.00021253051818348467, -0.019077522680163383, -0.030585261061787605, 0.027245651930570602, -0.019077522680163383, -0.017950404435396194, -0.02294590324163437, -0.01111116074025631, -0.03718098998069763, 0.014499473385512829, 0.013970701955258846, 0.0210117120295763, -0.0029047648422420025, 0.008536878041923046, 0.014666453935205936, -0.011932147666811943, -0.02784399874508381, -0.01874356158077717, 0.0060982671566307545, -0.004306705202907324, 0.008411642163991928, 0.02634117379784584, -0.028720645233988762, -0.03306213766336441, 0.020274216309189796, -0.009817061945796013, -0.0047867740504443645, 0.029110265895724297, 0.002162049524486065, 0.012238278985023499, -0.030390450730919838, 0.0016863291384652257, -0.02147090993821621, -0.01854874938726425, 0.016308428719639778, -0.011145948432385921, -0.023836465552449226, -0.01909143663942814, 0.004825040698051453, -0.015223055146634579, -0.016670219600200653, 0.010978967882692814, -0.011201607994735241, -0.02433740720152855, 0.004338014405220747, -0.029416397213935852, -0.023432929068803787, 0.02613244764506817, -0.009218715131282806, -0.02353033609688282, -0.005117256660014391, 0.012168703600764275, 0.0048841796815395355, -0.008481217548251152, 0.032644689083099365, 0.0017863434040918946, -0.0072427792474627495, 0.013845466077327728, -0.06250636279582977, 0.016030127182602882, 0.013671528548002243, -0.012836625799536705, 0.0005583410384133458, 0.020329875871539116, 0.005746912211179733, 0.007681102957576513, -0.022639771923422813, 0.022124916315078735, -0.012120001018047333, 0.004338014405220747, -0.032060254365205765, -0.03239421546459198, -0.01338627003133297, -0.01764427311718464, -0.0015750087331980467, -0.003428318304941058, 0.015236970037221909, 0.025826316326856613, -0.01836785487830639, 0.00466501759365201, 0.038934286683797836, 0.0259376373142004, 0.02115086279809475, 0.01247483491897583, -0.03514939546585083, 0.0330343097448349, 0.0023185936734080315, -0.017393801361322403, 0.028581494465470314, -0.0044841221533715725, -0.015056074596941471, 0.0067731463350355625, -0.007458462379872799, -0.004021447151899338, 0.005927807651460171, 0.019161012023687363, 0.028066638857126236, 0.01962020806968212, -0.030752241611480713, -0.016739794984459877, 0.008815178647637367, -0.02567325159907341, -0.006133054383099079, 0.01200868096202612, 0.0034996329341083765, -0.00670009246096015, 0.011069415137171745, 0.014102894812822342, 0.02958337776362896, -0.0022385823540389538, 1.4431419913307764e-05, -0.025227969512343407, -0.010164937935769558, 0.010805029422044754, 0.00839077029377222, -0.0020142022985965014, 0.023864295333623886, 0.010436281561851501, 0.039824847131967545, 0.004929403308779001, 0.007764593232423067, -0.010818944312632084, 0.008975202217698097, 0.0007305396720767021, -0.022041425108909607, -0.012808796018362045, 0.003708358621224761, -0.016670219600200653, 0.0007044489611871541, -0.007416717242449522, 0.0172407366335392, 0.0168093703687191, 0.016725879162549973, -0.01018581073731184, -0.0005083338473923504, 0.0027777901850640774, -0.009865764528512955, 0.030112149193882942, 0.022180575877428055, 0.011173778213560581, -0.014158554375171661, -0.0009323077974840999, 0.016698049381375313, -0.016642389819025993, -0.0019985479302704334, 0.023627741262316704, -0.004630229901522398, 0.0113755464553833, 0.007416717242449522, 0.006393961608409882, 0.008599495515227318, 0.019314076751470566, -0.003562250640243292, 0.02952771820127964, 0.01686502993106842, -0.0037779337726533413, 0.024615708738565445, 0.03178195655345917, 0.008724731393158436, -0.027315227314829826, -0.011354673653841019, -0.012662687338888645, -0.028414513915777206, 0.005715603474527597, -0.05577148497104645, -0.03091922216117382, 0.01109724584966898, 0.02958337776362896, 0.004289311356842518, 0.01871572993695736, 0.00032526408904232085, 0.0035483355168253183, -0.008293365128338337, 0.01721290685236454, 0.011236395686864853, -0.0008192480891011655, -0.020469026640057564, 0.013274949975311756, 0.01854874938726425, 0.0282475333660841, 0.04628142714500427, -0.007479334715753794, 0.038349855691194534, -0.00544773880392313, 0.029054606333374977, -0.020427281036973, 0.03161497414112091, 0.025088820606470108, -0.006748795043677092, 0.02312679961323738, 0.01654498465359211, -0.003315258538350463, -0.0006518327281810343, -0.004571090918034315, -0.01628059893846512, 0.014499473385512829, 0.02717607654631138, 0.10163546353578568, 0.0208586473017931, -0.005058117676526308, -0.0032143746502697468, -0.00030982709722593427, -0.013309736736118793, 0.011292056180536747, 0.010985924862325191, -0.0048528709448874, -0.027649186551570892, 0.004376280587166548, -0.0018141735345125198, 0.012217406183481216, -0.02845625951886177, -0.006414833944290876, -0.013998531736433506, 0.013796763494610786, 0.0038822966162115335, -0.01866007037460804, -0.005117256660014391, 0.03386921063065529, 0.000449629791546613, 0.021512653678655624, 0.018242619931697845, 0.004529345780611038, -0.010631091892719269, 0.021456994116306305, 0.014582963660359383, -0.0033204767387360334, -0.01801997795701027, 0.011953020468354225, -0.0059660738334059715, -0.04355407878756523, -0.031281013041734695, 0.0007214079378172755, -0.028525834903120995, -0.005037244874984026, -0.018757475540041924, 0.0282475333660841, -0.004946797154843807, -0.0014019404770806432, 0.012544410303235054, -0.016378004103899002, -0.013768933713436127, 0.0033622218761593103, -0.021637888625264168, -0.010965052992105484, -0.02045511081814766, -0.024420898407697678], 'text': 'overall GPT-4 training budget. When mixing in data from these math benchmarks, a portion of the\\ntraining data was held back, so each individual training example may or may not have been seen by\\nGPT-4 during training.\\nWe conducted contamination checking to verify the test set for GSM-8K is not included in the training\\nset (see Appendix D). We recommend interpreting the performance results reported for GPT-4\\nGSM-8K in Table 2 as something in-between true few-shot transfer and full benchmark-specific\\ntuning.\\nF Multilingual MMLU\\nWe translated all questions and answers from MMLU [ 49] using Azure Translate. We used an\\nexternal model to perform the translation, instead of relying on GPT-4 itself, in case the model had\\nunrepresentative performance for its own translations. We selected a range of languages that cover\\ndifferent geographic regions and scripts, we show an example question taken from the astronomy', 'SimilarityScore': 0.8271415481962364}), 0.8271415481962364)\n" + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 1: 0.8394796122122777\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"cddfb7ac-d953-46f4-8a48-76655f116bcf\"},\"page_content\":\"GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\",\"type\":\"Document\"}\n", + "Score 2: 0.8286253517208215\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"ba814d15-2c12-40d2-8934-db58b393ecb8\"},\"page_content\":\"model capability results, as well as model safety improvements and results, in more detail in later\\nsections.\\nThis report also discusses a key challenge of the project, developing deep learning infrastructure and\\noptimization methods that behave predictably across a wide range of scales. This allowed us to make\\npredictions about the expected performance of GPT-4 (based on small runs trained in similar ways)\\nthat were tested against the final run to increase confidence in our training.\\nDespite its capabilities, GPT-4 has similar limitations to earlier GPT models [ 1,37,38]: it is not fully\\nreliable (e.g. can suffer from “hallucinations”), has a limited context window, and does not learn\\n∗Please cite this work as “OpenAI (2023)\\\". Full authorship contribution statements appear at the end of the\\ndocument. Correspondence regarding this technical report can be sent to gpt4-report@openai.comarXiv:2303.08774v6 [cs.CL] 4 Mar 2024\",\"type\":\"Document\"}\n", + "Score 3: 0.8215997601854081\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"dd040c08-6ae1-4c73-8b85-7f034d337891\"},\"page_content\":\"these evaluations it performs quite well and often outscores the vast majority of human test takers.\\nFor example, on a simulated bar exam, GPT-4 achieves a score that falls in the top 10% of test takers.\\nThis contrasts with GPT-3.5, which scores in the bottom 10%.\\nOn a suite of traditional NLP benchmarks, GPT-4 outperforms both previous large language models\\nand most state-of-the-art systems (which often have benchmark-specific training or hand-engineering).\\nOn the MMLU benchmark [ 35,36], an English-language suite of multiple-choice questions covering\\n57 subjects, GPT-4 not only outperforms existing models by a considerable margin in English, but\\nalso demonstrates strong performance in other languages. On translated variants of MMLU, GPT-4\\nsurpasses the English-language state-of-the-art in 24 of 26 languages considered. We discuss these\\nmodel capability results, as well as model safety improvements and results, in more detail in later\\nsections.\",\"type\":\"Document\"}\n", + "Score 4: 0.8209972517303962\n", + "\n", + "\n" ] } ], "source": [ "query = \"What were the compute requirements for training GPT 4\"\n", "\n", + "pre_filter = {\n", + " \"conditions\": [\n", + " {\"property\": \"metadata.page\", \"operator\": \"$eq\", \"value\": 0},\n", + " ],\n", + "}\n", + "\n", "results = vector_search.similarity_search_with_score(\n", " query=query,\n", " k=5,\n", + " pre_filter=pre_filter,\n", ")\n", "\n", "# Display results\n", - "for result in results:\n", - " print(result)" + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" ] }, + { + "cell_type": "markdown", + "id": "9fd7b4932ed5f42a", + "metadata": {}, + "source": "## Full Text Search" + }, { "cell_type": "code", - "execution_count": 44, - "id": "90f412e2ade96761", + "execution_count": 42, + "id": "a58a4cbcc160a0b0", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:44:49.510723Z", + "start_time": "2024-12-13T21:44:48.914182Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"cddfb7ac-d953-46f4-8a48-76655f116bcf\"},\"page_content\":\"GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"dd040c08-6ae1-4c73-8b85-7f034d337891\"},\"page_content\":\"these evaluations it performs quite well and often outscores the vast majority of human test takers.\\nFor example, on a simulated bar exam, GPT-4 achieves a score that falls in the top 10% of test takers.\\nThis contrasts with GPT-3.5, which scores in the bottom 10%.\\nOn a suite of traditional NLP benchmarks, GPT-4 outperforms both previous large language models\\nand most state-of-the-art systems (which often have benchmark-specific training or hand-engineering).\\nOn the MMLU benchmark [ 35,36], an English-language suite of multiple-choice questions covering\\n57 subjects, GPT-4 not only outperforms existing models by a considerable margin in English, but\\nalso demonstrates strong performance in other languages. On translated variants of MMLU, GPT-4\\nsurpasses the English-language state-of-the-art in 24 of 26 languages considered. We discuss these\\nmodel capability results, as well as model safety improvements and results, in more detail in later\\nsections.\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"ba814d15-2c12-40d2-8934-db58b393ecb8\"},\"page_content\":\"model capability results, as well as model safety improvements and results, in more detail in later\\nsections.\\nThis report also discusses a key challenge of the project, developing deep learning infrastructure and\\noptimization methods that behave predictably across a wide range of scales. This allowed us to make\\npredictions about the expected performance of GPT-4 (based on small runs trained in similar ways)\\nthat were tested against the final run to increase confidence in our training.\\nDespite its capabilities, GPT-4 has similar limitations to earlier GPT models [ 1,37,38]: it is not fully\\nreliable (e.g. can suffer from “hallucinations”), has a limited context window, and does not learn\\n∗Please cite this work as “OpenAI (2023)\\\". Full authorship contribution statements appear at the end of the\\ndocument. Correspondence regarding this technical report can be sent to gpt4-report@openai.comarXiv:2303.08774v6 [cs.CL] 4 Mar 2024\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"9edf6760-a2d0-4a0b-a652-25fc89de1d34\"},\"page_content\":\"from experience. Care should be taken when using the outputs of GPT-4, particularly in contexts\\nwhere reliability is important.\\nGPT-4’s capabilities and limitations create significant and novel safety challenges, and we believe\\ncareful study of these challenges is an important area of research given the potential societal impact.\\nThis report includes an extensive system card (after the Appendix) describing some of the risks we\\nforesee around bias, disinformation, over-reliance, privacy, cybersecurity, proliferation, and more.\\nIt also describes interventions we made to mitigate potential harms from the deployment of GPT-4,\\nincluding adversarial testing with domain experts, and a model-assisted safety pipeline.\\n2 Scope and Limitations of this Technical Report\\nThis report focuses on the capabilities, limitations, and safety properties of GPT-4. GPT-4 is a\\nTransformer-style model [ 39] pre-trained to predict the next token in a document, using both publicly\",\"type\":\"Document\"}\n", + "\n", + "\n" + ] + } + ], + "source": [ + "from langchain_community.vectorstores.azure_cosmos_db_no_sql import CosmosDBQueryType\n", + "\n", + "query = \"What were the compute requirements for training GPT 4\"\n", + "pre_filter = {\n", + " \"conditions\": [\n", + " {\n", + " \"property\": \"text\",\n", + " \"operator\": \"$full_text_contains_any\",\n", + " \"value\": \"What were the compute requirements for training GPT 4\",\n", + " },\n", + " ],\n", + "}\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.FULL_TEXT_SEARCH,\n", + " pre_filter=pre_filter,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "2831548f1fb4eb90", + "metadata": {}, + "source": "## Full Text Search BM 25 Ranking" + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "73dacc61ce8bf7a1", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:47:25.253222Z", + "start_time": "2024-12-13T21:47:23.680741Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"f2746fd3-bbcb-4197-b2d5-ee7b355b6009\"},\"page_content\":\"the HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\\nline; this fit accurately predicts GPT-4’s performance. The x-axis is training compute normalized so that\\nGPT-4 is 1.\\n3\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"20153a6c-7c2c-4328-9b0e-e3502d7ac4dd\"},\"page_content\":\"safety considerations above against the scientific value of further transparency.\\n3 Predictable Scaling\\nA large focus of the GPT-4 project was building a deep learning stack that scales predictably. The\\nprimary reason is that for very large training runs like GPT-4, it is not feasible to do extensive\\nmodel-specific tuning. To address this, we developed infrastructure and optimization methods that\\nhave very predictable behavior across multiple scales. These improvements allowed us to reliably\\npredict some aspects of the performance of GPT-4 from smaller models trained using 1,000×–\\n10,000×less compute.\\n3.1 Loss Prediction\\nThe final loss of properly-trained large language models is thought to be well approximated by power\\nlaws in the amount of compute used to train the model [41, 42, 2, 14, 15].\\nTo verify the scalability of our optimization infrastructure, we predicted GPT-4’s final loss on our\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"6d88f369-4147-4530-9bfb-0ed008413211\"},\"page_content\":\"Observed\\nPrediction\\ngpt-4\\n100p 10n 1µ 100µ 0.01 1\\nCompute1.02.03.04.05.06.0Bits per wordOpenAI codebase next word predictionFigure 1. Performance of GPT-4 and smaller models. The metric is final loss on a dataset derived\\nfrom our internal codebase. This is a convenient, large dataset of code tokens which is not contained in\\nthe training set. We chose to look at loss because it tends to be less noisy than other measures across\\ndifferent amounts of training compute. A power law fit to the smaller models (excluding GPT-4) is\\nshown as the dotted line; this fit accurately predicts GPT-4’s final loss. The x-axis is training compute\\nnormalized so that GPT-4 is 1.\\nObserved\\nPrediction\\ngpt-4\\n1µ 10µ 100µ 0.001 0.01 0.1 1\\nCompute012345– Mean Log Pass RateCapability prediction on 23 coding problems\\nFigure 2. Performance of GPT-4 and smaller models. The metric is mean log pass rate on a subset of\\nthe HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"90e4bafe-55bb-406b-afba-a0143c810842\"},\"page_content\":\"which measures the ability to synthesize Python functions of varying complexity. We successfully\\npredicted the pass rate on a subset of the HumanEval dataset by extrapolating from models trained\\nwith at most 1,000×less compute (Figure 2).\\nFor an individual problem in HumanEval, performance may occasionally worsen with scale. Despite\\nthese challenges, we find an approximate power law relationship −EP[log(pass _rate(C))] = α∗C−k\\n2In addition to the accompanying system card, OpenAI will soon publish additional thoughts on the social\\nand economic implications of AI systems, including the need for effective regulation.\\n2\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":71,\"id\":\"10ff5a7f-6638-4446-85b2-6e4314eca938\"},\"page_content\":\"Unsupervised Multitask Learners,” 2019.\\n[23]G. C. Bowker and S. L. Star, Sorting Things Out . MIT Press, Aug. 2000.\\n[24]L. Weidinger, J. Uesato, M. Rauh, C. Griffin, P.-S. Huang, J. Mellor, A. Glaese, M. Cheng,\\nB. Balle, A. Kasirzadeh, C. Biles, S. Brown, Z. Kenton, W. Hawkins, T. Stepleton, A. Birhane,\\nL. A. Hendricks, L. Rimell, W. Isaac, J. Haas, S. Legassick, G. Irving, and I. Gabriel, “Taxonomy\\nof Risks posed by Language Models,” in 2022 ACM Conference on Fairness, Accountability,\\nand Transparency , FAccT ’22, (New York, NY, USA), pp. 214–229, Association for Computing\\nMachinery, June 2022.\\n72\",\"type\":\"Document\"}\n", + "\n", + "\n" + ] + } + ], + "source": [ + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.FULL_TEXT_RANK,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "920e4eb20a141031", + "metadata": {}, + "source": "## Hybrid Search" + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "5e1f7c17b02579b7", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:48:41.281808Z", + "start_time": "2024-12-13T21:48:32.232591Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":97,\"id\":\"36dfcd6c-d3cf-4e34-a5d6-cc4d63013cba\"},\"page_content\":\"Figure 11: Results on IF evaluations across GPT3.5, GPT3.5-Turbo, GPT-4-launch\\n98\",\"type\":\"Document\"}\n", + "Score 1: 0.8173275975778744\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":7,\"id\":\"3d6e4715-4a38-40b1-89f1-e768bad5f9c8\"},\"page_content\":\"Preliminary results on a narrow set of academic vision benchmarks can be found in the GPT-4 blog\\npost [ 65]. We plan to release more information about GPT-4’s visual capabilities in follow-up work.\\n8\",\"type\":\"Document\"}\n", + "Score 2: 0.8176419674549597\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"f2746fd3-bbcb-4197-b2d5-ee7b355b6009\"},\"page_content\":\"the HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\\nline; this fit accurately predicts GPT-4’s performance. The x-axis is training compute normalized so that\\nGPT-4 is 1.\\n3\",\"type\":\"Document\"}\n", + "Score 3: 0.8053881702559759\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 4: 0.8394796122122777\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"20153a6c-7c2c-4328-9b0e-e3502d7ac4dd\"},\"page_content\":\"safety considerations above against the scientific value of further transparency.\\n3 Predictable Scaling\\nA large focus of the GPT-4 project was building a deep learning stack that scales predictably. The\\nprimary reason is that for very large training runs like GPT-4, it is not feasible to do extensive\\nmodel-specific tuning. To address this, we developed infrastructure and optimization methods that\\nhave very predictable behavior across multiple scales. These improvements allowed us to reliably\\npredict some aspects of the performance of GPT-4 from smaller models trained using 1,000×–\\n10,000×less compute.\\n3.1 Loss Prediction\\nThe final loss of properly-trained large language models is thought to be well approximated by power\\nlaws in the amount of compute used to train the model [41, 42, 2, 14, 15].\\nTo verify the scalability of our optimization infrastructure, we predicted GPT-4’s final loss on our\",\"type\":\"Document\"}\n", + "Score 5: 0.8213247840132897\n", + "\n", + "\n" + ] + } + ], + "source": [ + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.HYBRID,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "1178119f59653565", + "metadata": {}, + "source": "## Hybrid Search with filtering" + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "a644e56095c897fd", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:52:54.152486Z", - "start_time": "2024-05-25T01:52:54.140651Z" + "end_time": "2024-12-13T21:56:00.251680Z", + "start_time": "2024-12-13T21:55:55.253274Z" } }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":97,\"id\":\"36dfcd6c-d3cf-4e34-a5d6-cc4d63013cba\"},\"page_content\":\"Figure 11: Results on IF evaluations across GPT3.5, GPT3.5-Turbo, GPT-4-launch\\n98\",\"type\":\"Document\"}\n", + "Score 1: 0.8173275975778744\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":7,\"id\":\"3d6e4715-4a38-40b1-89f1-e768bad5f9c8\"},\"page_content\":\"Preliminary results on a narrow set of academic vision benchmarks can be found in the GPT-4 blog\\npost [ 65]. We plan to release more information about GPT-4’s visual capabilities in follow-up work.\\n8\",\"type\":\"Document\"}\n", + "Score 2: 0.8176419674549597\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"f2746fd3-bbcb-4197-b2d5-ee7b355b6009\"},\"page_content\":\"the HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\\nline; this fit accurately predicts GPT-4’s performance. The x-axis is training compute normalized so that\\nGPT-4 is 1.\\n3\",\"type\":\"Document\"}\n", + "Score 3: 0.8053881702559759\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 4: 0.8394796122122777\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"20153a6c-7c2c-4328-9b0e-e3502d7ac4dd\"},\"page_content\":\"safety considerations above against the scientific value of further transparency.\\n3 Predictable Scaling\\nA large focus of the GPT-4 project was building a deep learning stack that scales predictably. The\\nprimary reason is that for very large training runs like GPT-4, it is not feasible to do extensive\\nmodel-specific tuning. To address this, we developed infrastructure and optimization methods that\\nhave very predictable behavior across multiple scales. These improvements allowed us to reliably\\npredict some aspects of the performance of GPT-4 from smaller models trained using 1,000×–\\n10,000×less compute.\\n3.1 Loss Prediction\\nThe final loss of properly-trained large language models is thought to be well approximated by power\\nlaws in the amount of compute used to train the model [41, 42, 2, 14, 15].\\nTo verify the scalability of our optimization infrastructure, we predicted GPT-4’s final loss on our\",\"type\":\"Document\"}\n", + "Score 5: 0.8213247840132897\n", + "\n", + "\n" + ] + } + ], + "source": [ + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "pre_filter = {\n", + " \"conditions\": [\n", + " {\n", + " \"property\": \"text\",\n", + " \"operator\": \"$full_text_contains_any\",\n", + " \"value\": \"compute requirements\",\n", + " },\n", + " {\"property\": \"metadata.page\", \"operator\": \"$eq\", \"value\": 0},\n", + " ],\n", + " \"logical_operator\": \"$and\",\n", + "}\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.HYBRID,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de996623625ab622", + "metadata": {}, "outputs": [], "source": [] } diff --git a/libs/community/langchain_community/vectorstores/azure_cosmos_db.py b/libs/community/langchain_community/vectorstores/azure_cosmos_db.py index 446e628c68f78..b96ec2055cc98 100644 --- a/libs/community/langchain_community/vectorstores/azure_cosmos_db.py +++ b/libs/community/langchain_community/vectorstores/azure_cosmos_db.py @@ -131,6 +131,7 @@ def from_connection_string( connection_string: The MongoDB vCore instance connection string namespace: The namespace (database.collection) embedding: The embedding utility + application_name: The user agent for telemetry **kwargs: Dynamic keyword arguments Returns: diff --git a/libs/community/langchain_community/vectorstores/azure_cosmos_db_no_sql.py b/libs/community/langchain_community/vectorstores/azure_cosmos_db_no_sql.py index 2317af9da0250..48110a182a982 100644 --- a/libs/community/langchain_community/vectorstores/azure_cosmos_db_no_sql.py +++ b/libs/community/langchain_community/vectorstores/azure_cosmos_db_no_sql.py @@ -2,17 +2,42 @@ import uuid import warnings +from enum import Enum from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple import numpy as np from langchain_core.documents import Document from langchain_core.embeddings import Embeddings from langchain_core.vectorstores import VectorStore +from pydantic import BaseModel, Field from langchain_community.vectorstores.utils import maximal_marginal_relevance if TYPE_CHECKING: - from azure.cosmos.cosmos_client import CosmosClient + from azure.cosmos import CosmosClient + from azure.identity import DefaultAzureCredential + +USER_AGENT = ("LangChain-CDBNoSql-VectorStore-Python",) + + +class Condition(BaseModel): + property: str + operator: str + value: Any + + +class PreFilter(BaseModel): + conditions: List[Condition] = Field(default_factory=list) + logical_operator: Optional[str] = None + + +class CosmosDBQueryType(str, Enum): + """CosmosDB Query Type""" + + VECTOR = "vector" + FULL_TEXT_SEARCH = "full_text_search" + FULL_TEXT_RANK = "full_text_rank" + HYBRID = "hybrid" class AzureCosmosDBNoSqlVectorSearch(VectorStore): @@ -21,8 +46,11 @@ class AzureCosmosDBNoSqlVectorSearch(VectorStore): To use, you should have both: - the ``azure-cosmos`` python package installed - You can read more about vector search using AzureCosmosDBNoSQL here: + You can read more about vector search, full text search + and hybrid search using AzureCosmosDBNoSQL here: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/vector-search + https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search + https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/hybrid-search """ def __init__( @@ -34,9 +62,14 @@ def __init__( indexing_policy: Dict[str, Any], cosmos_container_properties: Dict[str, Any], cosmos_database_properties: Dict[str, Any], + full_text_policy: Optional[Dict[str, Any]] = None, database_name: str = "vectorSearchDB", container_name: str = "vectorSearchContainer", + text_key: str = "text", + embedding_key: str = "embedding", + metadata_key: str = "metadata", create_container: bool = True, + full_text_search_enabled: bool = False, ): """ Constructor for AzureCosmosDBNoSqlVectorSearch @@ -47,30 +80,42 @@ def __init__( container_name: Name of the container to be created. embedding: Text embedding model to use. vector_embedding_policy: Vector Embedding Policy for the container. + full_text_policy: Full Text Policy for the container. indexing_policy: Indexing Policy for the container. cosmos_container_properties: Container Properties for the container. cosmos_database_properties: Database Properties for the container. + text_key: Text key to use for text property which will be + embedded in the data schema. + embedding_key: Embedding key to use for vector embedding. + metadata_key: Metadata key to use for data schema. + create_container: Set to true if the container does not exist. + full_text_search_enabled: Set to true if the full text search is enabled. """ self._cosmos_client = cosmos_client self._database_name = database_name self._container_name = container_name self._embedding = embedding self._vector_embedding_policy = vector_embedding_policy + self._full_text_policy = full_text_policy self._indexing_policy = indexing_policy self._cosmos_container_properties = cosmos_container_properties self._cosmos_database_properties = cosmos_database_properties + self._text_key = text_key + self._embedding_key = embedding_key + self._metadata_key = metadata_key self._create_container = create_container + self._full_text_search_enabled = full_text_search_enabled if self._create_container: if ( - indexing_policy["vectorIndexes"] is None - or len(indexing_policy["vectorIndexes"]) == 0 + self._indexing_policy["vectorIndexes"] is None + or len(self._indexing_policy["vectorIndexes"]) == 0 ): raise ValueError( "vectorIndexes cannot be null or empty in the indexing_policy." ) if ( - vector_embedding_policy is None + self._vector_embedding_policy is None or len(vector_embedding_policy["vectorEmbeddings"]) == 0 ): raise ValueError( @@ -81,6 +126,23 @@ def __init__( raise ValueError( "partition_key cannot be null or empty for a container." ) + if self._full_text_search_enabled: + if ( + self._indexing_policy["fullTextIndexes"] is None + or len(self._indexing_policy["fullTextIndexes"]) == 0 + ): + raise ValueError( + "fullTextIndexes cannot be null or empty in the " + "indexing_policy if full text search is enabled." + ) + if ( + self._full_text_policy is None + or len(self._full_text_policy["fullTextPaths"]) == 0 + ): + raise ValueError( + "fullTextPaths cannot be null or empty in the " + "full_text_policy if full text search is enabled." + ) # Create the database if it already doesn't exist self._database = self._cosmos_client.create_database_if_not_exists( @@ -116,12 +178,9 @@ def __init__( session_token=self._cosmos_container_properties.get("session_token"), initial_headers=self._cosmos_container_properties.get("initial_headers"), vector_embedding_policy=self._vector_embedding_policy, + full_text_policy=self._full_text_policy, ) - self._embedding_key = self._vector_embedding_policy["vectorEmbeddings"][0][ - "path" - ][1:] - def add_texts( self, texts: Iterable[str], @@ -187,9 +246,14 @@ def _from_kwargs( indexing_policy: Dict[str, Any], cosmos_container_properties: Dict[str, Any], cosmos_database_properties: Dict[str, Any], + full_text_policy: Optional[Dict[str, Any]] = None, database_name: str = "vectorSearchDB", container_name: str = "vectorSearchContainer", + text_key: str = "text", + embedding_key: str = "embedding", + metadata_key: str = "metadata", create_container: bool = True, + full_text_search_enabled: bool = False, **kwargs: Any, ) -> AzureCosmosDBNoSqlVectorSearch: if kwargs: @@ -205,12 +269,17 @@ def _from_kwargs( embedding=embedding, cosmos_client=cosmos_client, vector_embedding_policy=vector_embedding_policy, + full_text_policy=full_text_policy, indexing_policy=indexing_policy, cosmos_container_properties=cosmos_container_properties, cosmos_database_properties=cosmos_database_properties, database_name=database_name, container_name=container_name, + text_key=text_key, + embedding_key=embedding_key, + metadata_key=metadata_key, create_container=create_container, + full_text_search_enabled=full_text_search_enabled, ) @classmethod @@ -242,6 +311,46 @@ def from_texts( ) return vectorstore + @classmethod + def from_connection_string_and_aad( + cls, + connection_string: str, + defaultAzureCredential: DefaultAzureCredential, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + **kwargs: Any, + ) -> AzureCosmosDBNoSqlVectorSearch: + cosmos_client = CosmosClient( + connection_string, defaultAzureCredential, user_agent=USER_AGENT + ) + kwargs["cosmos_client"] = cosmos_client + vectorstore = cls._from_kwargs(embedding, **kwargs) + vectorstore.add_texts( + texts=texts, + metadatas=metadatas, + ) + return vectorstore + + @classmethod + def from_connection_string_and_key( + cls, + connection_string: str, + key: str, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + **kwargs: Any, + ) -> AzureCosmosDBNoSqlVectorSearch: + cosmos_client = CosmosClient(connection_string, key, user_agent=USER_AGENT) + kwargs["cosmos_client"] = cosmos_client + vectorstore = cls._from_kwargs(embedding, **kwargs) + vectorstore.add_texts( + texts=texts, + metadatas=metadatas, + ) + return vectorstore + def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]: if ids is None: raise ValueError("No document ids provided to delete.") @@ -262,85 +371,169 @@ def delete_document_by_id(self, document_id: Optional[str] = None) -> None: def _similarity_search_with_score( self, + query_type: CosmosDBQueryType, embeddings: List[float], k: int = 4, - pre_filter: Optional[Dict] = None, + pre_filter: Optional[PreFilter] = None, with_embedding: bool = False, + offset_limit: Optional[str] = None, + *, + projection_mapping: Optional[Dict[str, Any]] = None, + **kwargs: Any, ) -> List[Tuple[Document, float]]: - query = "SELECT " - - # If limit_offset_clause is not specified, add TOP clause - if pre_filter is None or pre_filter.get("limit_offset_clause") is None: - query += "TOP @limit " - - query += ( - "c.id, c[@embeddingKey], c.text, c.metadata, " - "VectorDistance(c[@embeddingKey], @embeddings) AS SimilarityScore FROM c" + query, parameters = self._construct_query( + k=k, + query_type=query_type, + embeddings=embeddings, + pre_filter=pre_filter, + offset_limit=offset_limit, + projection_mapping=projection_mapping, ) - # Add where_clause if specified - if pre_filter is not None and pre_filter.get("where_clause") is not None: - query += " {}".format(pre_filter["where_clause"]) - - query += " ORDER BY VectorDistance(c[@embeddingKey], @embeddings)" - - # Add limit_offset_clause if specified - if pre_filter is not None and pre_filter.get("limit_offset_clause") is not None: - query += " {}".format(pre_filter["limit_offset_clause"]) - parameters = [ - {"name": "@limit", "value": k}, - {"name": "@embeddingKey", "value": self._embedding_key}, - {"name": "@embeddings", "value": embeddings}, - ] + return self._execute_query( + query=query, + query_type=query_type, + parameters=parameters, + with_embedding=with_embedding, + projection_mapping=projection_mapping, + ) - docs_and_scores = [] + def _full_text_search( + self, + query_type: CosmosDBQueryType, + search_text: Optional[str] = None, + k: int = 4, + pre_filter: Optional[PreFilter] = None, + offset_limit: Optional[str] = None, + *, + projection_mapping: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> List[Tuple[Document, float]]: + query, parameters = self._construct_query( + k=k, + query_type=query_type, + search_text=search_text, + pre_filter=pre_filter, + offset_limit=offset_limit, + projection_mapping=projection_mapping, + ) - items = list( - self._container.query_items( - query=query, parameters=parameters, enable_cross_partition_query=True - ) + return self._execute_query( + query=query, + query_type=query_type, + parameters=parameters, + with_embedding=False, + projection_mapping=projection_mapping, ) - for item in items: - text = item["text"] - metadata = item["metadata"] - score = item["SimilarityScore"] - if with_embedding: - metadata[self._embedding_key] = item[self._embedding_key] - docs_and_scores.append( - (Document(page_content=text, metadata=metadata), score) - ) - return docs_and_scores - def similarity_search_with_score( + def _hybrid_search_with_score( self, - query: str, + query_type: CosmosDBQueryType, + embeddings: List[float], + search_text: str, k: int = 4, - pre_filter: Optional[Dict] = None, + pre_filter: Optional[PreFilter] = None, with_embedding: bool = False, + offset_limit: Optional[str] = None, + *, + projection_mapping: Optional[Dict[str, Any]] = None, + **kwargs: Any, ) -> List[Tuple[Document, float]]: - embeddings = self._embedding.embed_query(query) - docs_and_scores = self._similarity_search_with_score( - embeddings=embeddings, + query, parameters = self._construct_query( k=k, + query_type=query_type, + embeddings=embeddings, + search_text=search_text, pre_filter=pre_filter, + offset_limit=offset_limit, + projection_mapping=projection_mapping, + ) + return self._execute_query( + query=query, + query_type=query_type, + parameters=parameters, with_embedding=with_embedding, + projection_mapping=projection_mapping, ) + + def similarity_search_with_score( + self, + query: str, + k: int = 4, + pre_filter: Optional[PreFilter] = None, + with_embedding: bool = False, + query_type: CosmosDBQueryType = CosmosDBQueryType.VECTOR, + offset_limit: Optional[str] = None, + **kwargs: Any, + ) -> List[Tuple[Document, float]]: + embeddings = self._embedding.embed_query(query) + docs_and_scores = [] + if query_type == CosmosDBQueryType.VECTOR: + docs_and_scores = self._similarity_search_with_score( + query_type=query_type, + embeddings=embeddings, + k=k, + pre_filter=pre_filter, + with_embedding=with_embedding, + offset_limit=offset_limit, + **kwargs, + ) + elif query_type == CosmosDBQueryType.FULL_TEXT_SEARCH: + docs_and_scores = self._full_text_search( + k=k, + query_type=query_type, + pre_filter=pre_filter, + offset_limit=offset_limit, + **kwargs, + ) + + elif query_type == CosmosDBQueryType.FULL_TEXT_RANK: + docs_and_scores = self._full_text_search( + search_text=query, + k=k, + query_type=query_type, + pre_filter=pre_filter, + offset_limit=offset_limit, + **kwargs, + ) + elif query_type == CosmosDBQueryType.HYBRID: + docs_and_scores = self._hybrid_search_with_score( + query_type=query_type, + embeddings=embeddings, + search_text=query, + k=k, + pre_filter=pre_filter, + with_embedding=with_embedding, + offset_limit=offset_limit, + **kwargs, + ) return docs_and_scores def similarity_search( self, query: str, k: int = 4, - pre_filter: Optional[Dict] = None, + pre_filter: Optional[PreFilter] = None, with_embedding: bool = False, + query_type: CosmosDBQueryType = CosmosDBQueryType.VECTOR, + offset_limit: Optional[str] = None, **kwargs: Any, ) -> List[Document]: - docs_and_scores = self.similarity_search_with_score( - query, - k=k, - pre_filter=pre_filter, - with_embedding=with_embedding, - ) + if query_type not in CosmosDBQueryType.__members__.values(): + raise ValueError( + f"Invalid query_type: {query_type}. " + f"Expected one of: {', '.join(t.value for t in CosmosDBQueryType)}." + ) + else: + docs_and_scores = self.similarity_search_with_score( + query, + k=k, + pre_filter=pre_filter, + with_embedding=with_embedding, + query_type=query_type, + offset_limit=offset_limit, + kwargs=kwargs, + ) return [doc for doc, _ in docs_and_scores] @@ -350,18 +543,20 @@ def max_marginal_relevance_search_by_vector( k: int = 4, fetch_k: int = 20, lambda_mult: float = 0.5, + query_type: CosmosDBQueryType = CosmosDBQueryType.VECTOR, + pre_filter: Optional[PreFilter] = None, + with_embedding: bool = False, **kwargs: Any, ) -> List[Document]: # Retrieves the docs with similarity scores - pre_filter = {} - with_embedding = False - if kwargs["pre_filter"]: - pre_filter = kwargs["pre_filter"] - if kwargs["with_embedding"]: - with_embedding = kwargs["with_embedding"] + # if kwargs["pre_filter"]: + # pre_filter = kwargs["pre_filter"] + # if kwargs["with_embedding"]: + # with_embedding = kwargs["with_embedding"] docs = self._similarity_search_with_score( embeddings=embedding, k=fetch_k, + query_type=query_type, pre_filter=pre_filter, with_embedding=with_embedding, ) @@ -383,15 +578,16 @@ def max_marginal_relevance_search( k: int = 4, fetch_k: int = 20, lambda_mult: float = 0.5, + query_type: CosmosDBQueryType = CosmosDBQueryType.VECTOR, + pre_filter: Optional[PreFilter] = None, + with_embedding: bool = False, **kwargs: Any, ) -> List[Document]: # compute the embeddings vector from the query string - pre_filter = {} - with_embedding = False - if kwargs["pre_filter"]: - pre_filter = kwargs["pre_filter"] - if kwargs["with_embedding"]: - with_embedding = kwargs["with_embedding"] + # if kwargs["pre_filter"]: + # pre_filter = kwargs["pre_filter"] + # if kwargs["with_embedding"]: + # with_embedding = kwargs["with_embedding"] embeddings = self._embedding.embed_query(query) docs = self.max_marginal_relevance_search_by_vector( @@ -400,6 +596,266 @@ def max_marginal_relevance_search( fetch_k=fetch_k, lambda_mult=lambda_mult, pre_filter=pre_filter, + query_type=query_type, with_embedding=with_embedding, ) return docs + + def _construct_query( + self, + k: int, + query_type: CosmosDBQueryType, + embeddings: Optional[List[float]] = None, + search_text: Optional[str] = None, + pre_filter: Optional[PreFilter] = None, + offset_limit: Optional[str] = None, + projection_mapping: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, List[Dict[str, Any]]]: + if ( + query_type == CosmosDBQueryType.FULL_TEXT_RANK + or query_type == CosmosDBQueryType.HYBRID + ): + query = f"SELECT {'TOP ' + str(k) + ' ' if not offset_limit else ''}" + else: + query = f"""SELECT {'TOP @limit ' if not offset_limit else ''}""" + query += self._generate_projection_fields( + projection_mapping, query_type, embeddings + ) + query += " FROM c " + + # Add where_clause if specified + if pre_filter: + where_clause = self._build_where_clause(pre_filter) + query += f"""{where_clause}""" + + # TODO: Update the code to use parameters once parametrized queries + # are allowed for these query functions + if query_type == CosmosDBQueryType.FULL_TEXT_RANK: + if search_text is None: + raise ValueError( + "search text cannot be None for FULL_TEXT_RANK queries." + ) + query += f""" ORDER BY RANK FullTextScore(c.{self._text_key}, + [{", ".join(f"'{term}'" for term in search_text.split())}])""" + elif query_type == CosmosDBQueryType.VECTOR: + query += " ORDER BY VectorDistance(c[@embeddingKey], @embeddings)" + elif query_type == CosmosDBQueryType.HYBRID: + if search_text is None: + raise ValueError("search text cannot be None for HYBRID queries.") + query += f""" ORDER BY RANK RRF(FullTextScore(c.{self._text_key}, + [{", ".join(f"'{term}'" for term in search_text.split())}]), + VectorDistance(c.{self._embedding_key}, {embeddings}))""" + else: + query += "" + + # Add limit_offset_clause if specified + if offset_limit is not None: + query += f""" {offset_limit}""" + + # TODO: Remove this if check once parametrized queries + # are allowed for these query functions + parameters = [] + if ( + query_type == CosmosDBQueryType.FULL_TEXT_SEARCH + or query_type == CosmosDBQueryType.VECTOR + ): + parameters = self._build_parameters( + k=k, + query_type=query_type, + embeddings=embeddings, + projection_mapping=projection_mapping, + ) + return query, parameters + + def _generate_projection_fields( + self, + projection_mapping: Optional[Dict[str, Any]], + query_type: CosmosDBQueryType, + embeddings: Optional[List[float]] = None, + ) -> str: + # TODO: Remove this if check once parametrized queries + # are allowed for these query functions + if ( + query_type == CosmosDBQueryType.FULL_TEXT_RANK + or query_type == CosmosDBQueryType.HYBRID + ): + if projection_mapping: + projection = ", ".join( + f"c.{key} as {alias}" for key, alias in projection_mapping.items() + ) + else: + projection = ( + f"c.id, c.{self._text_key} as text, " + f"c.{self._metadata_key} as metadata" + ) + if query_type == CosmosDBQueryType.HYBRID: + projection += ( + f", c.{self._embedding_key} as embedding, " + f"VectorDistance(c.{self._embedding_key}, " + f"{embeddings}) as SimilarityScore" + ) + else: + if projection_mapping: + projection = ", ".join( + f"c.[@{key}] as {alias}" + for key, alias in projection_mapping.items() + ) + else: + projection = "c.id, c[@textKey] as text, c[@metadataKey] as metadata" + + if ( + query_type == CosmosDBQueryType.VECTOR + or query_type == CosmosDBQueryType.HYBRID + ): + projection += ( + ", c[@embeddingKey] as embedding, " + "VectorDistance(c[@embeddingKey], " + "@embeddings) as SimilarityScore" + ) + return projection + + def _build_parameters( + self, + k: int, + query_type: CosmosDBQueryType, + embeddings: Optional[List[float]], + search_terms: Optional[List[str]] = None, + projection_mapping: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + parameters: List[Dict[str, Any]] = [ + {"name": "@limit", "value": k}, + {"name": "@textKey", "value": self._text_key}, + ] + + if projection_mapping: + for key in projection_mapping.keys(): + parameters.append({"name": f"@{key}", "value": key}) + else: + parameters.append({"name": "@metadataKey", "value": self._metadata_key}) + + if ( + query_type == CosmosDBQueryType.FULL_TEXT_RANK + or query_type == CosmosDBQueryType.HYBRID + ): + parameters.append({"name": "@searchTerms", "value": search_terms}) + elif ( + query_type == CosmosDBQueryType.VECTOR + or query_type == CosmosDBQueryType.HYBRID + ): + parameters.append({"name": "@embeddingKey", "value": self._embedding_key}) + parameters.append({"name": "@embeddings", "value": embeddings}) + + return parameters + + def _build_where_clause(self, pre_filter: PreFilter) -> str: + """ + Builds a where clause based on the given pre_filter. + """ + + operator_map = self._where_clause_operator_map() + + if ( + pre_filter.logical_operator + and pre_filter.logical_operator not in operator_map + ): + raise ValueError( + f"unsupported logical_operator: {pre_filter.logical_operator}" + ) + + sql_logical_operator = operator_map.get(pre_filter.logical_operator or "", "") + clauses = [] + + for condition in pre_filter.conditions: + if condition.operator not in operator_map: + raise ValueError(f"Unsupported operator: {condition.operator}") + + if "full_text" in condition.operator: + if not isinstance(condition.value, str): + raise ValueError( + f"Expected a string for {condition.operator}, " + f"got {type(condition.value)}" + ) + search_terms = ", ".join( + f"'{term}'" for term in condition.value.split() + ) + sql_function = operator_map[condition.operator] + clauses.append( + f"{sql_function}(c.{condition.property}, {search_terms})" + ) + else: + sql_operator = operator_map[condition.operator] + if isinstance(condition.value, str): + value = f"'{condition.value}'" + elif isinstance(condition.value, list): + # e.g., for IN clauses + value = f"({', '.join(map(str, condition.value))})" + clauses.append(f"c.{condition.property} {sql_operator} {value}") + return f""" WHERE {' {} '.format(sql_logical_operator).join(clauses)}""".strip() + + def _execute_query( + self, + query: str, + query_type: CosmosDBQueryType, + parameters: List[Dict[str, Any]], + with_embedding: bool, + projection_mapping: Optional[Dict[str, Any]], + ) -> List[Tuple[Document, float]]: + docs_and_scores = [] + items = list( + self._container.query_items( + query=query, parameters=parameters, enable_cross_partition_query=True + ) + ) + for item in items: + text = item[self._text_key] + metadata = item.pop(self._metadata_key, {}) + score = 0.0 + + if projection_mapping: + for key, alias in projection_mapping.items(): + if key == self._text_key: + continue + metadata[alias] = item[alias] + else: + metadata["id"] = item["id"] + + if ( + query_type == CosmosDBQueryType.VECTOR + or query_type == CosmosDBQueryType.HYBRID + ): + score = item["SimilarityScore"] + if with_embedding: + metadata[self._embedding_key] = item[self._embedding_key] + docs_and_scores.append( + (Document(page_content=text, metadata=metadata), score) + ) + return docs_and_scores + + def _where_clause_operator_map(self) -> Dict[str, str]: + operator_map = { + "$eq": "=", + "$ne": "!=", + "$lt": "<", + "$lte": "<=", + "$gt": ">", + "$gte": ">=", + "$add": "+", + "$sub": "-", + "$mul": "*", + "$div": "/", + "$mod": "%", + "$or": "OR", + "$and": "AND", + "$not": "NOT", + "$concat": "||", + "$bit_or": "|", + "$bit_and": "&", + "$bit_xor": "^", + "$bit_lshift": "<<", + "$bit_rshift": ">>", + "$bit_zerofill_rshift": ">>>", + "$full_text_contains": "FullTextContains", + "$full_text_contains_all": "FullTextContainsAll", + "$full_text_contains_any": "FullTextContainsAny", + } + return operator_map diff --git a/libs/community/tests/integration_tests/vectorstores/test_azure_cosmos_db_no_sql.py b/libs/community/tests/integration_tests/vectorstores/test_azure_cosmos_db_no_sql.py index c8a8f87a599a4..63c0d0a17efa1 100644 --- a/libs/community/tests/integration_tests/vectorstores/test_azure_cosmos_db_no_sql.py +++ b/libs/community/tests/integration_tests/vectorstores/test_azure_cosmos_db_no_sql.py @@ -3,7 +3,7 @@ import logging import os from time import sleep -from typing import Any +from typing import Any, Dict, List, Tuple import pytest from langchain_core.documents import Document @@ -11,6 +11,9 @@ from langchain_community.embeddings import OpenAIEmbeddings from langchain_community.vectorstores.azure_cosmos_db_no_sql import ( AzureCosmosDBNoSqlVectorSearch, + Condition, + CosmosDBQueryType, + PreFilter, ) logging.basicConfig(level=logging.DEBUG) @@ -60,6 +63,7 @@ def get_vector_indexing_policy(embedding_type: str) -> dict: "includedPaths": [{"path": "/*"}], "excludedPaths": [{"path": '/"_etag"/?'}], "vectorIndexes": [{"path": "/embedding", "type": embedding_type}], + "fullTextIndexes": [{"path": "/text"}], } @@ -78,6 +82,13 @@ def get_vector_embedding_policy( } +def get_full_text_policy() -> dict: + return { + "defaultLanguage": "en-US", + "fullTextPaths": [{"path": "/text", "language": "en-US"}], + } + + class TestAzureCosmosDBNoSqlVectorSearch: def test_from_documents_cosine_distance( self, @@ -86,12 +97,7 @@ def test_from_documents_cosine_distance( azure_openai_embeddings: OpenAIEmbeddings, ) -> None: """Test end to end construction and search.""" - documents = [ - Document(page_content="Dogs are tough.", metadata={"a": 1}), - Document(page_content="Cats have fluff.", metadata={"b": 1}), - Document(page_content="What is a sandwich?", metadata={"c": 1}), - Document(page_content="That fence is purple.", metadata={"d": 1, "e": 2}), - ] + documents = self._get_documents() store = AzureCosmosDBNoSqlVectorSearch.from_documents( documents, @@ -105,13 +111,16 @@ def test_from_documents_cosine_distance( indexing_policy=get_vector_indexing_policy("flat"), cosmos_container_properties={"partition_key": partition_key}, cosmos_database_properties={}, + full_text_policy=get_full_text_policy(), + full_text_search_enabled=True, ) sleep(1) # waits for Cosmos DB to save contents to the collection - output = store.similarity_search("Dogs", k=2) + output = store.similarity_search("intelligent herders", k=5) assert output - assert output[0].page_content == "Dogs are tough." + assert len(output) == 5 + assert "Border Collies" in output[0].page_content safe_delete_database(cosmos_client) def test_from_texts_cosine_distance_delete_one( @@ -120,13 +129,7 @@ def test_from_texts_cosine_distance_delete_one( partition_key: Any, azure_openai_embeddings: OpenAIEmbeddings, ) -> None: - texts = [ - "Dogs are tough.", - "Cats have fluff.", - "What is a sandwich?", - "That fence is purple.", - ] - metadatas = [{"a": 1}, {"b": 1}, {"c": 1}, {"d": 1, "e": 2}] + texts, metadatas = self._get_texts_and_metadata() store = AzureCosmosDBNoSqlVectorSearch.from_texts( texts, @@ -141,20 +144,24 @@ def test_from_texts_cosine_distance_delete_one( indexing_policy=get_vector_indexing_policy("flat"), cosmos_container_properties={"partition_key": partition_key}, cosmos_database_properties={}, + full_text_policy=get_full_text_policy(), + full_text_search_enabled=True, ) sleep(1) # waits for Cosmos DB to save contents to the collection - output = store.similarity_search("Dogs", k=1) + output = store.similarity_search("intelligent herders", k=1) assert output - assert output[0].page_content == "Dogs are tough." + assert len(output) == 1 + assert "Border Collies" in output[0].page_content # delete one document store.delete_document_by_id(str(output[0].metadata["id"])) sleep(2) - output2 = store.similarity_search("Dogs", k=1) + output2 = store.similarity_search("intelligent herders", k=1) assert output2 - assert output2[0].page_content != "Dogs are tough." + assert len(output2) == 1 + assert "Border Collies" not in output2[0].page_content safe_delete_database(cosmos_client) def test_from_documents_cosine_distance_with_filtering( @@ -164,12 +171,7 @@ def test_from_documents_cosine_distance_with_filtering( azure_openai_embeddings: OpenAIEmbeddings, ) -> None: """Test end to end construction and search.""" - documents = [ - Document(page_content="Dogs are tough.", metadata={"a": 1}), - Document(page_content="Cats have fluff.", metadata={"a": 1}), - Document(page_content="What is a sandwich?", metadata={"c": 1}), - Document(page_content="That fence is purple.", metadata={"d": 1, "e": 2}), - ] + documents = self._get_documents() store = AzureCosmosDBNoSqlVectorSearch.from_documents( documents, @@ -183,33 +185,321 @@ def test_from_documents_cosine_distance_with_filtering( indexing_policy=get_vector_indexing_policy("flat"), cosmos_container_properties={"partition_key": partition_key}, cosmos_database_properties={}, + full_text_policy=get_full_text_policy(), + full_text_search_enabled=True, ) sleep(1) # waits for Cosmos DB to save contents to the collection - output = store.similarity_search("Dogs", k=4) + output = store.similarity_search("intelligent herders", k=4) assert len(output) == 4 - assert output[0].page_content == "Dogs are tough." + assert "Border Collies" in output[0].page_content assert output[0].metadata["a"] == 1 - pre_filter = { - "where_clause": "WHERE c.metadata.a=1", - } + # pre_filter = { + # "conditions": [ + # {"property": "metadata.a", "operator": "$eq", "value": 1}, + # ], + # } + pre_filter = PreFilter( + conditions=[ + Condition(property="metadata.a", operator="$eq", value=1), + ], + ) output = store.similarity_search( - "Dogs", k=4, pre_filter=pre_filter, with_embedding=True + "intelligent herders", k=4, pre_filter=pre_filter, with_embedding=True ) - assert len(output) == 2 - assert output[0].page_content == "Dogs are tough." + assert len(output) == 3 + assert "Border Collies" in output[0].page_content assert output[0].metadata["a"] == 1 - pre_filter = { - "where_clause": "WHERE c.metadata.a=1", - "limit_offset_clause": "OFFSET 0 LIMIT 1", - } + # pre_filter = { + # "conditions": [ + # {"property": "metadata.a", "operator": "$eq", "value": 1}, + # ], + # } + pre_filter = PreFilter( + conditions=[ + Condition(property="metadata.a", operator="$eq", value=1), + ], + ) + offset_limit = "OFFSET 0 LIMIT 1" - output = store.similarity_search("Dogs", k=4, pre_filter=pre_filter) + output = store.similarity_search( + "intelligent herders", k=4, pre_filter=pre_filter, offset_limit=offset_limit + ) assert len(output) == 1 - assert output[0].page_content == "Dogs are tough." + assert "Border Collies" in output[0].page_content assert output[0].metadata["a"] == 1 safe_delete_database(cosmos_client) + + def test_from_documents_full_text_and_hybrid( + self, + cosmos_client: Any, + partition_key: Any, + azure_openai_embeddings: OpenAIEmbeddings, + ) -> None: + """Test end to end construction and search.""" + documents = self._get_documents() + + store = AzureCosmosDBNoSqlVectorSearch.from_documents( + documents, + embedding=azure_openai_embeddings, + cosmos_client=cosmos_client, + database_name=database_name, + container_name=container_name, + vector_embedding_policy=get_vector_embedding_policy( + "cosine", "float32", 1536 + ), + full_text_policy=get_full_text_policy(), + indexing_policy=get_vector_indexing_policy("diskANN"), + cosmos_container_properties={"partition_key": partition_key}, + cosmos_database_properties={}, + full_text_search_enabled=True, + ) + + sleep(480) # waits for Cosmos DB to save contents to the collection + + # Full text search contains any + # pre_filter = { + # "conditions": [ + # { + # "property": "text", + # "operator": "$full_text_contains_any", + # "value": "intelligent herders", + # }, + # ], + # } + pre_filter = PreFilter( + conditions=[ + Condition( + property="text", + operator="$full_text_contains_all", + value="intelligent herders", + ), + ], + ) + output = store.similarity_search( + "intelligent herders", + k=5, + pre_filter=pre_filter, + query_type=CosmosDBQueryType.FULL_TEXT_SEARCH, + ) + + assert output + assert len(output) == 3 + assert "Border Collies" in output[0].page_content + + # Full text search contains all + # pre_filter = { + # "conditions": [ + # { + # "property": "text", + # "operator": "$full_text_contains_all", + # "value": "intelligent herders", + # }, + # ], + # } + pre_filter = PreFilter( + conditions=[ + Condition( + property="text", + operator="$full_text_contains_all", + value="intelligent herders", + ), + ], + ) + + output = store.similarity_search( + "intelligent herders", + k=5, + pre_filter=pre_filter, + query_type=CosmosDBQueryType.FULL_TEXT_SEARCH, + ) + + assert output + assert len(output) == 1 + assert "Border Collies" in output[0].page_content + + # Full text search BM25 ranking + output = store.similarity_search( + "intelligent herders", k=5, query_type=CosmosDBQueryType.FULL_TEXT_RANK + ) + + assert output + assert len(output) == 5 + assert "Standard Poodles" in output[0].page_content + + # Full text search BM25 ranking with filtering + # pre_filter = { + # "conditions": [ + # {"property": "metadata.a", "operator": "$eq", "value": 1}, + # ], + # } + pre_filter = PreFilter( + conditions=[ + Condition(property="metadata.a", operator="$eq", value=1), + ], + ) + output = store.similarity_search( + "intelligent herders", + k=5, + pre_filter=pre_filter, + query_type=CosmosDBQueryType.FULL_TEXT_RANK, + ) + + assert output + assert len(output) == 3 + assert "Border Collies" in output[0].page_content + + # Hybrid search RRF ranking combination of full text search and vector search + output = store.similarity_search( + "intelligent herders", k=5, query_type=CosmosDBQueryType.HYBRID + ) + + assert output + assert len(output) == 5 + assert "Border Collies" in output[0].page_content + + # Hybrid search RRF ranking with filtering + # pre_filter = { + # "conditions": [ + # {"property": "metadata.a", "operator": "$eq", "value": 1}, + # ], + # } + pre_filter = PreFilter( + conditions=[ + Condition(property="metadata.a", operator="$eq", value=1), + ], + ) + output = store.similarity_search( + "intelligent herders", + k=5, + pre_filter=pre_filter, + query_type=CosmosDBQueryType.HYBRID, + ) + + assert output + assert len(output) == 3 + assert "Border Collies" in output[0].page_content + + # Full text search BM25 ranking with full text filtering + # pre_filter = { + # "conditions": [ + # { + # "property": "text", + # "operator": "$full_text_contains", + # "value": "energetic", + # }, + # ] + # } + + pre_filter = PreFilter( + conditions=[ + Condition( + property="text", operator="$full_text_contains", value="energetic" + ), + ], + ) + output = store.similarity_search( + "intelligent herders", + k=5, + pre_filter=pre_filter, + query_type=CosmosDBQueryType.FULL_TEXT_RANK, + ) + + assert output + assert len(output) == 3 + assert "Border Collies" in output[0].page_content + + # Full text search BM25 ranking with full text filtering + # pre_filter = { + # "conditions": [ + # { + # "property": "text", + # "operator": "$full_text_contains", + # "value": "energetic", + # }, + # {"property": "metadata.a", "operator": "$eq", "value": 2}, + # ], + # "logical_operator": "$and", + # } + pre_filter = PreFilter( + conditions=[ + Condition( + property="text", operator="$full_text_contains", value="energetic" + ), + Condition(property="metadata.a", operator="$eq", value=2), + ], + logical_operator="$and", + ) + output = store.similarity_search( + "intelligent herders", + k=5, + pre_filter=pre_filter, + query_type=CosmosDBQueryType.FULL_TEXT_RANK, + ) + + assert output + assert len(output) == 2 + assert "Standard Poodles" in output[0].page_content + + def _get_documents(self) -> List[Document]: + return [ + Document( + page_content="Border Collies are intelligent, energetic " + "herders skilled in outdoor activities.", + metadata={"a": 1}, + ), + Document( + page_content="Golden Retrievers are friendly, loyal companions " + "with excellent retrieving skills.", + metadata={"a": 2}, + ), + Document( + page_content="Labrador Retrievers are playful, eager " + "learners and skilled retrievers.", + metadata={"a": 1}, + ), + Document( + page_content="Australian Shepherds are agile, energetic " + "herders excelling in outdoor tasks.", + metadata={"a": 2, "b": 1}, + ), + Document( + page_content="German Shepherds are brave, loyal protectors " + "excelling in versatile tasks.", + metadata={"a": 1, "b": 2}, + ), + Document( + page_content="Standard Poodles are intelligent, energetic " + "learners excelling in agility.", + metadata={"a": 2, "b": 3}, + ), + ] + + def _get_texts_and_metadata(self) -> Tuple[List[str], List[Dict[str, Any]]]: + texts = [ + "Border Collies are intelligent, " + "energetic herders skilled in outdoor activities.", + "Golden Retrievers are friendly, " + "loyal companions with excellent retrieving skills.", + "Labrador Retrievers are playful, " + "eager learners and skilled retrievers.", + "Australian Shepherds are agile, " + "energetic herders excelling in outdoor tasks.", + "German Shepherds are brave, " + "loyal protectors excelling in versatile tasks.", + "Standard Poodles are intelligent, " + "energetic learners excelling in agility.", + ] + metadatas = [ + {"a": 1}, + {"a": 2}, + {"a": 1}, + {"a": 2, "b": 1}, + {"a": 1, "b": 2}, + {"a": 2, "b": 1}, + ] + return texts, metadatas