diff --git a/README.md b/README.md index 77af8a3..4be80fe 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ pip install webtranspose X - - License + + License License @@ -45,6 +45,9 @@ In the near future, **nobody will open websites**. Instead, we will be directly ```python import webtranspose as webt +import os +os.environ['WEBTRANSPOSE_API_KEY'] = "YOUR WEBT API KEY" + crawl = webt.Crawl( "https://www.example.com", max_pages=100, @@ -58,6 +61,9 @@ await crawl.crawl() # crawl.queue_crawl() for async ```python import webtranspose as webt +import os +os.environ['WEBTRANSPOSE_API_KEY'] = "YOUR WEBT API KEY" + schema = { "Merchant Name": "string", "Title of Product": "string", @@ -67,11 +73,28 @@ schema = { scraper = webt.Scraper( schema, render_js=True, - api_key="YOUR_WEBTRANSPOSE_API_KEY" ) out_json = scraper.scrape("https://www.example.com") ``` +## Web Search (SERP API) + +```python +import webtranspose as webt + +import os +os.environ['WEBTRANSPOSE_API_KEY'] = "YOUR WEBT API KEY" + +results = webt.search("what caused the fourth great ninja war?") +# results.keys() +# ['results'] + +# AI Filter +results = webt.search_filter("Paul Graham's Blog") +# results.keys() +# ['results', 'filtered_results'] +``` + ## Installation diff --git a/pyproject.toml b/pyproject.toml index 784549b..c6e934c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "webtranspose" -version = "0.2.1" +version = "0.3.0" description = "APIs for the internet" authors = ["Mike Gee "] diff --git a/src/webtranspose/__init__.py b/src/webtranspose/__init__.py index 3fd9853..e4f2fa5 100644 --- a/src/webtranspose/__init__.py +++ b/src/webtranspose/__init__.py @@ -2,9 +2,10 @@ __author__ = """Mike Gee""" __email__ = "mike@webtranspose.com" -__version__ = "0.1.0" +__version__ = "0.3.0" from .chat import * from .crawl import * from .openai import * from .scrape import * +from .search import * \ No newline at end of file diff --git a/src/webtranspose/chat.py b/src/webtranspose/chat.py index c3d2497..4bd4db5 100644 --- a/src/webtranspose/chat.py +++ b/src/webtranspose/chat.py @@ -164,3 +164,31 @@ def delete_crawls(self, crawl_id_list: list): "crawl_id_list": crawl_id_list, } run_webt_api(query_json, "v1/chat/crawls/delete", self.api_key) + + +def get_chatbot(chatbot_id: str, api_key = None) -> Chatbot: + """ + Get a chatbot. + + :param chatbot_id: The ID of the chatbot. + :return: The chatbot. + """ + if api_key is None: + api_key = os.environ.get("WEBTRANSPOSE_API_KEY") + if api_key is None: + raise ValueError( + "No Web Transpose API provided. \n\nTo use Chatbots, set the WEBTRANSPOSE_API_KEY from https://webtranspose.com." + ) + get_json = { + "chatbot_id": chatbot_id, + } + chat_json = run_webt_api(get_json, "v1/chat/get", api_key) + chatbot_data = chat_json.get('chatbot', {}) + chatbot = Chatbot( + chatbot_id=chatbot_data.get('id'), + name=chatbot_data.get('name'), + max_pages=chatbot_data.get('num_run', 100), + verbose=False, + _created=True + ) + return chatbot \ No newline at end of file diff --git a/src/webtranspose/scrape.py b/src/webtranspose/scrape.py index c8c2d4b..86479ba 100644 --- a/src/webtranspose/scrape.py +++ b/src/webtranspose/scrape.py @@ -50,8 +50,7 @@ def __init__( self.scraper_id = scraper_id if self.scraper is None: self.scraper = OpenAIScraper() - self.scraper_id = None - if self.api_key is None: + if self.scraper_id is None: self.scraper_id = str(uuid.uuid4()) self.created = _created diff --git a/src/webtranspose/search.py b/src/webtranspose/search.py new file mode 100644 index 0000000..aea0af9 --- /dev/null +++ b/src/webtranspose/search.py @@ -0,0 +1,38 @@ +import os +import requests + +from .webt_api import run_webt_api + + +def search(query): + if api_key is None: + api_key = os.environ.get("WEBTRANSPOSE_API_KEY") + + if api_key is not None: + out_json = run_webt_api( + { + "query": query, + }, + "/v1/search", + api_key, + ) + return out_json + + raise ValueError("Must provide api_key or set WEBTRANSPOSE_API_KEY in environment variables.") + + +def search_filter(query): + if api_key is None: + api_key = os.environ.get("WEBTRANSPOSE_API_KEY") + + if api_key is not None: + out_json = run_webt_api( + { + "query": query, + }, + "/v1/search/filter", + api_key, + ) + return out_json + + raise ValueError("Must provide api_key or set WEBTRANSPOSE_API_KEY in environment variables.") \ No newline at end of file