From 5bf8fb474116dd20b8828bb2fd2881971b1d7aea Mon Sep 17 00:00:00 2001 From: Nulo Date: Sun, 16 Jun 2024 18:03:31 -0300 Subject: [PATCH] feat: getTweetsAndReplies (#88) --- src/api-data.ts | 4 +++- src/scraper.ts | 28 +++++++++++++++++++++++ src/tweets.ts | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/api-data.ts b/src/api-data.ts index 1311972f..76f31110 100644 --- a/src/api-data.ts +++ b/src/api-data.ts @@ -8,7 +8,9 @@ import stringify from 'json-stable-stringify'; const endpoints = { // TODO: Migrate other endpoint URLs here UserTweets: - 'https://twitter.com/i/api/graphql/H8OOoI-5ZE4NxgRr8lfyWg/UserTweets?variables=%7B%22userId%22%3A%2244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D', + 'https://twitter.com/i/api/graphql/V7H0Ap3_Hh2FyS75OCDO3Q/UserTweets?variables=%7B%22userId%22%3A%224020276615%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D', + UserTweetsAndReplies: + 'https://twitter.com/i/api/graphql/E4wA5vo2sjVyvpliUffSCw/UserTweetsAndReplies?variables=%7B%22userId%22%3A%224020276615%22%2C%22count%22%3A40%2C%22cursor%22%3A%22DAABCgABGPWl-F-ATiIKAAIY9YfiF1rRAggAAwAAAAEAAA%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D', UserLikedTweets: 'https://twitter.com/i/api/graphql/eSSNbhECHHWWALkkQq-YTA/Likes?variables=%7B%22userId%22%3A%222244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D', TweetDetail: diff --git a/src/scraper.ts b/src/scraper.ts index a4d8e184..18ebc536 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -30,6 +30,8 @@ import { TweetQuery, getTweet, fetchListTweets, + getTweetsAndRepliesByUserId, + getTweetsAndReplies, } from './tweets'; import fetch from 'cross-fetch'; @@ -272,6 +274,32 @@ export class Scraper { return getTweetsByUserId(userId, maxTweets, this.auth); } + /** + * Fetches tweets and replies from a Twitter user. + * @param user The user whose tweets should be returned. + * @param maxTweets The maximum number of tweets to return. Defaults to `200`. + * @returns An {@link AsyncGenerator} of tweets from the provided user. + */ + public getTweetsAndReplies( + user: string, + maxTweets = 200, + ): AsyncGenerator { + return getTweetsAndReplies(user, maxTweets, this.auth); + } + + /** + * Fetches tweets and replies from a Twitter user using their ID. + * @param userId The user whose tweets should be returned. + * @param maxTweets The maximum number of tweets to return. Defaults to `200`. + * @returns An {@link AsyncGenerator} of tweets from the provided user. + */ + public getTweetsAndRepliesByUserId( + userId: string, + maxTweets = 200, + ): AsyncGenerator { + return getTweetsAndRepliesByUserId(userId, maxTweets, this.auth); + } + /** * Fetches the first tweet matching the given query. * diff --git a/src/tweets.ts b/src/tweets.ts index 0bd7cddd..a94aa398 100644 --- a/src/tweets.ts +++ b/src/tweets.ts @@ -129,6 +129,38 @@ export async function fetchTweets( return parseTimelineTweetsV2(res.value); } +export async function fetchTweetsAndReplies( + userId: string, + maxTweets: number, + cursor: string | undefined, + auth: TwitterAuth, +): Promise { + if (maxTweets > 40) { + maxTweets = 40; + } + + const userTweetsRequest = + apiRequestFactory.createUserTweetsAndRepliesRequest(); + userTweetsRequest.variables.userId = userId; + userTweetsRequest.variables.count = maxTweets; + userTweetsRequest.variables.includePromotedContent = false; // true on the website + + if (cursor != null && cursor != '') { + userTweetsRequest.variables['cursor'] = cursor; + } + + const res = await requestApi( + userTweetsRequest.toRequestUrl(), + auth, + ); + + if (!res.success) { + throw res.err; + } + + return parseTimelineTweetsV2(res.value); +} + export async function fetchListTweets( listId: string, maxTweets: number, @@ -187,6 +219,34 @@ export function getTweetsByUserId( }); } +export function getTweetsAndReplies( + user: string, + maxTweets: number, + auth: TwitterAuth, +): AsyncGenerator { + return getTweetTimeline(user, maxTweets, async (q, mt, c) => { + const userIdRes = await getUserIdByScreenName(q, auth); + + if (!userIdRes.success) { + throw userIdRes.err; + } + + const { value: userId } = userIdRes; + + return fetchTweetsAndReplies(userId, mt, c, auth); + }); +} + +export function getTweetsAndRepliesByUserId( + userId: string, + maxTweets: number, + auth: TwitterAuth, +): AsyncGenerator { + return getTweetTimeline(userId, maxTweets, (q, mt, c) => { + return fetchTweetsAndReplies(q, mt, c, auth); + }); +} + export async function fetchLikedTweets( userId: string, maxTweets: number,