add fetchListTweets function to get tweets from list

src/api-data.ts

@@ -13,6 +13,8 @@ const endpoints = {
     'https://twitter.com/i/api/graphql/xOhkmRac04YFZmOzU9PJHg/TweetDetail?variables=%7B%22focalTweetId%22%3A%221237110546383724547%22%2C%22with_rux_injections%22%3Afalse%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Afalse%7D',
   TweetResultByRestId:
     'https://twitter.com/i/api/graphql/DJS3BdhUhcaEpZ7B7irJDg/TweetResultByRestId?variables=%7B%22tweetId%22%3A%221237110546383724547%22%2C%22withCommunity%22%3Afalse%2C%22includePromotedContent%22%3Afalse%2C%22withVoice%22%3Afalse%7D&features=%7B%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D',
+  ListTweets:
+    'https://twitter.com/i/api/graphql/whF0_KH1fCkdLLoyNPMoEw/ListLatestTweetsTimeline?variables=%7B%22listId%22%3A%221736495155002106192%22%2C%22count%22%3A20%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D',
 } as const;
 
 export interface EndpointFieldInfo {

src/scraper.ts

@@ -28,6 +28,7 @@ import {
   getTweetsByUserId,
   TweetQuery,
   getTweet,
+  fetchListTweets,
 } from './tweets';
 import fetch from 'cross-fetch';
 
@@ -158,6 +159,21 @@ export class Scraper {
     return fetchSearchProfiles(query, maxProfiles, this.auth, cursor);
   }
 
+  /**
+   * Fetches list tweets from Twitter.
+   * @param listId The list id
+   * @param maxTweets The maximum number of tweets to return.
+   * @param cursor The search cursor, which can be passed into further requests for more results.
+   * @returns A page of results, containing a cursor that can be used in further requests.
+   */
+  public fetchListTweets(
+      listId: string,
+      maxTweets: number,
+      cursor?: string,
+  ): Promise<QueryTweetsResponse> {
+    return fetchListTweets(listId, maxTweets, cursor, this.auth);
+  }
+
   /**
    * Fetch the profiles a user is following
    * @param userId The user whose following should be returned

src/timeline-list.ts

@@ -0,0 +1,57 @@
+import { QueryTweetsResponse } from './timeline-v1';
+import { SearchEntryRaw, parseAndPush } from './timeline-v2';
+import { Tweet } from './tweets';
+
+export interface ListTimeline {
+  data?: {
+    list?: {
+      tweets_timeline?: {
+        timeline?: {
+          instructions?: {
+            entries?: SearchEntryRaw[];
+            entry?: SearchEntryRaw;
+            type?: string;
+          }[];
+        };
+      };
+    };
+  };
+}
+
+export function parseListTimelineTweets(
+  timeline: ListTimeline,
+): QueryTweetsResponse {
+  let bottomCursor: string | undefined;
+  let topCursor: string | undefined;
+  const tweets: Tweet[] = [];
+  const instructions =
+    timeline.data?.list?.tweets_timeline?.timeline
+      ?.instructions ?? [];
+  for (const instruction of instructions) {
+    const entries = instruction.entries ?? [];
+
+    for (const entry of entries) {
+      const entryContent = entry.content;
+      if (!entryContent) continue;
+
+      if (entryContent.cursorType === 'Bottom') {
+        bottomCursor = entryContent.value;
+        continue;
+      } else if (entryContent.cursorType === 'Top') {
+        topCursor = entryContent.value;
+        continue;
+      }
+
+      const idStr = entry.entryId;
+      if (!idStr.startsWith('tweet')) {
+        continue;
+      }
+
+      if (entryContent.itemContent) {
+        parseAndPush(tweets, entryContent.itemContent, idStr);
+      }
+    }
+  }
+
+  return { tweets, next: bottomCursor, previous: topCursor };
+}

src/timeline-v2.ts

@@ -334,7 +334,7 @@ export function parseTimelineEntryItemContentRaw(
   return null;
 }
 
-function parseAndPush(
+export function parseAndPush(
   tweets: Tweet[],
   content: TimelineEntryItemContentRaw,
   entryId: string,

src/tweets.test.ts

@@ -1,5 +1,7 @@
 import { getScraper } from './test-utils';
 import { Mention, Tweet } from './tweets';
+import {QueryTweetsResponse} from "./timeline-v1";
+import {SearchMode} from "./search";
 
 test('scraper can get tweet', async () => {
   const expected: Tweet = {
@@ -50,6 +52,27 @@ test('scraper can get tweets without logging in', async () => {
   expect(counter).toBeGreaterThanOrEqual(1);
 });
 
+test.only('scraper can get tweets without logging in', async () => {
+  const scraper = await getScraper();
+
+  let cursor: string | undefined = undefined;
+  const maxTweets = 30;
+  let nTweets = 0;
+  while (nTweets < maxTweets) {
+    const res: QueryTweetsResponse = await scraper.fetchListTweets(
+        '1736495155002106192',
+        maxTweets,
+        cursor,
+    );
+    console.log('res', res);
+
+    expect(res.next).toBeTruthy();
+
+    nTweets += res.tweets.length;
+    cursor = res.next;
+  }
+});
+
 test('scraper can get first tweet matching query', async () => {
   const scraper = await getScraper();
 

src/tweets.ts

@@ -12,6 +12,7 @@ import {
 } from './timeline-v2';
 import { getTweetTimeline } from './timeline-async';
 import { apiRequestFactory } from './api-data';
+import {ListTimeline, parseListTimelineTweets} from "./timeline-list";
 
 export interface Mention {
   id: string;
@@ -127,6 +128,36 @@ export async function fetchTweets(
   return parseTimelineTweetsV2(res.value);
 }
 
+export async function fetchListTweets(
+    listId: string,
+    maxTweets: number,
+    cursor: string | undefined,
+    auth: TwitterAuth,
+): Promise<QueryTweetsResponse> {
+  if (maxTweets > 200) {
+    maxTweets = 200;
+  }
+
+  const listTweetsRequest = apiRequestFactory.createListTweetsRequest();
+  listTweetsRequest.variables.listId = listId;
+  listTweetsRequest.variables.count = maxTweets;
+
+  if (cursor != null && cursor != '') {
+    listTweetsRequest.variables['cursor'] = cursor;
+  }
+
+  const res = await requestApi<ListTimeline>(
+      listTweetsRequest.toRequestUrl(),
+      auth,
+  );
+
+  if (!res.success) {
+    throw res.err;
+  }
+
+  return parseListTimelineTweets(res.value);
+}
+
 export function getTweets(
   user: string,
   maxTweets: number,