test: add support for cookie auth and request proxy in jest

.gitignore

@@ -68,6 +68,9 @@ typings/
 # Yarn Integrity file
 .yarn-integrity
 
+# Yarn cache
+.yarn
+
 # dotenv environment variables file
 .env
 .env.test

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

README.md

@@ -148,5 +148,23 @@ const scraper = new Scraper({
 });
 ```
 
+## Testing
+This package includes unit tests for all major functionality. Given the speed at which Twitter's private API
+changes, failing tests are to be expected.
+
+```sh
+npm run test
+```
+
+Before running tests, you should configure environment variables for authentication.
+
+```
+TWITTER_USERNAME=    # Account username
+TWITTER_PASSWORD=    # Account password
+TWITTER_EMAIL=       # Account email
+TWITTER_COOKIES=     # JSON-serialized array of cookies of an authenticated session
+PROXY_URL=           # HTTP(s) proxy for requests (optional)
+```
+
 ## Contributing
 We use [Conventional Commits](https://www.conventionalcommits.org), and enforce this with precommit checks.

package.json

@@ -41,6 +41,7 @@
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-prettier": "^4.2.1",
     "gh-pages": "^5.0.0",
+    "https-proxy-agent": "^7.0.2",
     "husky": "^8.0.3",
     "jest": "^29.5.0",
     "lint-staged": "^13.2.2",

src/auth.test.ts

@@ -1,29 +1,19 @@
-import { Scraper } from './scraper';
-
-export async function authSearchScraper() {
-  const username = process.env['TWITTER_USERNAME'];
-  const password = process.env['TWITTER_PASSWORD'];
-  const email = process.env['TWITTER_EMAIL'];
-  if (!username || !password) {
-    throw new Error(
-      'TWITTER_USERNAME and TWITTER_PASSWORD variables must be defined.',
-    );
-  }
-
-  const scraper = new Scraper();
-  await scraper.login(username, password, email);
-  return scraper;
-}
+import { getScraper } from './test-utils'
 
 test('scraper can log in', async () => {
-  const scraper = await authSearchScraper();
+  const scraper = await getScraper({ authMethod: 'password' });
+  await expect(scraper.isLoggedIn()).resolves.toBeTruthy();
+}, 15000);
+
+test('scraper can log in with cookies', async () => {
+  const scraper = await getScraper();
   await expect(scraper.isLoggedIn()).resolves.toBeTruthy();
 });
 
 test('scraper can restore its login state from cookies', async () => {
-  const scraper = await authSearchScraper();
+  const scraper = await getScraper();
   await expect(scraper.isLoggedIn()).resolves.toBeTruthy();
-  const scraper2 = new Scraper();
+  const scraper2 = await getScraper({ authMethod: 'anonymous' });
   await expect(scraper2.isLoggedIn()).resolves.toBeFalsy();
 
   const cookies = await scraper.getCookies();
@@ -33,10 +23,10 @@ test('scraper can restore its login state from cookies', async () => {
 });
 
 test('scraper can log out', async () => {
-  const scraper = await authSearchScraper();
+  const scraper = await getScraper({ authMethod: 'password' });
   await expect(scraper.isLoggedIn()).resolves.toBeTruthy();
 
   await scraper.logout();
 
   await expect(scraper.isLoggedIn()).resolves.toBeFalsy();
-});
+}, 15000);

src/profile.test.ts

@@ -1,5 +1,5 @@
 import { Profile } from './profile';
-import { Scraper } from './scraper';
+import { getScraper } from './test-utils';
 
 test('scraper can get profile', async () => {
   const expected: Profile = {
@@ -19,7 +19,7 @@ test('scraper can get profile', async () => {
     website: 'https://nomadic.name',
   };
 
-  const scraper = new Scraper();
+  const scraper = await getScraper();
 
   const actual = await scraper.getProfile('nomadic_ua');
   expect(actual.avatar).toEqual(expected.avatar);
@@ -56,7 +56,7 @@ test('scraper can get partial private profile', async () => {
     website: undefined,
   };
 
-  const scraper = new Scraper();
+  const scraper = await getScraper();
 
   const actual = await scraper.getProfile('tomdumont');
   expect(actual.avatar).toEqual(expected.avatar);
@@ -75,16 +75,16 @@ test('scraper can get partial private profile', async () => {
 });
 
 test('scraper cannot get suspended profile', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   expect(scraper.getProfile('123')).rejects.toThrow();
 });
 
 test('scraper cannot get not found profile', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   expect(scraper.getProfile('sample3123131')).rejects.toThrow();
 });
 
 test('scraper can get profile by screen name', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   await scraper.getProfile('Twitter');
 });

src/search.test.ts

@@ -1,9 +1,9 @@
-import { authSearchScraper } from './auth.test';
+import { getScraper } from './test-utils';
 import { SearchMode } from './search';
 import { QueryTweetsResponse } from './timeline-v1';
 
 test('scraper can process search cursor', async () => {
-  const scraper = await authSearchScraper();
+  const scraper = await getScraper();
 
   let cursor: string | undefined = undefined;
   const maxTweets = 30;
@@ -24,7 +24,7 @@ test('scraper can process search cursor', async () => {
 }, 30000);
 
 test('scraper can search profiles', async () => {
-  const scraper = await authSearchScraper();
+  const scraper = await getScraper();
 
   const seenProfiles = new Map<string, boolean>();
   const maxProfiles = 150;
@@ -47,7 +47,7 @@ test('scraper can search profiles', async () => {
 }, 30000);
 
 test('scraper can search tweets', async () => {
-  const scraper = await authSearchScraper();
+  const scraper = await getScraper();
 
   const seenTweets = new Map<string, boolean>();
   const maxTweets = 150;

src/test-utils.ts

@@ -0,0 +1,56 @@
+import { HttpsProxyAgent } from 'https-proxy-agent'
+import { Scraper } from './scraper';
+
+export interface ScraperTestOptions {
+  /**
+   * Force the scraper to use username/password to authenticate instead of cookies. Only used
+   * by this file for testing auth, but very unreliable. Should always use cookies to resume
+   * session when possible.
+   */
+  authMethod: 'password' | 'cookies' | 'anonymous';
+}
+
+export async function getScraper(options: Partial<ScraperTestOptions> = { authMethod: 'cookies' }) {
+  const username = process.env['TWITTER_USERNAME'];
+  const password = process.env['TWITTER_PASSWORD'];
+  const email = process.env['TWITTER_EMAIL'];
+  const cookies = process.env['TWITTER_COOKIES'];
+  const proxyUrl = process.env['PROXY_URL'];
+  let agent: any;
+
+  if (options.authMethod === 'cookies' && !cookies) {
+    console.warn('TWITTER_COOKIES variable is not defined, reverting to password auth (not recommended)')
+    options.authMethod = 'password'
+  }
+
+  if (options.authMethod === 'password' && !(username && password)) {
+    throw new Error(
+      'TWITTER_USERNAME and TWITTER_PASSWORD variables must be defined.',
+    )
+  }
+
+  if (proxyUrl) {
+    agent = new HttpsProxyAgent(proxyUrl, {
+      rejectUnauthorized: false,
+    })
+  }
+
+  const scraper = new Scraper({
+    transform: {
+      request: (input, init) => {
+        if (agent) {
+          return [input, { ...init, agent }]
+        }
+        return [input, init]
+      }
+    }
+  });
+
+  if (options.authMethod === 'password') {
+    await scraper.login(username!, password!, email);
+  } else if (options.authMethod === 'cookies') {
+    await scraper.setCookies(JSON.parse(cookies!));
+  }
+
+  return scraper;
+}

src/trends.test.ts

@@ -1,8 +1,8 @@
-import { Scraper } from './scraper';
+import { getScraper } from './test-utils';
 
 test('scraper can get trends', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   const trends = await scraper.getTrends();
   expect(trends).toHaveLength(20);
   trends.forEach((trend) => expect(trend).not.toBeFalsy());
-});
+}, 15000);

src/tweets.test.ts

@@ -1,4 +1,4 @@
-import { Scraper } from './scraper';
+import { getScraper } from './test-utils';
 import { Mention, Tweet } from './tweets';
 
 test('scraper can get tweet', async () => {
@@ -28,7 +28,7 @@ test('scraper can get tweet', async () => {
     sensitiveContent: false,
   };
 
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   const actual = await scraper.getTweet('1585338303800578049');
   delete actual?.likes;
   delete actual?.replies;
@@ -38,7 +38,7 @@ test('scraper can get tweet', async () => {
 });
 
 test('scraper can get tweets without logging in', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper({ authMethod: 'anonymous' });
   const tweets = scraper.getTweets('elonmusk', 10);
 
   let counter = 0;
@@ -52,7 +52,7 @@ test('scraper can get tweets without logging in', async () => {
 });
 
 test('scraper can get first tweet matching query', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
 
   const timeline = scraper.getTweets('elonmusk');
   const latestQuote = await scraper.getTweetWhere(timeline, { isQuoted: true });
@@ -61,7 +61,7 @@ test('scraper can get first tweet matching query', async () => {
 });
 
 test('scraper can get all tweets matching query', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
 
   // Sample size of 20 should be enough without taking long.
   const timeline = scraper.getTweets('elonmusk', 20);
@@ -78,7 +78,7 @@ test('scraper can get all tweets matching query', async () => {
 }, 20000);
 
 test('scraper can get latest tweet', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
 
   // OLD APPROACH (without retweet filtering)
   const tweets = scraper.getTweets('elonmusk', 1);
@@ -102,7 +102,7 @@ test('scraper can get user mentions in tweets', async () => {
     },
   ];
 
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   const tweet = await scraper.getTweet('1554522888904101890');
   expect(tweet?.mentions).toEqual(expected);
 });
@@ -138,7 +138,7 @@ test('scraper can get tweet quotes and replies', async () => {
     sensitiveContent: false,
   };
 
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   const quote = await scraper.getTweet('1237110897597976576');
   expect(quote?.isQuoted).toBeTruthy();
   delete quote?.quotedStatus?.likes;
@@ -185,7 +185,7 @@ test('scraper can get retweet', async () => {
     sensitiveContent: false,
   };
 
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   const retweet = await scraper.getTweet('1685032881872330754');
   expect(retweet?.isRetweet).toBeTruthy();
   delete retweet?.retweetedStatus?.likes;
@@ -220,7 +220,7 @@ test('scraper can get tweet views', async () => {
     sensitiveContent: false,
   };
 
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   const actual = await scraper.getTweet('1606055187348688896');
   expect(actual?.views).toBeTruthy();
   delete actual?.likes;
@@ -231,7 +231,7 @@ test('scraper can get tweet views', async () => {
 });
 
 test('scraper can get tweet thread', async () => {
-  const scraper = new Scraper();
+  const scraper = await getScraper();
   const tweet = await scraper.getTweet('1665602315745673217');
   expect(tweet).not.toBeNull();
   expect(tweet?.isSelfThread).toBeTruthy();

tsconfig.json

@@ -1,6 +1,6 @@
 {
   "extends": "@tsconfig/node16/tsconfig.json",
-  "exclude": ["node_modules", "dist", "**/*.test.ts"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts", "src/test-utils.ts"],
   "compilerOptions": {
     // TODO: Remove "dom" from this when support for Node 16 is dropped
     "lib": ["es2021", "dom"],