Skip to content

Commit

Permalink
Ensured only http or https proxies are passed
Browse files Browse the repository at this point in the history
  • Loading branch information
wik3d committed Sep 23, 2024
1 parent 8147893 commit 29bdb29
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 11 deletions.
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ npm i reddit-extractor
```ts
import { Scraper, Post } from 'reddit-extractor';

// Proxies are not required, but recommended for large applications
// Proxies are not required, but recommended for large applications (Only http(s) proxies are supported)
// Reddit's JSON API rate limits if you make ~100 requests within quick succession
const proxyConfig = {
protocol: '',
protocol: 'http',
host: '',
port: 12321,
auth: {
Expand All @@ -27,7 +27,7 @@ const proxyConfig = {
},
};

const RedditExtractor = new Scraper('./tempFiles', proxyConfig)
const RedditExtractor = new Scraper('./tempFiles', proxyConfig);
```
<br>

Expand All @@ -44,7 +44,7 @@ console.log(postData);
```
<br>

### Get multiple recent posts from a subreddit
### Get recent posts from a subreddit
```ts
// Will return the 5 most recent posts from r/memes
const subreddit = 'memes';
Expand All @@ -54,7 +54,7 @@ if (!latestFivePosts.length) return console.error('No posts found');
const mostRecentPostData = latestFivePosts[0];

if ('error' in mostRecentPostData) {
return console.error('Error in most recent post', mostRecentPostData.error)
return console.error('Error in most recent post', mostRecentPostData.error);
}

console.log(mostRecentPostData);
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "reddit-extractor",
"version": "1.2.1",
"version": "1.2.2",
"description": "Gets Reddit post data",
"main": "production/_exports.js",
"scripts": {
Expand Down
2 changes: 1 addition & 1 deletion src/_exports.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export { Scraper } from './index';
export { Post } from './types/index';
export { Post, proxyType, Media } from './types/index';
9 changes: 5 additions & 4 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,17 @@ export class Scraper {
private jar: CookieJar;

constructor(downloadPath: string = './', private proxy?: proxyType, private forceProxy = false) {
this.downloadPath = downloadPath;
this.ensureDirectoryExists(this.downloadPath);
this.jar = new CookieJar();

if (proxy) {
if (!['http', 'https'].includes(proxy.protocol.toLowerCase())) throw new Error('Only http and https proxies are supported');
const proxyAgent = new HttpsProxyAgent(
`${this.proxy.protocol}://${Object.keys(this.proxy.auth || {}).length > 0 ? `${this.proxy.auth?.username}:${this.proxy.auth?.password}` : ''}@${this.proxy.host}:${this.proxy.port}`,
);
this.agent = proxyAgent;
}

this.downloadPath = downloadPath;
this.ensureDirectoryExists(this.downloadPath);
this.jar = new CookieJar();
}

// Fetch a single reddit post
Expand Down

0 comments on commit 29bdb29

Please sign in to comment.