Filter Type Refinement (#102)

* fixed broken test command * switched filterType to use enum for filtering posts,etc * trading services version bump
dills122 · Oct 8, 2021 · 8f78ec2 · 8f78ec2
1 parent 216df27
commit 8f78ec2
Show file tree

Hide file tree

Showing 43 changed files with 247 additions and 377 deletions.
diff --git a/apps/cron-app/CHANGELOG.json b/apps/cron-app/CHANGELOG.json
@@ -1,6 +1,12 @@
 {
   "name": "cron-app",
   "entries": [
+    {
+      "version": "0.2.12",
+      "tag": "cron-app_v0.2.12",
+      "date": "Fri, 08 Oct 2021 01:14:44 GMT",
+      "comments": {}
+    },
     {
       "version": "0.2.11",
       "tag": "cron-app_v0.2.11",

diff --git a/apps/cron-app/CHANGELOG.md b/apps/cron-app/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - cron-app
 
-This log was last generated on Thu, 07 Oct 2021 00:12:18 GMT and should not be manually modified.
+This log was last generated on Fri, 08 Oct 2021 01:14:44 GMT and should not be manually modified.
+
+## 0.2.12
+Fri, 08 Oct 2021 01:14:44 GMT
+
+_Version update only_
 
 ## 0.2.11
 Thu, 07 Oct 2021 00:12:18 GMT

diff --git a/apps/cron-app/package.json b/apps/cron-app/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cron-app",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "runs all trading cron jobs",
   "main": "cron-runner.js",
   "scripts": {
@@ -17,7 +17,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "trading-services": "^0.2.11",
+    "trading-services": "^0.2.12",
     "node-cron": "~2.0.3",
     "express": "~4.17.1",
     "pm2": "^5.1.0"

diff --git a/apps/sentiment-rpc-server/CHANGELOG.json b/apps/sentiment-rpc-server/CHANGELOG.json
@@ -1,6 +1,12 @@
 {
   "name": "sentiment-rpc-server",
   "entries": [
+    {
+      "version": "0.2.12",
+      "tag": "sentiment-rpc-server_v0.2.12",
+      "date": "Fri, 08 Oct 2021 01:14:44 GMT",
+      "comments": {}
+    },
     {
       "version": "0.2.11",
       "tag": "sentiment-rpc-server_v0.2.11",

diff --git a/apps/sentiment-rpc-server/CHANGELOG.md b/apps/sentiment-rpc-server/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - sentiment-rpc-server
 
-This log was last generated on Thu, 07 Oct 2021 00:12:18 GMT and should not be manually modified.
+This log was last generated on Fri, 08 Oct 2021 01:14:44 GMT and should not be manually modified.
+
+## 0.2.12
+Fri, 08 Oct 2021 01:14:44 GMT
+
+_Version update only_
 
 ## 0.2.11
 Thu, 07 Oct 2021 00:12:18 GMT

diff --git a/apps/sentiment-rpc-server/package.json b/apps/sentiment-rpc-server/package.json
@@ -1,6 +1,6 @@
 {
   "name": "sentiment-rpc-server",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -25,8 +25,8 @@
     "@grpc/proto-loader": "~0.6.4",
     "dotenv": "~10.0.0",
     "google-protobuf": "^3.17.3",
-    "sentiment-service": "~0.2.11",
-    "grpc-tools": "~0.2.11",
+    "sentiment-service": "~0.2.12",
+    "grpc-tools": "~0.2.12",
     "pm2": "^5.1.0"
   },
   "devDependencies": {

diff --git a/common/changes/is-ticker-symbol/post-filter-type-refactor_2021-10-08-01-14.json b/common/changes/is-ticker-symbol/post-filter-type-refactor_2021-10-08-01-14.json
@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "changeType": 1,
+      "comment": "Dependency api-service version bump from ^0.2.11 to ^0.2.12.",
+      "packageName": "is-ticker-symbol"
+    }
+  ],
+  "packageName": "is-ticker-symbol",
+  "email": "dylansteele57@gmail.com"
+}
diff --git a/common/config/rush/version-policies.json b/common/config/rush/version-policies.json
@@ -90,7 +90,7 @@
   {
     "policyName": "traderService",
     "definitionName": "lockStepVersion",
-    "version": "0.2.11",
+    "version": "0.2.12",
     "nextBump": "patch"
   },
   {

diff --git a/lib/grpc-tools/CHANGELOG.json b/lib/grpc-tools/CHANGELOG.json
@@ -1,6 +1,12 @@
 {
   "name": "grpc-tools",
   "entries": [
+    {
+      "version": "0.2.12",
+      "tag": "grpc-tools_v0.2.12",
+      "date": "Fri, 08 Oct 2021 01:14:44 GMT",
+      "comments": {}
+    },
     {
       "version": "0.2.11",
       "tag": "grpc-tools_v0.2.11",

diff --git a/lib/grpc-tools/CHANGELOG.md b/lib/grpc-tools/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - grpc-tools
 
-This log was last generated on Thu, 07 Oct 2021 00:12:18 GMT and should not be manually modified.
+This log was last generated on Fri, 08 Oct 2021 01:14:44 GMT and should not be manually modified.
+
+## 0.2.12
+Fri, 08 Oct 2021 01:14:44 GMT
+
+_Version update only_
 
 ## 0.2.11
 Thu, 07 Oct 2021 00:12:18 GMT

diff --git a/lib/grpc-tools/package.json b/lib/grpc-tools/package.json
@@ -1,6 +1,6 @@
 {
   "name": "grpc-tools",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "",
   "main": "index.js",
   "scripts": {

diff --git a/lib/is-ticker-symbol/package.json b/lib/is-ticker-symbol/package.json
@@ -31,7 +31,7 @@
     "@types/lodash": "~4.14.168",
     "proxyquire": "~2.1.3",
     "@types/proxyquire": "~1.3.28",
-    "api-service": "^0.2.11",
+    "api-service": "^0.2.12",
     "meow": "~9.0.0",
     "@types/meow": "~5.0.0",
     "eslint": "~7.22.0",

diff --git a/lib/trader-sdk/CHANGELOG.json b/lib/trader-sdk/CHANGELOG.json
@@ -1,6 +1,12 @@
 {
   "name": "trader-sdk",
   "entries": [
+    {
+      "version": "0.2.12",
+      "tag": "trader-sdk_v0.2.12",
+      "date": "Fri, 08 Oct 2021 01:14:44 GMT",
+      "comments": {}
+    },
     {
       "version": "0.2.11",
       "tag": "trader-sdk_v0.2.11",

diff --git a/lib/trader-sdk/CHANGELOG.md b/lib/trader-sdk/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - trader-sdk
 
-This log was last generated on Thu, 07 Oct 2021 00:12:18 GMT and should not be manually modified.
+This log was last generated on Fri, 08 Oct 2021 01:14:44 GMT and should not be manually modified.
+
+## 0.2.12
+Fri, 08 Oct 2021 01:14:44 GMT
+
+_Version update only_
 
 ## 0.2.11
 Thu, 07 Oct 2021 00:12:18 GMT

diff --git a/lib/trader-sdk/package.json b/lib/trader-sdk/package.json
@@ -1,6 +1,6 @@
 {
   "name": "trader-sdk",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "",
   "main": "index.js",
   "scripts": {

diff --git a/services/api-service/CHANGELOG.json b/services/api-service/CHANGELOG.json
@@ -1,6 +1,12 @@
 {
   "name": "api-service",
   "entries": [
+    {
+      "version": "0.2.12",
+      "tag": "api-service_v0.2.12",
+      "date": "Fri, 08 Oct 2021 01:14:44 GMT",
+      "comments": {}
+    },
     {
       "version": "0.2.11",
       "tag": "api-service_v0.2.11",

diff --git a/services/api-service/CHANGELOG.md b/services/api-service/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - api-service
 
-This log was last generated on Thu, 07 Oct 2021 00:12:18 GMT and should not be manually modified.
+This log was last generated on Fri, 08 Oct 2021 01:14:44 GMT and should not be manually modified.
+
+## 0.2.12
+Fri, 08 Oct 2021 01:14:44 GMT
+
+_Version update only_
 
 ## 0.2.11
 Thu, 07 Oct 2021 00:12:18 GMT

diff --git a/services/api-service/package.json b/services/api-service/package.json
@@ -1,6 +1,6 @@
 {
   "name": "api-service",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -31,7 +31,7 @@
   "dependencies": {
     "got": "~11.8.2",
     "dotenv": "~10.0.0",
-    "trader-sdk": "^0.2.11",
+    "trader-sdk": "^0.2.12",
     "lodash": "~4.17.21",
     "snoowrap": "~1.22.0"
   }

diff --git a/services/sentiment-service/CHANGELOG.json b/services/sentiment-service/CHANGELOG.json
@@ -1,6 +1,12 @@
 {
   "name": "sentiment-service",
   "entries": [
+    {
+      "version": "0.2.12",
+      "tag": "sentiment-service_v0.2.12",
+      "date": "Fri, 08 Oct 2021 01:14:44 GMT",
+      "comments": {}
+    },
     {
       "version": "0.2.11",
       "tag": "sentiment-service_v0.2.11",

diff --git a/services/sentiment-service/CHANGELOG.md b/services/sentiment-service/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Change Log - sentiment-service
 
-This log was last generated on Thu, 07 Oct 2021 00:12:18 GMT and should not be manually modified.
+This log was last generated on Fri, 08 Oct 2021 01:14:44 GMT and should not be manually modified.
+
+## 0.2.12
+Fri, 08 Oct 2021 01:14:44 GMT
+
+_Version update only_
 
 ## 0.2.11
 Thu, 07 Oct 2021 00:12:18 GMT

diff --git a/services/sentiment-service/integrations/front-page-tester.spec.ts b/services/sentiment-service/integrations/front-page-tester.spec.ts
@@ -1,4 +1,5 @@
 import { assert, expect } from 'chai';
+import { FilterType } from '../lib/reddit/filters';
 import { Generic } from '../lib/services';
 
 describe('Tester::', function () {
@@ -8,9 +9,7 @@ describe('Tester::', function () {
       analyzer: 'natural',
       serviceAnalysisType: 'front-page',
       socialSource: 'reddit',
-      filterFlags: {
-        discussionMode: true
-      },
+      filterType: FilterType.general,
       subreddit: 'wallstreetbets'
     });
 

diff --git a/services/sentiment-service/lib/reddit/filters/comment-filter.ts b/services/sentiment-service/lib/reddit/filters/comment-filter.ts
@@ -1,16 +1,17 @@
 import _ from 'lodash';
 import BadWords from 'bad-words';
 import { Socials } from 'api-service';
-import { SentimentAnalysisFilterFlags } from '../../shared-types';
 import { EquityFilter } from './equity-filter';
 import { EntityFilter } from './entity-filter';
 import { config, SubredditConfigSchema } from '../config';
+import { FilterType } from './';
 
-export interface CommentFilterArgs extends SentimentAnalysisFilterFlags {
+export interface CommentFilterArgs {
   comments: Socials.Reddit.Types.Comment[];
   subreddit: string;
   equityWhitelistEnabled?: boolean;
   equityWhitelist?: string[];
+  filterType: FilterType; //TODO does not seemed to be used here
 }
 
 export class CommentFilter {
@@ -19,11 +20,13 @@ export class CommentFilter {
   private whitelist: string[];
   private subreddit: string;
   private subredditConfig: SubredditConfigSchema;
+  private filterType: FilterType;
 
   constructor(args: CommentFilterArgs) {
     _.assign(this, args);
     this.subreddit = args.subreddit;
     this.subredditConfig = config.subreddits[this.subreddit];
+    this.matureFilter = this.shouldEnableMatureLanguageFilter();
     if (args.equityWhitelistEnabled || (args.equityWhitelist && args.equityWhitelist.length > 0)) {
       this.whitelist = args.equityWhitelist || this.subredditConfig.whitelist;
     }
@@ -64,6 +67,10 @@ export class CommentFilter {
     return commentsWithTickers;
   }
 
+  private shouldEnableMatureLanguageFilter() {
+    return ![FilterType.shitpost, FilterType.chaos].includes(this.filterType);
+  }
+
   private removeStickiedComments() {
     this.comments = this.comments.filter((comment) => {
       return !comment.stickied;