From 0b41188c6af19aabfff6348adb87fc3be199f879 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Mon, 23 Aug 2021 22:13:22 +0100 Subject: [PATCH 01/17] Adds missing migrations --- .../migrations/0185_auto_20210823_2105.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 app/dashboard/migrations/0185_auto_20210823_2105.py diff --git a/app/dashboard/migrations/0185_auto_20210823_2105.py b/app/dashboard/migrations/0185_auto_20210823_2105.py new file mode 100644 index 00000000000..a81a79c2faf --- /dev/null +++ b/app/dashboard/migrations/0185_auto_20210823_2105.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.24 on 2021-08-23 21:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dashboard', '0184_hackathonevent_total_prize'), + ] + + operations = [ + migrations.AlterField( + model_name='bounty', + name='web3_type', + field=models.CharField(choices=[('legacy_gitcoin', 'Legacy Bounty'), ('bounties_network', 'Bounties Network'), ('qr', 'QR Code'), ('web3_modal', 'Web3 Modal'), ('polkadot_ext', 'Polkadot Ext'), ('binance_ext', 'Binance Ext'), ('harmony_ext', 'Harmony Ext'), ('rsk_ext', 'RSK Ext'), ('xinfin_ext', 'Xinfin Ext'), ('nervos_ext', 'Nervos Ext'), ('algorand_ext', 'Algorand Ext'), ('sia_ext', 'Sia Ext'), ('tezos_ext', 'Tezos Ext'), ('casper_ext', 'Casper Ext'), ('fiat', 'Fiat'), ('manual', 'Manual')], default='bounties_network', max_length=50), + ), + migrations.AlterField( + model_name='bountyfulfillment', + name='payout_type', + field=models.CharField(blank=True, choices=[('bounties_network', 'bounties_network'), ('qr', 'qr'), ('fiat', 'fiat'), ('web3_modal', 'web3_modal'), ('polkadot_ext', 'polkadot_ext'), ('binance_ext', 'binance_ext'), ('harmony_ext', 'harmony_ext'), ('rsk_ext', 'rsk_ext'), ('xinfin_ext', 'xinfin_ext'), ('nervos_ext', 'nervos_ext'), ('algorand_ext', 'algorand_ext'), ('sia_ext', 'sia_ext'), ('tezos_ext', 'tezos_ext'), ('casper_ext', 'casper_ext'), ('manual', 'manual')], help_text='payment type used to make the payment', max_length=20, null=True), + ), + migrations.AlterField( + model_name='bountyfulfillment', + name='tenant', + field=models.CharField(blank=True, choices=[('BTC', 'BTC'), ('ETH', 'ETH'), ('ETC', 'ETC'), ('ZIL', 'ZIL'), ('CELO', 'CELO'), ('PYPL', 'PYPL'), ('POLKADOT', 'POLKADOT'), ('BINANCE', 'BINANCE'), ('HARMONY', 'HARMONY'), ('FILECOIN', 'FILECOIN'), ('RSK', 'RSK'), ('XINFIN', 'XINFIN'), ('NERVOS', 'NERVOS'), ('ALGORAND', 'ALGORAND'), ('SIA', 'SIA'), ('TEZOS', 'TEZOS'), ('CASPER', 'CASPER'), ('OTHERS', 'OTHERS')], help_text='specific tenant type under the payout_type', max_length=10, null=True), + ), + migrations.AlterField( + model_name='profile', + name='trust_profile', + field=models.BooleanField(default=False, help_text='If this option is chosen, the user is able to submit a ens domain registration even if they are new to github'), + ), + ] From c94fa52e55f7f9f27494919dad84cc14ae1b79e0 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Tue, 24 Aug 2021 06:20:36 +0530 Subject: [PATCH 02/17] chore: allow GET request for CORS --- app/grants/views.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/app/grants/views.py b/app/grants/views.py index f4bb92c94ee..4316be60e56 100644 --- a/app/grants/views.py +++ b/app/grants/views.py @@ -3499,16 +3499,20 @@ def handle_ingestion(profile, network, identifier, do_write): @csrf_exempt def get_trust_bonus(request): ''' - JSON POST endpoint which returns the trust bonus score of given addresses + JSON POST/GET endpoint which returns the trust bonus score of given addresses ''' - try: - json_body = json.loads(request.body) - addresses = json_body.get('addresses') - if not addresses: - return HttpResponse(status=204) - except: - return HttpResponse(status=400) + addresses = request.GET.get('addresses') + if addresses: + addresses = addresses.split(',') + else: + try: + json_body = json.loads(request.body) + addresses = json_body.get('addresses') + if not addresses: + return HttpResponse(status=204) + except: + return HttpResponse(status=400) query = Q() for address in addresses: From 19c8fc6769a08ed76a8b5b4852c7e72c60948112 Mon Sep 17 00:00:00 2001 From: octavioamu Date: Wed, 25 Aug 2021 19:34:56 -0300 Subject: [PATCH 03/17] fix form failing if input have comma gitc-355 --- app/grants/templates/grants/new_match.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/grants/templates/grants/new_match.html b/app/grants/templates/grants/new_match.html index bb10f2b3281..3bc5f89a446 100644 --- a/app/grants/templates/grants/new_match.html +++ b/app/grants/templates/grants/new_match.html @@ -205,7 +205,7 @@
Grants to Fund
Required - +
[[errors.amount]] From 891b1c928d1e7881f97ffb7e25bbac924afab808 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Sun, 29 Aug 2021 21:36:25 +0530 Subject: [PATCH 04/17] Update views.py --- app/dashboard/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/dashboard/views.py b/app/dashboard/views.py index 0cec516ba7c..387c91d73c2 100644 --- a/app/dashboard/views.py +++ b/app/dashboard/views.py @@ -3242,12 +3242,14 @@ def verify_user_twitter(request, handle): last_tweet = api.user_timeline(screen_name=twitter_handle, count=1, tweet_mode="extended", include_rts=False, exclude_replies=False)[0] - except tweepy.TweepError: + except tweepy.TweepError as e: + logger.error(f"error: verify_user_twitter TweepError {e}") return JsonResponse({ 'ok': False, 'msg': f'Sorry, we couldn\'t get the last tweet from @{twitter_handle}' }) - except IndexError: + except IndexError as e: + logger.error(f"error: verify_user_twitter IndexError {e}") return JsonResponse({ 'ok': False, 'msg': 'Sorry, we couldn\'t retrieve the last tweet from your timeline' From 5382e9163da5a0e96d1f847c0fae4aa0605acbb6 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Tue, 31 Aug 2021 12:57:09 +0530 Subject: [PATCH 05/17] feat: sybil data input endpoint for round (#9371) * feat: sybil data input endpoint for round * stale nav (#9382) Co-authored-by: Kevin Owocki --- app/dashboard/templates/shared/menu.html | 6 +- app/grants/router.py | 2 +- app/grants/urls.py | 12 ++-- app/grants/views.py | 72 +++++++++++++++++++++++- 4 files changed, 81 insertions(+), 11 deletions(-) diff --git a/app/dashboard/templates/shared/menu.html b/app/dashboard/templates/shared/menu.html index 4266dab737c..fecb7e22cdc 100644 --- a/app/dashboard/templates/shared/menu.html +++ b/app/dashboard/templates/shared/menu.html @@ -139,7 +139,7 @@
- +
@@ -149,8 +149,8 @@
- {% trans "Govern" %} - {% trans "Decide the future of the open web" %} + {% trans "DAO" %} + {% trans "Get involved in the decentralized GitcoinDAO community." %}
diff --git a/app/grants/router.py b/app/grants/router.py index 72b2bb8438d..a3864e29129 100644 --- a/app/grants/router.py +++ b/app/grants/router.py @@ -1,6 +1,6 @@ from datetime import datetime, timedelta -from django.core.paginator import Paginator +from django.core.paginator import EmptyPage, Paginator import django_filters.rest_framework from ratelimit.decorators import ratelimit diff --git a/app/grants/urls.py b/app/grants/urls.py index 8f06fd35c85..77b6961912d 100644 --- a/app/grants/urls.py +++ b/app/grants/urls.py @@ -24,11 +24,11 @@ collection_thumbnail, contribute_to_grants_v1, contribution_addr_from_all_as_json, contribution_addr_from_grant_as_json, contribution_addr_from_grant_during_round_as_json, contribution_addr_from_round_as_json, contribution_info_from_grant_during_round_as_json, create_matching_pledge_v1, - flag, get_collection, get_collections_list, get_ethereum_cart_data, get_grant_payload, get_grants, - get_interrupted_contributions, get_replaced_tx, get_trust_bonus, grant_activity, grant_categories, grant_details, - grant_details_api, grant_details_contributions, grant_details_contributors, grant_edit, grant_fund, grant_new, - grants, grants_addr_as_json, grants_bulk_add, grants_by_grant_type, grants_cart_view, grants_info, grants_landing, - grants_type_redirect, ingest_contributions, ingest_contributions_view, invoice, leaderboard, + flag, get_clr_sybil_input, get_collection, get_collections_list, get_ethereum_cart_data, get_grant_payload, + get_grants, get_interrupted_contributions, get_replaced_tx, get_trust_bonus, grant_activity, grant_categories, + grant_details, grant_details_api, grant_details_contributions, grant_details_contributors, grant_edit, grant_fund, + grant_new, grants, grants_addr_as_json, grants_bulk_add, grants_by_grant_type, grants_cart_view, grants_info, + grants_landing, grants_type_redirect, ingest_contributions, ingest_contributions_view, invoice, leaderboard, manage_ethereum_cart_data, new_matching_partner, profile, quickstart, remove_grant_from_collection, save_collection, toggle_grant_favorite, verify_grant, ) @@ -114,5 +114,7 @@ path('v1/api/export_addresses/grant_round.json', contribution_addr_from_grant_during_round_as_json, name='contribution_addr_from_grant_during_round_as_json'), path('v1/api/export_info/grant_round.json', contribution_info_from_grant_during_round_as_json, name='contribution_addr_from_grant_during_round_as_json'), + # custom API + path('v1/api/get-clr-data/', get_clr_sybil_input, name='get_clr_sybil_input') ] diff --git a/app/grants/views.py b/app/grants/views.py index 4316be60e56..3df83e68219 100644 --- a/app/grants/views.py +++ b/app/grants/views.py @@ -23,7 +23,6 @@ import logging import math import re -import time import uuid from datetime import datetime from urllib.parse import urlencode @@ -38,6 +37,7 @@ from django.db import connection, transaction from django.db.models import Q, Subquery from django.http import Http404, HttpResponse, JsonResponse +from django.http.response import HttpResponseBadRequest, HttpResponseServerError from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse from django.templatetags.static import static @@ -67,6 +67,7 @@ from economy.models import Token as FTokens from economy.utils import convert_token_to_usdt from eth_account.messages import defunct_hash_message +from grants.clr import fetch_data from grants.models import ( CartActivity, Contribution, Flag, Grant, GrantAPIKey, GrantBrandingRoutingPolicy, GrantCategory, GrantCLR, GrantCollection, GrantType, MatchPledge, Subscription, @@ -81,7 +82,7 @@ from kudos.models import BulkTransferCoupon, Token from marketing.mails import grant_cancellation, new_grant_flag_admin from marketing.models import Keyword, Stat -from perftools.models import JSONStore +from perftools.models import JSONStore, StaticJsonEnv from ratelimit.decorators import ratelimit from retail.helpers import get_ip from townsquare.models import Announcement, Favorite, PinnedPost @@ -3496,6 +3497,73 @@ def handle_ingestion(profile, network, identifier, do_write): return JsonResponse({ 'success': True, 'ingestion_types': ingestion_types }) + +def get_clr_sybil_input(request, round_id): + ''' + This returns a paginated JSON response to return contributions + which are considered while calculating the QF match for a given CLR + ''' + token = request.headers['token'] + page = request.GET.get('page', 1) + + data = StaticJsonEnv.objects.get(key='BSCI_SYBIL_TOKEN').data + + if not round_id or not token or not data['token']: + return HttpResponseBadRequest("error: missing arguments") + + if token != data['token']: + return HttpResponseBadRequest("error: invalid token") + + clr = GrantCLR.objects.filter(pk=round_id).first() + if not clr: + return HttpResponseBadRequest("error: round not found") + + try: + limit = data['limit'] if data['limit'] else 100 + + # fetch grant contributions needed for round + __, all_clr_contributions = fetch_data(clr) + total_count = all_clr_contributions.count() + + # extract only needed fields + all_clr_contributions = list(all_clr_contributions.values( + 'created_on', 'profile_for_clr__handle', 'profile_for_clr_id', + 'match', 'normalized_data' + )) + + # paginate contributions + contributions = Paginator(all_clr_contributions, limit) + try: + contributions_queryset = contributions.page(page) + except EmptyPage: + response = { + 'metadata': { + 'count': 0, + 'current_page': 0, + 'num_pages': 0, + 'has_next': False + }, + 'contributions': [] + } + return HttpResponse(response) + + response = { + 'metadata': { + 'count': total_count, + 'current_page': page, + 'total_pages': contributions.num_pages, + 'has_next': contributions_queryset.has_next() + }, + 'contributions': contributions_queryset.object_list + } + + except Exception as e: + print(e) + return HttpResponseServerError() + + return JsonResponse(response) + + @csrf_exempt def get_trust_bonus(request): ''' From 7cbeef16a9a8c5421a81e49857147565766b8cc3 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Tue, 31 Aug 2021 15:16:35 +0100 Subject: [PATCH 06/17] Fixes grants explorer --- app/grants/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/grants/views.py b/app/grants/views.py index 3df83e68219..1edea62ad4d 100644 --- a/app/grants/views.py +++ b/app/grants/views.py @@ -23,6 +23,7 @@ import logging import math import re +import time import uuid from datetime import datetime from urllib.parse import urlencode @@ -1564,7 +1565,7 @@ def grant_edit(request, grant_id): kusama_payout_address == '0x0' and harmony_payout_address == '0x0' and binance_payout_address == '0x0' and - rsk_payout_address == '0x0' and + rsk_payout_address == '0x0' and algorand_payout_address == '0x0' ): response['message'] = 'error: payout_address is a mandatory parameter' From b4288b8e4b4cbb45cd028a92fb296b8b138ee706 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Wed, 1 Sep 2021 23:08:56 +0100 Subject: [PATCH 07/17] Adds method to allow cors on trust-bonus api --- app/app/utils.py | 8 ++++++++ app/grants/views.py | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/app/utils.py b/app/app/utils.py index 446afe0661a..4908a91df59 100644 --- a/app/app/utils.py +++ b/app/app/utils.py @@ -433,3 +433,11 @@ def notion_api_call(url='', payload=None): # return success as dict return response + + +def allow_all_origins(response): + """Pass in a response and add header to allow all CORs requests""" + + response["Access-Control-Allow-Origin"] = "*" + + return response diff --git a/app/grants/views.py b/app/grants/views.py index 1edea62ad4d..6fd943d98a4 100644 --- a/app/grants/views.py +++ b/app/grants/views.py @@ -58,7 +58,7 @@ EMAIL_ACCOUNT_VALIDATION, TWITTER_ACCESS_SECRET, TWITTER_ACCESS_TOKEN, TWITTER_CONSUMER_KEY, TWITTER_CONSUMER_SECRET, ) -from app.utils import get_profile +from app.utils import allow_all_origins, get_profile from bs4 import BeautifulSoup from cacheops import cached_view from dashboard.brightid_utils import get_brightid_status @@ -3579,9 +3579,9 @@ def get_trust_bonus(request): json_body = json.loads(request.body) addresses = json_body.get('addresses') if not addresses: - return HttpResponse(status=204) + return allow_all_origins(HttpResponse(status=204)) except: - return HttpResponse(status=400) + return allow_all_origins(HttpResponse(status=400)) query = Q() for address in addresses: @@ -3598,4 +3598,4 @@ def get_trust_bonus(request): }) _addrs.append(subscription.contributor_address) - return JsonResponse(response, safe=False) + return allow_all_origins(JsonResponse(response, safe=False)) From 6542cd02ace4c1f1cef8d19431521f8f026dbb7e Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Thu, 2 Sep 2021 21:08:12 +0530 Subject: [PATCH 08/17] remove twitter feed + slack post (#9424) --- app/dashboard/helpers.py | 6 +- app/dashboard/models.py | 4 +- app/dashboard/notifications.py | 192 ------------------ app/dashboard/tasks.py | 14 -- app/dashboard/utils.py | 3 - app/dashboard/views.py | 15 +- .../management/commands/hypercharge_mode.py | 9 +- .../management/commands/remarket_bounties.py | 47 ----- .../management/commands/share_activity.py | 60 ------ .../commands/test_remarket_tweet.py | 65 ------ scripts/crontab | 3 - 11 files changed, 5 insertions(+), 413 deletions(-) delete mode 100644 app/marketing/management/commands/remarket_bounties.py delete mode 100644 app/marketing/management/commands/share_activity.py delete mode 100644 app/marketing/tests/management/commands/test_remarket_tweet.py diff --git a/app/dashboard/helpers.py b/app/dashboard/helpers.py index b741ece2a0b..c8253b8f57b 100644 --- a/app/dashboard/helpers.py +++ b/app/dashboard/helpers.py @@ -36,7 +36,7 @@ HackathonEvent, UserAction, ) from dashboard.notifications import ( - maybe_market_to_email, maybe_market_to_github, maybe_market_to_slack, maybe_market_to_user_slack, + maybe_market_to_email, maybe_market_to_github, notify_of_lowball_bounty, ) from dashboard.tokens import addr_to_token @@ -960,8 +960,6 @@ def process_bounty_changes(old_bounty, new_bounty): # marketing if event_name != 'unknown_event': print("============ posting ==============") - did_post_to_slack = maybe_market_to_slack(new_bounty, event_name) - did_post_to_user_slack = maybe_market_to_user_slack(new_bounty, event_name) did_post_to_github = maybe_market_to_github(new_bounty, event_name, profile_pairs) did_post_to_email = maybe_market_to_email(new_bounty, event_name) print("============ done posting ==============") @@ -971,8 +969,6 @@ def process_bounty_changes(old_bounty, new_bounty): 'did_bsr': did_bsr, 'did_post_to_email': did_post_to_email, 'did_post_to_github': did_post_to_github, - 'did_post_to_slack': did_post_to_slack, - 'did_post_to_user_slack': did_post_to_user_slack, 'did_post_to_twitter': False, } diff --git a/app/dashboard/models.py b/app/dashboard/models.py index 010b8f60670..099dd633beb 100644 --- a/app/dashboard/models.py +++ b/app/dashboard/models.py @@ -71,7 +71,7 @@ from unidecode import unidecode from web3 import Web3 -from .notifications import maybe_market_to_github, maybe_market_to_slack, maybe_market_to_user_slack +from .notifications import maybe_market_to_github from .signals import m2m_changed_interested logger = logging.getLogger(__name__) @@ -2195,8 +2195,6 @@ def auto_user_approve(interest, bounty): interest.acceptance_date = timezone.now() start_work_approved(interest, bounty) maybe_market_to_github(bounty, 'work_started', profile_pairs=bounty.profile_pairs) - maybe_market_to_slack(bounty, 'worker_approved') - maybe_market_to_user_slack(bounty, 'worker_approved') @receiver(post_save, sender=Interest, dispatch_uid="psave_interest") diff --git a/app/dashboard/notifications.py b/app/dashboard/notifications.py index 2fc612758c8..0042119ee44 100644 --- a/app/dashboard/notifications.py +++ b/app/dashboard/notifications.py @@ -83,148 +83,6 @@ def github_org_to_twitter_tags(github_org): return twitter_tags -def maybe_market_to_twitter(bounty, event_name): - """Tweet the specified Bounty event. - - Args: - bounty (dashboard.models.Bounty): The Bounty to be marketed. - event_name (str): The name of the event. - - Returns: - bool: Whether or not the twitter notification was sent successfully. - - """ - if not bounty.is_notification_eligible(var_to_check=settings.TWITTER_CONSUMER_KEY): - return False - - api = twitter.Api( - consumer_key=settings.TWITTER_CONSUMER_KEY, - consumer_secret=settings.TWITTER_CONSUMER_SECRET, - access_token_key=settings.TWITTER_ACCESS_TOKEN, - access_token_secret=settings.TWITTER_ACCESS_SECRET, - ) - tweet_txts = [ - "Earn {} {} {} now by completing this task: \n\n{}", - "Oppy to earn {} {} {} for completing this task: \n\n{}", - "Is today the day you (a) boost your OSS rep (b) make some extra cash? 🤔 {} {} {} \n\n{}", - ] - if event_name == 'remarket_bounty': - tweet_txts = tweet_txts + [ - "Gitcoin open task of the day is worth {} {} {} ⚡️ \n\n{}", - "Task of the day 💰 {} {} {} ⚡️ \n\n{}", - ] - elif event_name == 'new_bounty': - tweet_txts = tweet_txts + [ - "Extra! Extra 🗞🗞 New Funded Issue, Read all about it 👇 {} {} {} \n\n{}", - "Hot off the blockchain! 🔥🔥🔥 There's a new task worth {} {} {} \n\n{}", - "💰 New Task Alert.. 💰 Earn {} {} {} for working on this 👇 \n\n{}", - ] - elif event_name == 'increased_bounty': - tweet_txts = [ - 'Increased Payout on {} {} {}\n{}' - ] - elif event_name == 'start_work': - tweet_txts = [ - 'Work started on {} {} {}\n{}' - ] - elif event_name == 'stop_work': - tweet_txts = [ - 'Work stopped on {} {} {}\n{}' - ] - elif event_name == 'work_done': - tweet_txts = [ - 'Work done on {} {} {}\n{}' - ] - elif event_name == 'work_submitted': - tweet_txts = [ - 'Work submitted on {} {} {}\n{}' - ] - elif event_name == 'killed_bounty': - tweet_txts = [ - 'Bounty killed on {} {} {}\n{}' - ] - elif event_name == 'worker_rejected': - tweet_txts = [ - 'Worked rejected on {} {} {}\n{}' - ] - elif event_name == 'worker_approved': - tweet_txts = [ - 'Worked approved on {} {} {}\n{}' - ] - - random.shuffle(tweet_txts) - tweet_txt = tweet_txts[0] - utm = '' - if bounty.metadata.get('hyper_tweet_counter', False): - utm = f'utm_source=hypercharge-auto&utm_medium=twitter&utm_campaign={bounty.title}' - - url = f'{bounty.get_absolute_url()}?{utm}' - is_short = False - for shortener in ['Tinyurl', 'Adfly', 'Isgd', 'QrCx']: - try: - if not is_short: - shortener = Shortener(shortener) - response = shortener.short(url) - if response != 'Error' and 'http' in response: - url = response - is_short = True - except Exception: - pass - - new_tweet = tweet_txt.format( - round(bounty.get_natural_value(), 4), - bounty.token_name, - f"({bounty.value_in_usdt_now} USD @ ${round(convert_token_to_usdt(bounty.token_name),2)}/{bounty.token_name})" if bounty.value_in_usdt_now else "", - url - ) - new_tweet = new_tweet + " " + github_org_to_twitter_tags(bounty.org_name) # twitter tags - if bounty.keywords: # hashtags - for keyword in bounty.keywords.split(','): - _new_tweet = new_tweet + " #" + str(keyword).lower().strip() - if len(_new_tweet) < 140: - new_tweet = _new_tweet - - try: - api.PostUpdate(new_tweet) - except Exception as e: - print(e) - return False - return True - - -def maybe_market_to_slack(bounty, event_name): - """Send a Slack message for the specified Bounty. - - Args: - bounty (dashboard.models.Bounty): The Bounty to be marketed. - event_name (str): The name of the event. - - Returns: - bool: Whether or not the Slack notification was sent successfully. - - """ - if not bounty.is_notification_eligible(var_to_check=settings.SLACK_TOKEN): - return False - - msg = build_message_for_integration(bounty, event_name) - if not msg: - return False - - try: - channel = 'notif-gitcoin' - sc = SlackClient(settings.SLACK_TOKEN) - sc.api_call( - "chat.postMessage", - channel=channel, - text=msg, - icon_url=settings.GITCOIN_SLACK_ICON_URL, - ) - except Exception as e: - print(e) - return False - return True - - def build_message_for_integration(bounty, event_name): """Build message to be posted to integrated service (e.g. slack). @@ -257,56 +115,6 @@ def build_message_for_integration(bounty, event_name): return msg -def maybe_market_to_user_slack(bounty, event_name): - from dashboard.tasks import maybe_market_to_user_slack - maybe_market_to_user_slack.delay(bounty.pk, event_name) - - -def maybe_market_to_user_slack_helper(bounty, event_name): - """Send a Slack message to the user's slack channel for the specified Bounty. - - Args: - bounty (dashboard.models.Bounty): The Bounty to be marketed. - event_name (str): The name of the event. - - Returns: - bool: Whether or not the Slack notification was sent successfully. - - """ - from dashboard.models import Profile - if bounty.get_natural_value() < 0.0001: - return False - if bounty.network != settings.ENABLE_NOTIFICATIONS_ON_NETWORK: - return False - - msg = build_message_for_integration(bounty, event_name) - if not msg: - return False - - url = bounty.github_url - sent = False - try: - repo = org_name(url) + '/' + repo_name(url) - subscribers = Profile.objects.filter(slack_repos__contains=[repo]) - subscribers = subscribers & Profile.objects.exclude(slack_token='', slack_channel='') - for subscriber in subscribers: - try: - sc = SlackClient(subscriber.slack_token) - sc.api_call( - "chat.postMessage", - channel=subscriber.slack_channel, - text=msg, - icon_url=settings.GITCOIN_SLACK_ICON_URL, - ) - sent = True - except Exception as e: - print(e) - except Exception as e: - print(e) - - return sent - - def maybe_market_tip_to_email(tip, emails): """Send an email for the specified Tip. diff --git a/app/dashboard/tasks.py b/app/dashboard/tasks.py index 3cd4ffe69b5..33870f9b41b 100644 --- a/app/dashboard/tasks.py +++ b/app/dashboard/tasks.py @@ -261,20 +261,6 @@ def update_trust_bonus(self, pk): profile.save() -@app.shared_task(bind=True) -def maybe_market_to_user_slack(self, bounty_pk, event_name, retry: bool = True) -> None: - """ - :param self: - :param bounty_pk: - :param event_name: - :return: - """ - with redis.lock("maybe_market_to_user_slack:bounty", timeout=LOCK_TIMEOUT): - bounty = Bounty.objects.get(pk=bounty_pk) - from dashboard.notifications import maybe_market_to_user_slack_helper - maybe_market_to_user_slack_helper(bounty, event_name) - - @app.shared_task(bind=True, soft_time_limit=600, time_limit=660, max_retries=3) def grant_update_email_task(self, pk, retry: bool = True) -> None: """ diff --git a/app/dashboard/utils.py b/app/dashboard/utils.py index 24cdd554203..ee435439ffa 100644 --- a/app/dashboard/utils.py +++ b/app/dashboard/utils.py @@ -65,7 +65,6 @@ from web3.middleware import geth_poa_middleware from .abi import erc20_abi -from .notifications import maybe_market_to_slack logger = logging.getLogger(__name__) @@ -1054,8 +1053,6 @@ def re_market_bounty(bounty, auto_save = True): if auto_save: bounty.save() - maybe_market_to_slack(bounty, 'issue_remarketed') - result_msg = 'The issue will appear at the top of the issue explorer. ' further_permitted_remarket_count = settings.RE_MARKET_LIMIT - bounty.remarketed_count if further_permitted_remarket_count >= 1: diff --git a/app/dashboard/views.py b/app/dashboard/views.py index 387c91d73c2..ddc3dfa315a 100644 --- a/app/dashboard/views.py +++ b/app/dashboard/views.py @@ -128,9 +128,7 @@ MediaFile, Option, Poll, PortfolioItem, Profile, ProfileSerializer, ProfileVerification, ProfileView, Question, SearchHistory, Sponsor, Tool, TribeMember, UserAction, UserVerificationModel, ) -from .notifications import ( - maybe_market_to_email, maybe_market_to_github, maybe_market_to_slack, maybe_market_to_user_slack, -) +from .notifications import maybe_market_to_email, maybe_market_to_github from .poh_utils import is_registered_on_poh from .router import HackathonEventSerializer, TribesSerializer from .utils import ( @@ -284,8 +282,6 @@ def create_new_interest_helper(bounty, user, issue_message): record_bounty_activity(bounty, user, 'start_work' if not approval_required else 'worker_applied', interest=interest) bounty.interested.add(interest) record_user_action(user, 'start_work', interest) - maybe_market_to_slack(bounty, 'start_work' if not approval_required else 'worker_applied') - maybe_market_to_user_slack(bounty, 'start_work' if not approval_required else 'worker_applied') return interest @@ -614,8 +610,6 @@ def remove_interest(request, bounty_id): bounty.interested.remove(interest) interest.delete() - maybe_market_to_slack(bounty, 'stop_work') - maybe_market_to_user_slack(bounty, 'stop_work') except Interest.DoesNotExist: return JsonResponse({ 'errors': [_('You haven\'t expressed interest on this bounty.')], @@ -768,8 +762,6 @@ def uninterested(request, bounty_id, profile_id): try: interest = Interest.objects.get(profile_id=profile_id, bounty=bounty) bounty.interested.remove(interest) - maybe_market_to_slack(bounty, 'stop_work') - maybe_market_to_user_slack(bounty, 'stop_work') if is_staff or is_moderator: event_name = "bounty_removed_slashed_by_staff" if slashed else "bounty_removed_by_staff" else: @@ -1846,9 +1838,7 @@ def helper_handle_approvals(request, bounty): start_work_approved(interest, bounty) maybe_market_to_github(bounty, 'work_started', profile_pairs=bounty.profile_pairs) - maybe_market_to_slack(bounty, 'worker_approved') record_bounty_activity(bounty, request.user, 'worker_approved', interest) - maybe_market_to_user_slack(bounty, 'worker_approved') else: start_work_rejected(interest, bounty) @@ -1856,9 +1846,6 @@ def helper_handle_approvals(request, bounty): bounty.interested.remove(interest) interest.delete() - maybe_market_to_slack(bounty, 'worker_rejected') - maybe_market_to_user_slack(bounty, 'worker_rejected') - messages.success(request, _(f'{worker} has been {mutate_worker_action_past_tense}')) else: messages.warning(request, _('Only the funder of this bounty may perform this action.')) diff --git a/app/marketing/management/commands/hypercharge_mode.py b/app/marketing/management/commands/hypercharge_mode.py index 88ccbe7b17d..af111092b93 100644 --- a/app/marketing/management/commands/hypercharge_mode.py +++ b/app/marketing/management/commands/hypercharge_mode.py @@ -25,7 +25,6 @@ from django.utils import timezone from dashboard.models import Bounty, BountyEvent, Profile -from dashboard.notifications import maybe_market_to_twitter from marketing.mails import bounty_hypercharged from townsquare.models import Offer @@ -75,19 +74,15 @@ def handle(self, *args, **options): offer_title = f'Work on "{bounty.title}" and receive {floatformat(bounty.value_true)} {bounty.token_name}' offer_desc = '' - event_name = '' counter = bounty.metadata['hyper_tweet_counter'] + # counter is 0 -> new_bounty + # counter is 1 -> remarket_bounty if counter == 0: - event_name = 'new_bounty' notify_previous_workers(bounty) make_secret_offer(profile, offer_title, offer_desc, bounty) - elif counter == 1: - event_name = 'remarket_bounty' elif counter % 2 == 0: make_secret_offer(profile, offer_title, offer_desc, bounty) bounty.metadata['hyper_tweet_counter'] += 1 bounty.hyper_next_publication = now + timedelta(hours=12) bounty.save() - - maybe_market_to_twitter(bounty, event_name) diff --git a/app/marketing/management/commands/remarket_bounties.py b/app/marketing/management/commands/remarket_bounties.py deleted file mode 100644 index da56e07c9e5..00000000000 --- a/app/marketing/management/commands/remarket_bounties.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -#!/usr/bin/env python3 -''' - Copyright (C) 2021 Gitcoin Core - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -''' -from django.core.management.base import BaseCommand - -from dashboard.models import Bounty -from dashboard.notifications import maybe_market_to_slack, maybe_market_to_twitter - - -class Command(BaseCommand): - - help = 'sends bounties quotes to twitter' - - def handle(self, *args, **options): - bounties = Bounty.objects.current().filter( - network='mainnet', - idx_status='open') - if bounties.count() < 3: - print('count is only {}. exiting'.format(bounties.count())) - return - bounty = bounties.order_by('?').first() - - remarket_bounties = bounties.filter(admin_mark_as_remarket_ready=True) - if remarket_bounties.exists(): - bounty = remarket_bounties.order_by('?').first() - - print(bounty) - did_tweet = maybe_market_to_twitter(bounty, 'remarket_bounty') - did_slack = maybe_market_to_slack(bounty, 'this funded issue could use some action!') - print("did tweet", did_tweet) - print("did slack", did_slack) diff --git a/app/marketing/management/commands/share_activity.py b/app/marketing/management/commands/share_activity.py deleted file mode 100644 index 871dece87d5..00000000000 --- a/app/marketing/management/commands/share_activity.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -#!/usr/bin/env python3 -''' - Copyright (C) 2021 Gitcoin Core - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -''' -from django.conf import settings -from django.core.management.base import BaseCommand -from django.utils import timezone - -import twitter -from dashboard.models import Activity - - -class Command(BaseCommand): - - help = 'sends activity feed to twitter' - - def handle(self, *args, **options): - - api = twitter.Api( - consumer_key=settings.TWITTER_CONSUMER_KEY, - consumer_secret=settings.TWITTER_CONSUMER_SECRET, - access_token_key=settings.TWITTER_ACCESS_TOKEN, - access_token_secret=settings.TWITTER_ACCESS_SECRET, - ) - - created_before = (timezone.now() - timezone.timedelta(minutes=1)) - if settings.DEBUG: - created_before = (timezone.now() - timezone.timedelta(days=20)) - activities = Activity.objects.filter(created_on__gt=created_before) - print(f" got {activities.count()} activities") - for activity in activities: - - txt = activity.text - - txt = f". {txt} {activity.action_url}" - - if 'made an update to' in txt: - continue - - try: - print(txt) - new_tweet = txt - api.PostUpdate(new_tweet) - except Exception as e: - print(e) diff --git a/app/marketing/tests/management/commands/test_remarket_tweet.py b/app/marketing/tests/management/commands/test_remarket_tweet.py deleted file mode 100644 index 75e1f0dc3a7..00000000000 --- a/app/marketing/tests/management/commands/test_remarket_tweet.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -"""Handle marketing commands related tests. - -Copyright (C) 2021 Gitcoin Core - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published -by the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . - -""" -from unittest.mock import patch - -from marketing.management.commands.remarket_bounties import Command -from test_plus.test import TestCase - - -class TestRemarketTweet(TestCase): - """Define tests for remarket tweet.""" - - @patch('marketing.management.commands.remarket_bounties.maybe_market_to_twitter') - def test_handle_no_bounties(self, mock_func): - """Test command remarket tweet with no bounties.""" - Command().handle() - - assert mock_func.call_count == 0 - -#TODO: uncomment when remarket_tweet logic will be activated -# @patch('marketing.management.commands.remarket_tweet.maybe_market_to_twitter') -# def test_handle_with_bounties(self, mock_func): -# """Test command remarket tweet with bounties.""" -# for i in range(3): -# bounty = Bounty.objects.create( -# title='foo', -# value_in_token=3, -# token_name='USDT', -# web3_created=datetime(2008, 10, 31), -# github_url='https://github.com/gitcoinco/web', -# token_address='0x0', -# issue_description='hello world', -# bounty_owner_github_username='flintstone', -# is_open=True, -# accepted=True, -# expires_date=timezone.now() + timedelta(days=1, hours=1), -# idx_project_length=5, -# project_length='Months', -# bounty_type='Feature', -# experience_level='Intermediate', -# raw_data={}, -# idx_status='open', -# bounty_owner_email='john@bar.com', -# current_bounty=True, -# network='mainnet' -# ) -# Command().handle() -# -# assert mock_func.call_count == 1 diff --git a/scripts/crontab b/scripts/crontab index 1ec1dee1f79..6f5bb08ac50 100644 --- a/scripts/crontab +++ b/scripts/crontab @@ -70,9 +70,6 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/us 30 */6 * * * cd gitcoin/coin; bash scripts/run_management_command_if_not_already_running.bash process_pending_kudos_distributions 20 20 2 0 >> /var/log/gitcoin/process_pending_kudos_distributions.log 2>&1 0 0 * * * cd gitcoin/coin; bash scripts/run_management_command_if_not_already_running.bash create_offer_if_none_exists >> /var/log/gitcoin/create_offer_if_none_exists.log 2>&1 -* * * * * cd gitcoin/coin; bash scripts/run_management_command_if_not_already_running.bash share_activity >> /var/log/gitcoin/share_activity.log 2>&1 -35 14 * * 1,6 cd gitcoin/coin; bash scripts/run_management_command.bash remarket_bounties >> /var/log/gitcoin/remarket_bounties.log 2>&1 -35 11 * * 0,4 cd gitcoin/coin; bash scripts/run_management_command.bash remarket_bounties >> /var/log/gitcoin/remarket_bounties.log 2>&1 45 10 * * * cd gitcoin/coin; bash scripts/run_management_command.bash expiration >> /var/log/gitcoin/expiration_bounty.log 2>&1 15 10 * * * cd gitcoin/coin; bash scripts/run_management_command.bash expiration_start_work >> /var/log/gitcoin/expiration_start_work.log 2>&1 15 1 * * * cd gitcoin/coin; bash scripts/run_management_command.bash sync_keywords >> /var/log/gitcoin/sync_keywords.log 2>&1 From 9b9e8276d0d9fffeb8880eac0842094a0b2475c3 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Mon, 6 Sep 2021 16:19:41 +0530 Subject: [PATCH 09/17] remove remarket_bounties --- pydocmd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/pydocmd.yml b/pydocmd.yml index 192d27ed1e8..f40d9dd2e66 100644 --- a/pydocmd.yml +++ b/pydocmd.yml @@ -146,7 +146,6 @@ generate: - marketing.management.commands.process_email_events++ - marketing.management.commands.pull_github++ - marketing.management.commands.pull_stats++ - - marketing.management.commands.remarket_bounties++ - marketing.management.commands.roundup++ - marketing.management.commands.send_quarterly_stats++ - marketing.management.commands.sync_github++ From 30cb469aec59244549323a4315447e7d207c31d3 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Mon, 6 Sep 2021 16:29:55 +0530 Subject: [PATCH 10/17] feat: toggle sybil/non sybil users (#9431) --- app/dashboard/helpers.py | 5 +--- app/grants/urls.py | 6 ++-- app/grants/views.py | 60 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/app/dashboard/helpers.py b/app/dashboard/helpers.py index c8253b8f57b..5d2fc61e85e 100644 --- a/app/dashboard/helpers.py +++ b/app/dashboard/helpers.py @@ -35,10 +35,7 @@ Activity, BlockedURLFilter, Bounty, BountyEvent, BountyFulfillment, BountyInvites, BountySyncRequest, Coupon, HackathonEvent, UserAction, ) -from dashboard.notifications import ( - maybe_market_to_email, maybe_market_to_github, - notify_of_lowball_bounty, -) +from dashboard.notifications import maybe_market_to_email, maybe_market_to_github, notify_of_lowball_bounty from dashboard.tokens import addr_to_token from economy.utils import convert_amount from git.utils import get_issue_details, get_url_dict, org_name diff --git a/app/grants/urls.py b/app/grants/urls.py index 77b6961912d..63a255c5a4c 100644 --- a/app/grants/urls.py +++ b/app/grants/urls.py @@ -30,7 +30,7 @@ grant_new, grants, grants_addr_as_json, grants_bulk_add, grants_by_grant_type, grants_cart_view, grants_info, grants_landing, grants_type_redirect, ingest_contributions, ingest_contributions_view, invoice, leaderboard, manage_ethereum_cart_data, new_matching_partner, profile, quickstart, remove_grant_from_collection, save_collection, - toggle_grant_favorite, verify_grant, + toggle_grant_favorite, toggle_user_sybil, verify_grant, ) app_name = 'grants/' @@ -115,6 +115,6 @@ path('v1/api/export_info/grant_round.json', contribution_info_from_grant_during_round_as_json, name='contribution_addr_from_grant_during_round_as_json'), # custom API - path('v1/api/get-clr-data/', get_clr_sybil_input, name='get_clr_sybil_input') - + path('v1/api/get-clr-data/', get_clr_sybil_input, name='get_clr_sybil_input'), + path('v1/api/toggle_user_sybil', toggle_user_sybil, name='toggle_user_sybil') ] diff --git a/app/grants/views.py b/app/grants/views.py index 6fd943d98a4..3247819f272 100644 --- a/app/grants/views.py +++ b/app/grants/views.py @@ -86,7 +86,7 @@ from perftools.models import JSONStore, StaticJsonEnv from ratelimit.decorators import ratelimit from retail.helpers import get_ip -from townsquare.models import Announcement, Favorite, PinnedPost +from townsquare.models import Announcement, Favorite, PinnedPost, SquelchProfile from townsquare.utils import can_pin from web3 import HTTPProvider, Web3 @@ -3599,3 +3599,61 @@ def get_trust_bonus(request): _addrs.append(subscription.contributor_address) return allow_all_origins(JsonResponse(response, safe=False)) + + +def toggle_user_sybil(request): + ''' + POST endpoint which allows to mark a list of users as sybil + or remove them the sybil tag from them. + This is intended to be used by BSCI to ensure they can toggle it + every 12 hours based on their findings as opposed to having it done + at the end. + ''' + + json_body = json.loads(request.body) + + token = request.headers['token'] + sybil_users = json_body.get('sybil_users') + non_sybil_users = json_body.get('non_sybil_users') + + data = StaticJsonEnv.objects.get(key='BSCI_SYBIL_TOKEN').data + + if (not sybil_users and not non_sybil_users ) or not token or not data['token']: + return HttpResponseBadRequest("error: missing arguments") + + if token != data['token']: + return HttpResponseBadRequest("error: invalid token") + + squelched_profiles = SquelchProfile.objects.all() + + if sybil_users: + # iterate through users which need to be packed as sybil + for user in sybil_users: + try: + # get user profile + profile = Profile.objects.get(pk=user.get('id')) + + # check if user has entry in SquelchProfile + if not squelched_profiles.filter(profile=profile).first(): + # mark user as sybil + SquelchProfile.objects.create( + profile=profile, + comments=f"sybil: marked by bsci - {user.get('comment')}" + ) + except Exception as e: + print(f"error: unable to mark user ${user.get('id')} as sybil. {e}") + + + if non_sybil_users: + # iterate and remove sybil from user + for user in non_sybil_users: + try: + profile = Profile.objects.get(pk=user.get('id')) + print(squelched_profiles.filter(profile=profile)) + squelched_profiles.filter(profile=profile).delete() + print(squelched_profiles.filter(profile=profile)) + + except Exception as e: + print(f"error: unable to mark ${user.get('id')} as non sybil. {e}") + + return JsonResponse({'success': 'ok'}, status=200) From e943545199dbb6d2c286f6efb06cfaa1f7984cd9 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Mon, 6 Sep 2021 12:53:04 +0100 Subject: [PATCH 11/17] Refactor clr3 (#9432) * Refactors clr3.py to reduce time complexity * Restore doc block --- app/grants/clr3.py | 162 ++++++++++-------- .../management/commands/analytics_clr3.py | 12 +- app/grants/tasks.py | 10 ++ 3 files changed, 105 insertions(+), 79 deletions(-) diff --git a/app/grants/clr3.py b/app/grants/clr3.py index 27d821beebe..7493bada5bd 100644 --- a/app/grants/clr3.py +++ b/app/grants/clr3.py @@ -24,10 +24,9 @@ import numpy as np from grants.models import Contribution, Grant, GrantCollection +from grants.tasks import save_clr_prediction_curve from townsquare.models import SquelchProfile -CLR_PERCENTAGE_DISTRIBUTED = 0 - def fetch_grants(clr_round, network='mainnet'): grant_filters = clr_round.grant_filters @@ -110,28 +109,28 @@ def get_totals_by_pair(contrib_dict): {user_id (str): {user_id (str): pair_total (float)}} ''' - tot_overlap = {} + pair_totals = {} # start pairwise match for _, contribz in contrib_dict.items(): for k1, v1 in contribz.items(): - if k1 not in tot_overlap: - tot_overlap[k1] = {} + if k1 not in pair_totals: + pair_totals[k1] = {} # pairwise matches to current round for k2, v2 in contribz.items(): - if k2 not in tot_overlap[k1]: - tot_overlap[k1][k2] = 0 - tot_overlap[k1][k2] += (v1 * v2) ** 0.5 + if k2 not in pair_totals[k1]: + pair_totals[k1][k2] = 0 + pair_totals[k1][k2] += (v1 * v2) ** 0.5 - return tot_overlap + return pair_totals -def calculate_clr(aggregated_contributions, pair_totals, trust_dict, v_threshold, total_pot): +def calculate_clr(curr_agg, pair_totals, trust_dict, v_threshold, total_pot): ''' calculates the clr amount at the given threshold and total pot args: - aggregated contributions by pair nested dict + curr_agg { grant_id (str): { user_id (str): aggregated_amount (float) @@ -145,18 +144,16 @@ def calculate_clr(aggregated_contributions, pair_totals, trust_dict, v_threshold float total_pot float - returns: total clr award by grant, analytics, normalized by the normalization factor [{'id': proj, 'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}] saturation point boolean ''' - bigtot = 0 totals = {} - for proj, contribz in aggregated_contributions.items(): + for proj, contribz in curr_agg.items(): tot = 0 _num = 0 _sum = 0 @@ -177,59 +174,23 @@ def calculate_clr(aggregated_contributions, pair_totals, trust_dict, v_threshold bigtot += tot totals[proj] = {'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot} - global CLR_PERCENTAGE_DISTRIBUTED - - if bigtot >= total_pot: # saturation reached - # print(f'saturation reached. Total Pot: ${total_pot} | Total Allocated ${bigtot}. Normalizing') - CLR_PERCENTAGE_DISTRIBUTED = 100 - for key, t in totals.items(): - t['clr_amount'] = ((t['clr_amount'] / bigtot) * total_pot) - else: - CLR_PERCENTAGE_DISTRIBUTED = (bigtot / total_pot) * 100 - if bigtot == 0: - bigtot = 1 - percentage_increase = np.log(total_pot / bigtot) / 100 - for key, t in totals.items(): - t['clr_amount'] = t['clr_amount'] * (1 + percentage_increase) - return totals + return bigtot, totals -def run_clr_calcs(curr_agg, trust_dict, v_threshold, total_pot): +def calculate_clr_for_prediction(bigtot, totals, curr_agg, trust_dict, v_threshold, total_pot, grant_id, amount): ''' - clubbed function that runs all calculation functions + clubbed function that runs all calculation functions and returns the result for a single grant_id args: - curr_agg : + bigtot : float + totals : { grantId (int): { - profileId (str): amount (float) + number_contributions (int), + contribution_amount (float), + clr_amount (float), } } - trust_dict : - { - profileId (str): trust_bonus (float) - } - v_threshold : float - total_pot : float - - returns: - grants clr award amounts (dict) - ''' - - # get pair totals - ptots = get_totals_by_pair(curr_agg) - - # clr calcluation - totals = calculate_clr(curr_agg, ptots, trust_dict, v_threshold, total_pot) - - return totals - - -def calculate_clr_for_donation(curr_agg, trust_dict, grant_id, amount, v_threshold, total_pot): - ''' - clubbed function that runs all calculation functions and returns the result for a single grant_id - - args: curr_agg : { grantId (int): { @@ -240,10 +201,10 @@ def calculate_clr_for_donation(curr_agg, trust_dict, grant_id, amount, v_thresho { profileId (str): trust_bonus (float) } - grant_id ; int - amount ; int v_threshold : float total_pot : float + grant_id ; int + amount ; int returns: (grant clr award amounts (dict), clr_amount (float), number_contributions (int), contribution_amount (float)) @@ -253,10 +214,40 @@ def calculate_clr_for_donation(curr_agg, trust_dict, grant_id, amount, v_thresho if curr_agg.get(grant_id) or not grant_id: # find grant in contributions list and add donation if amount: - trust_dict['999999999999'] = 1 - curr_agg[grant_id]['999999999999'] = amount + # set predictions against this user + dummy_user = '999999999999' + # take a copy to isolate changes + totals = copy.deepcopy(totals) + + # add prediction amount + curr_agg[grant_id][dummy_user] = amount + + # get the contributions for this grant only + contribz = curr_agg.get(grant_id) - grants_clr = run_clr_calcs(curr_agg, trust_dict, v_threshold, total_pot) + # unpack state from the selected grants totals entry + total = totals.get(grant_id, {}) + + tot = total.get('clr_amount', 0) + _num = total.get('number_contributions', 0) + 1 + _sum = total.get('contribution_amount', 0) + amount + + # remove old total from bigtot + bigtot -= tot + + # start pairwise matches + for k2, v2 in contribz.items(): + # pairwise matches to current round + if int(dummy_user) > int(k2): + tot += ((amount * v2) ** 0.5) / ((amount * v2) ** 0.5 / (v_threshold * max(trust_dict[k2], 1)) + 1) + + if type(tot) == complex: + tot = float(tot.real) + + totals[grant_id] = {'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot} + + # normalise the result + grants_clr = normalise(bigtot + tot, totals, total_pot) # find grant we added the contribution to and get the new clr amount if grants_clr.get(grant_id): @@ -272,6 +263,22 @@ def calculate_clr_for_donation(curr_agg, trust_dict, grant_id, amount, v_thresho # print(f'info: no contributions found for grant {grant}') return (grants_clr, 0.0, 0, 0.0) + + +def normalise(bigtot, totals, total_pot): + # check for saturation and normalise if reached + if bigtot >= total_pot: + # print(f'saturation reached. Total Pot: ${total_pot} | Total Allocated ${bigtot}. Normalizing') + for key, t in totals.items(): + t['clr_amount'] = ((t['clr_amount'] / bigtot) * total_pot) + else: + if bigtot == 0: + bigtot = 1 + percentage_increase = np.log(total_pot / bigtot) / 100 + for key, t in totals.items(): + t['clr_amount'] = t['clr_amount'] * (1 + percentage_increase) + + return totals def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainnet', only_grant_pk=None, what='full'): @@ -286,15 +293,15 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn v_threshold = float(clr_round.verified_threshold) print(f"- starting fetch_grants at {round(time.time(),1)}") - # grants, contributions = fetch_data(clr_round, network) grants = fetch_grants(clr_round, network) - if only_grant_pk: - grants = grants.filter(pk=only_grant_pk) - print(f"- starting get data and sum at {round(time.time(),1)}") # collect contributions for clr_round into temp table initial_query = get_summed_contribs_query(grants, clr_round.start_date, clr_round.end_date, clr_round.contribution_multiplier, network) + + if only_grant_pk: + grants = grants.filter(pk=40) + # open cursor and execute the groupBy sum for the round with connection.cursor() as cursor: counter = 0 @@ -313,8 +320,13 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n") return - print(f"- starting current grant calc (free of predictions) at {round(time.time(),1)}") - curr_grants_clr = run_clr_calcs(curr_agg, trust_dict, v_threshold, total_pot) + print(f"- starting current distributions calc at {round(time.time(),1)}") + # aggregate pairs and run calculation to get current distribution + pair_totals = get_totals_by_pair(curr_agg) + bigtot, totals = calculate_clr(curr_agg, pair_totals, trust_dict, v_threshold, total_pot) + + # normalise against a deepcopy of the totals to avoid mutations + curr_grants_clr = normalise(bigtot, copy.deepcopy(totals), total_pot) if what == 'slim': # if we are only calculating slim CLR calculations, return here and save 97% compute power @@ -354,7 +366,7 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn if counter % 10 == 0 or True: print(f"- {counter}/{total_count} grants iter, pk:{grant.id}, at {round(time.time(),1)}") - # if no contributions have been made for this grant then the pairwise will fail and no match will be discovered + # if no contributions have been made for this grant then the pairwise will fail and there will be no match for this grant if not curr_agg.get(grant.id): grants_clr = None potential_clr = [0.0 for x in range(0, 6)] @@ -373,9 +385,9 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn grants_clr = None predicted_clr = 0.0 else: - # calculate clr with each additional donation - grants_clr, predicted_clr, _, _ = calculate_clr_for_donation( - curr_agg, trust_dict, grant.id, amount, v_threshold, total_pot + # calculate clr with additional donation amount + grants_clr, predicted_clr, _, _ = calculate_clr_for_prediction( + bigtot, totals, curr_agg, trust_dict, v_threshold, total_pot, grant.id, amount ) # reset potential_donations @@ -399,11 +411,11 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn clr_prediction_curve = [[ele[0], ele[1], ele[1] - base] for ele in clr_prediction_curve ] else: clr_prediction_curve = [[0.0, 0.0, 0.0] for x in range(0, 6)] + print(clr_prediction_curve) # save the new predicition curve via the model - clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve) - _grant.save() + save_clr_prediction_curve.delay(grant.id, clr_prediction_curve) debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr}) diff --git a/app/grants/management/commands/analytics_clr3.py b/app/grants/management/commands/analytics_clr3.py index 6bc715e633e..5175b1975f8 100644 --- a/app/grants/management/commands/analytics_clr3.py +++ b/app/grants/management/commands/analytics_clr3.py @@ -22,7 +22,7 @@ from django.db import connection from django.utils import timezone -from grants.clr3 import calculate_clr_for_donation, fetch_grants, get_summed_contribs_query +from grants.clr3 import calculate_clr, fetch_grants, get_summed_contribs_query, get_totals_by_pair, normalise from grants.models import GrantCLR @@ -55,11 +55,15 @@ def analytics_clr(from_date=None, clr_round=None, network='mainnet'): trust_dict[_row[1]] = _row[3] curr_agg[_row[0]][_row[1]] = _row[2] + ptots = get_totals_by_pair(curr_agg) + bigtot, totals = calculate_clr(curr_agg, ptots, trust_dict, v_threshold, total_pot) + + # normalise against a deepcopy of the totals to avoid mutations + curr_grants_clr = normalise(bigtot, totals, total_pot) + # calculate clr analytics output for grant in grants: - _, clr_amount, num_contribs, contrib_amount = calculate_clr_for_donation( - curr_agg, trust_dict, grant.id, 0, v_threshold, total_pot - ) + num_contribs, contrib_amount, clr_amount = curr_grants_clr.get(grant.id, {'num_contribs': 0, 'contrib_amount': 0, 'clr_amount': None}).values() # debug_output.append([grant.id, grant.title, num_contribs, contrib_amount, clr_amount]) debug_output.append([grant.id, grant.title, grant.positive_round_contributor_count, float(grant.amount_received_in_round), clr_amount]) diff --git a/app/grants/tasks.py b/app/grants/tasks.py index 0bc744d93ea..8e4cf043911 100644 --- a/app/grants/tasks.py +++ b/app/grants/tasks.py @@ -421,3 +421,13 @@ def generate_collection_cache(self, collection_id): collection.generate_cache() except Exception as e: print(e) + + +@app.shared_task(bind=True, max_retries=3) +def save_clr_prediction_curve(self, grant_id, clr_prediction_curve): + try: + grant = Grant.objects.get(pk=grant_id) + clr_round.record_clr_prediction_curve(grant, clr_prediction_curve) + grant.save() + except Exception as e: + print(e) From 9d5959b84ae3f176cb7a38e54a772ce470f3a941 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Mon, 6 Sep 2021 13:08:20 +0100 Subject: [PATCH 12/17] Fixes indentation --- app/grants/clr3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/grants/clr3.py b/app/grants/clr3.py index 7493bada5bd..bd50a08b807 100644 --- a/app/grants/clr3.py +++ b/app/grants/clr3.py @@ -241,7 +241,7 @@ def calculate_clr_for_prediction(bigtot, totals, curr_agg, trust_dict, v_thresho if int(dummy_user) > int(k2): tot += ((amount * v2) ** 0.5) / ((amount * v2) ** 0.5 / (v_threshold * max(trust_dict[k2], 1)) + 1) - if type(tot) == complex: + if type(tot) == complex: tot = float(tot.real) totals[grant_id] = {'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot} From 53b18539655e3ac8ef1d2d7b2329705ecf8c9173 Mon Sep 17 00:00:00 2001 From: Aditya Anand M C Date: Mon, 6 Sep 2021 18:26:08 +0530 Subject: [PATCH 13/17] fix: GITC-383 Editing bounty value is changing the amount to non number --- app/assets/v2/js/pages/change_bounty.js | 8 ++++---- app/assets/v2/js/shared.js | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/assets/v2/js/pages/change_bounty.js b/app/assets/v2/js/pages/change_bounty.js index e726d19594e..381c6135ea8 100644 --- a/app/assets/v2/js/pages/change_bounty.js +++ b/app/assets/v2/js/pages/change_bounty.js @@ -226,11 +226,11 @@ $(document).ready(function() { const denomination = $('input[name=denomination]').val(); - setTimeout(() => setUsdAmount(denomination), 1000); + setTimeout(() => setUsdAmount(denomination, false), 1000); - $('input[name=hours]').keyup(() => setUsdAmount(denomination)); - $('input[name=hours]').blur(() => setUsdAmount(denomination)); - $('input[name=amount]').keyup(() => setUsdAmount(denomination)); + $('input[name=hours]').keyup(() => setUsdAmount(denomination, false)); + $('input[name=hours]').blur(() => setUsdAmount(denomination, false)); + $('input[name=amount]').keyup(() => setUsdAmount(denomination, false)); $('input[name=usd_amount]').on('focusin', function() { $('input[name=usd_amount]').attr('prev_usd_amount', $(this).val()); diff --git a/app/assets/v2/js/shared.js b/app/assets/v2/js/shared.js index c89612dfa2c..16eb96b251e 100644 --- a/app/assets/v2/js/shared.js +++ b/app/assets/v2/js/shared.js @@ -668,16 +668,19 @@ this.actions_page_warn_if_not_on_same_network = function() { attach_change_element_type(); -this.setUsdAmount = function(givenDenomination) { +this.setUsdAmount = function(givenDenomination, approx = true) { const amount = $('input[name=amount]').val(); const denomination = givenDenomination || $('#token option:selected').text(); getUSDEstimate(amount, denomination, function(estimate) { - if (estimate['value']) { + + const key = approx ? 'value' : 'value_unrounded'; + + if (estimate[key]) { $('#usd-amount-wrapper').show(); $('#usd_amount_text').show(); - $('#usd_amount').val(estimate['value']); + $('#usd_amount').val(estimate[key]); $('#usd_amount_text').html(estimate['rate_text']); $('#usd_amount').removeAttr('disabled'); } else { From e24ca3b22c10100446d00aae2b2393c9e7a81473 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Mon, 6 Sep 2021 14:10:06 +0100 Subject: [PATCH 14/17] Inlines prediction curve saving --- app/grants/clr3.py | 4 ++-- app/grants/tasks.py | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/app/grants/clr3.py b/app/grants/clr3.py index bd50a08b807..cfc1d5d942b 100644 --- a/app/grants/clr3.py +++ b/app/grants/clr3.py @@ -24,7 +24,6 @@ import numpy as np from grants.models import Contribution, Grant, GrantCollection -from grants.tasks import save_clr_prediction_curve from townsquare.models import SquelchProfile @@ -415,7 +414,8 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn print(clr_prediction_curve) # save the new predicition curve via the model - save_clr_prediction_curve.delay(grant.id, clr_prediction_curve) + clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve) + _grant.save() debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr}) diff --git a/app/grants/tasks.py b/app/grants/tasks.py index 8e4cf043911..0bc744d93ea 100644 --- a/app/grants/tasks.py +++ b/app/grants/tasks.py @@ -421,13 +421,3 @@ def generate_collection_cache(self, collection_id): collection.generate_cache() except Exception as e: print(e) - - -@app.shared_task(bind=True, max_retries=3) -def save_clr_prediction_curve(self, grant_id, clr_prediction_curve): - try: - grant = Grant.objects.get(pk=grant_id) - clr_round.record_clr_prediction_curve(grant, clr_prediction_curve) - grant.save() - except Exception as e: - print(e) From b239616eb28836f53743b773f0a3cbb631616936 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Mon, 6 Sep 2021 14:37:19 +0100 Subject: [PATCH 15/17] Enqueues prediction curve saving --- app/grants/clr3.py | 4 ++-- app/grants/tasks.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/app/grants/clr3.py b/app/grants/clr3.py index cfc1d5d942b..3970775ea76 100644 --- a/app/grants/clr3.py +++ b/app/grants/clr3.py @@ -24,6 +24,7 @@ import numpy as np from grants.models import Contribution, Grant, GrantCollection +from grants.tasks import save_clr_prediction_curve from townsquare.models import SquelchProfile @@ -414,8 +415,7 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn print(clr_prediction_curve) # save the new predicition curve via the model - clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve) - _grant.save() + save_clr_prediction_curve.delay(_grant.id, clr_round.id, clr_prediction_curve) debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr}) diff --git a/app/grants/tasks.py b/app/grants/tasks.py index 0bc744d93ea..1d6470a074c 100644 --- a/app/grants/tasks.py +++ b/app/grants/tasks.py @@ -11,7 +11,7 @@ from celery import app from celery.utils.log import get_task_logger from dashboard.models import Profile -from grants.models import Grant, GrantCollection, Subscription +from grants.models import Grant, GrantCLR, GrantCollection, Subscription from grants.utils import get_clr_rounds_metadata, save_grant_to_notion from marketing.mails import ( new_contributions, new_grant, new_grant_admin, notion_failure_email, thank_you_for_supporting, @@ -421,3 +421,14 @@ def generate_collection_cache(self, collection_id): collection.generate_cache() except Exception as e: print(e) + + +@app.shared_task(bind=True, max_retries=3) +def save_clr_prediction_curve(self, grant_id, clr_pk, clr_prediction_curve): + try: + grant = Grant.objects.get(pk=grant_id) + clr_round = GrantCLR.objects.filter(pk=clr_pk) + clr_round.record_clr_prediction_curve(grant, clr_prediction_curve) + grant.save() + except Exception as e: + print(e) From 5dd912487e9ccbf88d5bea9591eb25fc6d783998 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Mon, 6 Sep 2021 15:58:49 +0100 Subject: [PATCH 16/17] Back to inline Its too risky offloading this to the queue atm because of how much backlog the queue holds --- app/grants/clr3.py | 9 +++++---- app/grants/tasks.py | 11 ----------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/app/grants/clr3.py b/app/grants/clr3.py index 3970775ea76..4ce6abf95e5 100644 --- a/app/grants/clr3.py +++ b/app/grants/clr3.py @@ -24,7 +24,6 @@ import numpy as np from grants.models import Contribution, Grant, GrantCollection -from grants.tasks import save_clr_prediction_curve from townsquare.models import SquelchProfile @@ -300,7 +299,7 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn initial_query = get_summed_contribs_query(grants, clr_round.start_date, clr_round.end_date, clr_round.contribution_multiplier, network) if only_grant_pk: - grants = grants.filter(pk=40) + grants = grants.filter(pk=only_grant_pk) # open cursor and execute the groupBy sum for the round with connection.cursor() as cursor: @@ -414,8 +413,10 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn print(clr_prediction_curve) - # save the new predicition curve via the model - save_clr_prediction_curve.delay(_grant.id, clr_round.id, clr_prediction_curve) + clr_round.record_clr_prediction_curve(_grant, clr_prediction_curve) + + if from_date > (clr_calc_start_time - timezone.timedelta(hours=1)): + _grant.save() debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr}) diff --git a/app/grants/tasks.py b/app/grants/tasks.py index 1d6470a074c..d5c2b6367a6 100644 --- a/app/grants/tasks.py +++ b/app/grants/tasks.py @@ -421,14 +421,3 @@ def generate_collection_cache(self, collection_id): collection.generate_cache() except Exception as e: print(e) - - -@app.shared_task(bind=True, max_retries=3) -def save_clr_prediction_curve(self, grant_id, clr_pk, clr_prediction_curve): - try: - grant = Grant.objects.get(pk=grant_id) - clr_round = GrantCLR.objects.filter(pk=clr_pk) - clr_round.record_clr_prediction_curve(grant, clr_prediction_curve) - grant.save() - except Exception as e: - print(e) From 7070b5184910abdf2759cc34a924cfd7282951c9 Mon Sep 17 00:00:00 2001 From: Graham Dixon Date: Mon, 6 Sep 2021 17:25:22 +0100 Subject: [PATCH 17/17] Ensure we only modify latest on the latest GrantCLRCalculation --- app/grants/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/grants/models.py b/app/grants/models.py index 6b41d21b98d..4ee49b6c972 100644 --- a/app/grants/models.py +++ b/app/grants/models.py @@ -251,7 +251,7 @@ def grants(self): def record_clr_prediction_curve(self, grant, clr_prediction_curve): - for obj in self.clr_calculations.filter(grant=grant): + for obj in self.clr_calculations.filter(grant=grant, latest=True): obj.latest = False obj.save()