Bug/stuck but catchup false (#169)

* wip

* testing missing

* added README disclaimer

* replace stakerReward with poolReward (endpoint change)

* found out logical flaw

* README update

* added BadStatusException and generic Exception catches to try-catch. Default seed lists are now Multichain endpoints.

* readme

* fix unit tests

* addressed requested changes

README.md

@@ -1,8 +1,12 @@
 # thornode-telegram-bot ⚡🤖
 A telegram bot to monitor the status of THORNodes.
 
+**We highly encourage the community to create Pull Requests.**
+
 If you have questions feel free to open a github issue or contact us in our Telegram Channel https://t.me/block42_crypto!
 
+*The current maser is made for Multichain (Testnet and Chaosnet). The last single chain release is v1.13.2*
+
 ## Requirements
 * Telegram
 * Kubectl (if you want to run with Kubernetes)
@@ -306,9 +310,6 @@ kubectl delete -f kubernetes/k8s_thornode_bot_deployment_chaosnet.yaml -f kubern
 ### [Docker Standalone](#docker-standalone)
 To run the bot as a docker container, make sure you have docker installed (see: https://docs.docker.com/get-docker).
 
-Navigate to the root directory of this repository and execute the following commands:
-The environment includes a mongodb instance as well as the [Tendermint Client](https://github.com/block42-blockchain-company/tendermint-block-parser)
-
 Build the thornode bot docker image as described in the `Dockerfile`:
 
 ```

bot/constants/globals.py

@@ -67,7 +67,7 @@
 SLASH_POINTS_NOTIFICATION_THRESHOLD_DEFAULT = 3
 THORCHAIN_ONCHAIN_API_URL = "https://thorchain-service.b42.tech/v1/"
 
-DEFAULT_SEED_LIST = "https://chaosnet-seed.thorchain.info/" \
-    if NETWORK_TYPE == "CHAOSNET" else "https://testnet-seed.thorchain.info/"
+DEFAULT_SEED_LIST = "https://seed.thorchain.info/" \
+    if NETWORK_TYPE == "CHAOSNET" else "https://testnet.seed.thorchain.info/"
 
 SEED_LIST_URL = os.environ.get("SEED_LIST_URL", DEFAULT_SEED_LIST)

bot/handlers/handlers.py

@@ -41,7 +41,6 @@ def on_start_command(update, context):
     text += 'We develop for the community and depend on your input. Participate and shape the Thorchain Telegram Bot ' \
             'by joining the discussion on GitHub: https://github.com/block42-blockchain-company/thornode-telegram-bot/discussions/146\n'
 
-
     # Send message
     try_message_with_home_menu(context=context,
                                chat_id=update.message.chat.id,

bot/handlers/network_info_handlers.py

@@ -66,8 +66,8 @@ async def show_network_stats(update, context):
         text += "\n💰 Block Rewards:\n  *" + \
                 tor_to_rune(network['blockRewards']['blockReward']) + "* (total)\n  *" + \
                 tor_to_rune(network['blockRewards']['bondReward']) + "* (nodes)\n  *" + \
-                tor_to_rune(network['blockRewards']['stakeReward']) + "* (stakers)\n  *" + \
-                '{:.2f}'.format((int(network['blockRewards']['stakeReward']) / int(
+                tor_to_rune(network['blockRewards']['poolReward']) + "* (stakers)\n  *" + \
+                '{:.2f}'.format((int(network['blockRewards']['poolReward']) / int(
                     network['blockRewards']['blockReward']) * 100)) + " %* (staker share)\n"
 
         text += f"\n🔓 Network Security: *{network_security_ratio_to_string(get_network_security_ratio(network)).value}*\n"

bot/jobs/thorchain_node_jobs.py

@@ -35,7 +35,6 @@ def check_thornodes(context):
                          datetime.timestamp(
                              datetime.now() - timedelta(seconds=local_node['notification_timeout_in_seconds']))
         if is_not_blocked:
-
             message = build_notification_message_for_active_node(local_node, remote_node, context)
 
             if message:
@@ -233,6 +232,7 @@ def is_thornode_healthy(context, node_address) -> bool:
     was_healthy = node_data["healthy"]
 
     try:
+        # Check whether node answers. If it doesn't we get an Exception.
         get_latest_block_height(node_data['ip_address'])
 
         if not was_healthy:
@@ -241,7 +241,7 @@ def is_thornode_healthy(context, node_address) -> bool:
         context.job.context['chat_data']['nodes'][node_address]["healthy"] = True
         return True
 
-    except (Timeout, ConnectionError):
+    except (Timeout, ConnectionError, BadStatusException, Exception):
         if was_healthy:
             try_message_with_home_menu(context=context, chat_id=chat_id, text=get_node_health_warning_message(node_data))
 

bot/service/thorchain_network_service.py

@@ -6,7 +6,7 @@
 from requests.exceptions import Timeout, ConnectionError, HTTPError
 
 from constants.mock_values import thorchain_last_block_mock
-from service.general_network_service import get_request_json
+from service.general_network_service import get_request_json, BadStatusException
 from constants.globals import *
 from constants.node_ips import *
 
@@ -29,18 +29,29 @@ def get_latest_block_height(node_ip=None) -> int:
     return int(get_node_status(node_ip)['result']['sync_info']['latest_block_height'])
 
 
+def is_block_height_stuck(main_node_ip, reference_node_ip) -> bool:
+    main_block_height = get_latest_block_height(main_node_ip)
+    reference_block_height = get_latest_block_height(reference_node_ip)
+
+    # Block heights within plus minus 1 block are considered equal (network latency could lead to one block difference).
+    # If both nodes have the same blockheight it's very likely that they are "good" nodes. It's very unlikely
+    # that two nodes get stuck at the same block height.
+    block_height_difference = abs(main_block_height - reference_block_height)
+    return block_height_difference > 1
+
+
 def is_thorchain_catching_up(node_ip=None) -> bool:
     return get_node_status(node_ip)['result']['sync_info']['catching_up']
 
 
 def is_midgard_api_healthy(node_ip) -> bool:
     try:
         get_request_json_thorchain(url_path=":8080/v2/health", node_ip=node_ip)
-    except (Timeout, ConnectionError):
+    except (Timeout, ConnectionError, HTTPError):
         logger.warning(f"Timeout or Connection error with {node_ip}")
         return False
-    except HTTPError as e:
-        logger.info(f"Error {e.errno} in 'is_midgard_api_healthy({node_ip}).")
+    except (BadStatusException, Exception) as e:
+        logger.info(f"Error {e.message} in 'is_midgard_api_healthy({node_ip}).")
         return False
     return True
 
@@ -64,11 +75,11 @@ def get_network_data(node_ip=None):
     return get_request_json_thorchain(url_path=f":8080/v2/network", node_ip=node_ip)
 
 
-def get_thorchain_network_constants(node_ip=None):
+def get_thorchain_network_constants():
     return get_request_json_thorchain(url_path=f":8080/v2/thorchain/constants")
 
 
-def get_thorchain_blocks_per_year(node_ip=None):
+def get_thorchain_blocks_per_year():
     constants = get_thorchain_network_constants()
     return constants['int_64_values']['BlocksPerYear']
 
@@ -127,8 +138,15 @@ def get_request_json_thorchain(url_path: str, node_ip: str = None) -> dict:
     available_node_ips = requests.get(url=SEED_LIST_URL, timeout=CONNECTION_TIMEOUT).json()
 
     random.shuffle(available_node_ips)
-    for random_node_ip in available_node_ips:
-        if not is_thorchain_catching_up(random_node_ip):
+    for index in range(0, len(available_node_ips)):
+        random_node_ip = available_node_ips[index]
+
+        # Most performant way at hand to use a different node ip in the list.
+        # If index + 1 is out of bounce, it becomes 0 due to remainder division.
+        reference_index = (index + 1) % len(available_node_ips)
+        reference_node_ip = available_node_ips[reference_index]
+
+        if not is_block_height_stuck(random_node_ip, reference_node_ip):
             try:
                 return get_request_json(url=f"http://{random_node_ip}{url_path}{REQUEST_POSTFIX}")
             except Exception:

bot/service/utils.py

@@ -235,7 +235,7 @@ def get_network_security_ratio(network_json):
     """
 
     total_active_bond = int(network_json['bondMetrics']['totalActiveBond'])
-    total_staked = int(network_json['totalStaked'])
+    total_staked = int(network_json['totalPooledRune'])
     return total_active_bond / (total_active_bond + total_staked)
 
 

test/mock_files/status.json

@@ -1 +1 @@
-{"jsonrpc": "2.0", "id": "", "result": {"node_info": {"protocol_version": {"p2p": "7", "block": "10", "app": "0"}, "id": "36ddffb0b7e5fb013dda88e34f410fb619c8670e", "listen_addr": "tcp://0.0.0.0:26656", "network": "thorchain", "version": "0.32.9", "channels": "4020212223303800", "moniker": "local", "other": {"tx_index": "on", "rpc_address": "tcp://0.0.0.0:26657"}}, "sync_info": {"latest_block_hash": "DBEAA49A8FA0C601B1E43F4BF380ACE580A65DC0B626FCA83C440CCAC5678B03", "latest_app_hash": "F8AE0517FE02281C163591F60EF84D691FF52160AB1E961E31563B7F30DD64A4", "latest_block_height": "58773", "latest_block_time": "2020-04-23T04:26:15.472797738Z", "catching_up": false}, "validator_info": {"address": "5F5B897B5BA6F21FBB05D7C202DA30A0AE4B4972", "pub_key": {"type": "tendermint/PubKeyEd25519", "value": "1xcnwh+qkuRePR3kEVarDqo5OG5aI2qV+rtXLvAkjgc="}, "voting_power": "100"}}}
+{"jsonrpc": "2.0", "id": "", "result": {"node_info": {"protocol_version": {"p2p": "7", "block": "10", "app": "0"}, "id": "36ddffb0b7e5fb013dda88e34f410fb619c8670e", "listen_addr": "tcp://0.0.0.0:26656", "network": "thorchain", "version": "0.32.9", "channels": "4020212223303800", "moniker": "local", "other": {"tx_index": "on", "rpc_address": "tcp://0.0.0.0:26657"}}, "sync_info": {"latest_block_hash": "DBEAA49A8FA0C601B1E43F4BF380ACE580A65DC0B626FCA83C440CCAC5678B03", "latest_app_hash": "F8AE0517FE02281C163591F60EF84D691FF52160AB1E961E31563B7F30DD64A4", "latest_block_height": "58783", "latest_block_time": "2020-04-23T04:26:15.472797738Z", "catching_up": false}, "validator_info": {"address": "5F5B897B5BA6F21FBB05D7C202DA30A0AE4B4972", "pub_key": {"type": "tendermint/PubKeyEd25519", "value": "1xcnwh+qkuRePR3kEVarDqo5OG5aI2qV+rtXLvAkjgc="}, "voting_power": "100"}}}

test/unit_tests/test_jobs.py

@@ -8,7 +8,7 @@
 from jobs.other_nodes_jobs import check_health
 from jobs.thorchain_node_jobs import check_solvency, check_churning
 from models.nodes import Node, UnauthorizedException
-from helpers import network_data
+from unit_tests.helpers import network_data
 
 
 class ContextMock:
@@ -251,7 +251,7 @@ def test_check_churning(self, mock_get_pool_addresses_from_any_node, mock_get_ne
             'bondMetrics': {
                 'totalActiveBond': '10'
             },
-            'totalStaked': '100',
+            'totalPooledRune': '100',
             'bondingAPY': '100.01',
             'liquidityAPY': '99.01'
         }
@@ -263,7 +263,6 @@ def test_check_churning(self, mock_get_pool_addresses_from_any_node, mock_get_ne
                 }
             ]
 
-
         # first call: no churning, just initializing
         check_churning(self.context)
 

test/unit_tests/test_threshold.py

@@ -5,7 +5,7 @@
 from handlers.settings_handlers import set_slash_points_threshold
 from jobs.thorchain_node_jobs import build_notification_message_for_active_node
 from service.utils import get_slash_points_threshold
-from helpers import ContextMock, node_mock, JobContextMock
+from unit_tests.helpers import ContextMock, node_mock, JobContextMock
 
 
 class ThresholdTest(unittest.TestCase):