improve address encoding/decoding

gsrest/db/cassandra.py

@@ -1,6 +1,5 @@
 import re
 import time
-# import hashlib
 import asyncio
 import heapq
 from async_lru import alru_cache
@@ -19,7 +18,11 @@
 from gsrest.util.eth_logs import decode_db_logs
 from gsrest.errors import NotFoundException, BadUserInputException
 from gsrest.util import is_eth_like
-from gsrest.util.address import cannonicalize_address, address_to_user_format
+from gsrest.util.address import (address_to_user_format)
+from gsrest.util.evm import (bytes_to_hex, strip_0x)
+from gsrest.util.tron import partial_tron_to_partial_evm
+
+# import hashlib
 
 SMALL_PAGE_SIZE = 1000
 BIG_PAGE_SIZE = 5000
@@ -58,20 +61,6 @@ def transaction_ordering_key(tx_id_key, tx):
     return (tx[tx_id_key], -trace_index, -log_index)
 
 
-def evm_address_from_hex(currency, address):
-    # eth addresses are case insensitive
-    try:
-        if address.startswith("0x"):
-            return bytes.fromhex(address[2:].lower())
-        else:
-            return bytes.fromhex(address.lower())
-    except ValueError:
-        # bytes.fromHex throws value error if non hex chars are found
-        raise BadUserInputException(
-            "The address provided does not look"
-            f" like a {currency.upper()} address: {address}")
-
-
 def identity(y, x):
     return x
 
@@ -792,10 +781,6 @@ async def get_addresses_by_ids(self,
         result = await self.concurrent_with_args(currency, 'transformed',
                                                  query, params)
 
-        for row in result:
-            if is_eth_like(currency):
-                row['address'] = \
-                    address_to_user_format(currency, row['address'])
         return result
 
     async def get_new_address(self, currency, address):
@@ -805,7 +790,7 @@ async def get_new_address(self, currency, address):
         if not prefix:
             return None
         if is_eth_like(currency):
-            address = evm_address_from_hex(currency, address)
+            #address = evm_address_from_hex(currency, address)
             prefix = prefix.upper()
         prefix_length = self.get_prefix_lengths(currency)['address']
         query = ("SELECT * FROM new_addresses "
@@ -825,7 +810,7 @@ async def get_address_id(self, currency, address):
         if not prefix:
             return None
         if is_eth_like(currency):
-            address = evm_address_from_hex(currency, address)
+            #address = evm_address_from_hex(currency, address)
             prefix = prefix.upper()
         query = ("SELECT address_id FROM address_ids_by_address_prefix "
                  "WHERE address_prefix = %s AND address = %s")
@@ -914,10 +899,7 @@ async def new_address(self, currency, address):
     async def new_entity(self, currency, address):
         data = await self.new_address(currency, address)
         data['no_addresses'] = 1
-        if is_eth_like(currency):
-            data['root_address'] = address_to_user_format(currency, address)
-        else:
-            data['root_address'] = address
+        data['root_address'] = address
         return data
 
     async def get_address_by_address_id(self, currency, address_id):
@@ -932,7 +914,7 @@ async def get_address_by_address_id(self, currency, address_id):
                 return None
             raise NotFoundException(
                 f'Address {address_id} has no external transactions')
-        return address_to_user_format(currency, result["address"])
+        return result["address"]
 
     async def get_address(self, currency, address):
         try:
@@ -1105,7 +1087,9 @@ async def list_links(self,
     async def list_matching_addresses(self, currency, expression, limit=10):
         prefix_lengths = self.get_prefix_lengths(currency)
         expression_orginal = expression
-        expression = cannonicalize_address(currency, expression, partial=True)
+        if currency == 'trx':
+            expression = partial_tron_to_partial_evm(expression)
+
         if len(expression) < prefix_lengths['address']:
             return []
         norm = identity
@@ -1120,10 +1104,9 @@ async def list_matching_addresses(self, currency, expression, limit=10):
             prefix = prefix.upper()
         rows = []
 
-        async def collect(query, paging_state):
-            while paging_state and len(rows) < limit:
-                if paging_state is True:
-                    paging_state = None
+        async def collect(query):
+            paging_state = None
+            while len(rows) < limit:
                 result = await self.execute_async(currency,
                                                   'transformed',
                                                   query, [prefix],
@@ -1137,18 +1120,20 @@ async def collect(query, paging_state):
                         expression_orginal)
                 ])
                 paging_state = result.paging_state
+                if paging_state is None:
+                    break
 
         query = "SELECT address FROM address_ids_by_address_prefix "\
                 "WHERE address_prefix = %s"
 
-        await collect(query, True)
+        await collect(query)
 
         if len(rows) < limit:
             query = "SELECT address FROM new_addresses "\
                     "WHERE address_prefix = %s"
             if self.parameters[currency]["use_delta_updater_v1"]:
                 try:
-                    await collect(query, True)
+                    await collect(query)
                 except InvalidRequest as e:
                     if 'new_addresses' not in str(e):
                         raise e
@@ -1538,11 +1523,17 @@ async def list_matching_txs(self, currency, expression, limit):
 
         return rows[0:limit]
 
-    @eth
     def scrub_prefix(self, currency, expression):
+        if isinstance(expression, bytes):
+            expression = bytes_to_hex(expression)
+
+        if currency == 'eth':
+            expression = strip_0x(expression)
+
         if currency not in self.parameters:
             raise NotFoundException(f'{currency} not found')
-        bech32_prefix = self.parameters[currency]['bech_32_prefix']
+
+        bech32_prefix = self.parameters[currency].get('bech_32_prefix', '')
         return expression[len(bech32_prefix):] \
             if expression.startswith(bech32_prefix) \
             else expression
@@ -1614,7 +1605,7 @@ async def finish_entity(self, currency, row, with_txs=True):
         return await self.finish_address(currency, row, with_txs)
 
     async def finish_entity_eth(self, currency, row, with_txs=True):
-        row['root_address'] = address_to_user_format(currency, row['address'])
+        row['root_address'] = row['address']
         return await self.finish_address(currency, row, with_txs)
 
     async def finish_addresses(self, currency, rows, with_txs=True):
@@ -1661,8 +1652,6 @@ async def finish_address(self, currency, row, with_txs=True):
         return row
 
     async def finish_address_eth(self, currency, row, with_txs=True):
-        if 'address' in row:
-            row['address'] = address_to_user_format(currency, row['address'])
         row['cluster_id'] = row['address_id']
         row['total_received'] = \
             self.markup_currency(currency, row['total_received'])
@@ -1720,7 +1709,7 @@ async def is_address_dirty(self, currency, address):
         if not prefix:
             return None
         if is_eth_like(currency):
-            address = evm_address_from_hex(currency, address)
+            #address = evm_address_from_hex(currency, address)
             prefix = prefix.upper()
         prefix_length = self.get_prefix_lengths(currency)['address']
         query = ("SELECT address FROM dirty_addresses "
@@ -1786,6 +1775,7 @@ async def add_balance_eth(self, currency, row):
 
     def scrub_prefix_eth(self, currency, expression):
         # remove 0x prefix
+        expression = bytes_to_hex(expression)
         if expression.startswith("0x"):
             return expression[2:]
         else:
@@ -2064,10 +2054,8 @@ async def list_txs_by_node_type_eth(self,
                 token_tx = await self.fetch_token_transaction(
                     currency, full_tx, addr_tx["log_index"])
 
-                addr_tx['to_address'] = address_to_user_format(
-                    currency, token_tx['to_address'])
-                addr_tx['from_address'] = address_to_user_format(
-                    currency, token_tx['from_address'])
+                addr_tx['to_address'] = token_tx['to_address']
+                addr_tx['from_address'] = token_tx['from_address']
                 addr_tx['currency'] = token_tx["currency"]
                 addr_tx['token_tx_id'] = addr_tx["log_index"]
                 value = token_tx['value'] * \
@@ -2081,9 +2069,6 @@ async def list_txs_by_node_type_eth(self,
                     (-1 if addr_tx['is_outgoing'] else 1)
 
             contract_creation = full_tx.get('contract_creation', None)
-            if contract_creation is not None:
-                contract_creation = address_to_user_format(
-                    currency, contract_creation)
 
             addr_tx['contract_creation'] = contract_creation
             addr_tx['tx_hash'] = full_tx['tx_hash']
@@ -2145,20 +2130,16 @@ async def list_txs_by_hashes_eth(self,
         result_with_tokens = []
         for row in result:
 
-            row['from_address'] = address_to_user_format(
-                currency, row['from_address'])
             to_address = row['to_address']
             if to_address is None:
                 # this is a contract creation transaction
                 # set recipient to newly created contract
                 # and mark tx as creation
-                row['to_address'] = address_to_user_format(
-                    currency, row['receipt_contract_address'])
+                row['to_address'] = row['receipt_contract_address']
                 row['contract_creation'] = True
             else:
                 # normal transaction
-                row['to_address'] = address_to_user_format(
-                    currency, to_address)
+                row['to_address'] = to_address
                 # result['contract_creation'] = False
 
             result_with_tokens.append(row)
@@ -2189,15 +2170,12 @@ async def get_tx_by_hash_eth(self, currency, hash):
         if to_address is None:
             # this is a contract creation transaction
             # set recipient to newly created contract and mark tx as creation
-            result['to_address'] = address_to_user_format(
-                currency, result['receipt_contract_address'])
+            result['to_address'] = result['receipt_contract_address']
             result['contract_creation'] = True
         else:
             # normal transaction
-            result['to_address'] = address_to_user_format(currency, to_address)
+            result['to_address'] = to_address
             # result['contract_creation'] = False
-        result['from_address'] = address_to_user_format(
-            currency, result['from_address'])
         return result
 
     async def list_links_eth(self,
@@ -2324,8 +2302,6 @@ async def list_links_eth(self,
                                              tx_ids,
                                              include_token_txs=True)
 
-        neighbor = address_to_user_format(currency, neighbor)
-        address = address_to_user_format(currency, address)
         txs = [
             tx for tx in all_txs
             if tx["to_address"] == neighbor and tx["from_address"] == address

gsrest/service/addresses_service.py

@@ -3,7 +3,7 @@
 from gsrest.service.rates_service import get_rates
 import gsrest.service.common_service as common
 from gsrest.errors import NotFoundException
-from gsrest.util.address import cannonicalize_address
+from gsrest.util.address import cannonicalize_address, address_to_user_format
 from openapi_server.models.neighbor_addresses import NeighborAddresses
 from openapi_server.models.neighbor_address import NeighborAddress
 import asyncio
@@ -18,7 +18,6 @@ async def list_tags_by_address(request,
                                address,
                                page=None,
                                pagesize=None):
-    address = cannonicalize_address(currency, address)
     return await common.list_tags_by_address(request,
                                              currency,
                                              address,
@@ -76,7 +75,7 @@ async def list_address_neighbors(request,
         return NeighborAddresses()
     aws = [
         get_address(request, currency,
-                    cannonicalize_address(currency, row[dst + '_address']))
+                    address_to_user_format(currency, row[dst + '_address']))
         for row in results
     ]
 
@@ -113,7 +112,6 @@ async def list_address_links(request,
 
 async def try_get_delta_update_entity_dummy(request, currency, address,
                                             notfound):
-    address = cannonicalize_address(currency, address)
     db = request.app['db']
     try:
         aws = [get_rates(request, currency), db.new_entity(currency, address)]
@@ -127,22 +125,22 @@ async def try_get_delta_update_entity_dummy(request, currency, address,
 
 
 async def get_address_entity(request, currency, address):
-    address = cannonicalize_address(currency, address)
+    address_canonical = cannonicalize_address(currency, address)
     db = request.app['db']
 
     notfound = NotFoundException(
         'Entity for address {} not found'.format(address))
     try:
-        entity_id = await db.get_address_entity_id(currency, address)
+        entity_id = await db.get_address_entity_id(currency, address_canonical)
     except NotFoundException as e:
         if 'not found' not in str(e):
             raise e
         return await try_get_delta_update_entity_dummy(request, currency,
-                                                       address, notfound)
+                                                       address_canonical, notfound)
 
     if entity_id is None:
         return await try_get_delta_update_entity_dummy(request, currency,
-                                                       address, notfound)
+                                                       address_canonical, notfound)
 
     result = await get_entity(request,
                               currency,