From e39a8f3210507bd9ea3b096577a3e2cca4fe595f Mon Sep 17 00:00:00 2001 From: Rose Heart Date: Wed, 23 Oct 2024 16:15:37 +0000 Subject: [PATCH] Bug fixes and additinal diagnostics added. Version update. Changes to be committed: modified: Base/JackrabbitLocker modified: Base/JackrabbitOliverTwist modified: Base/JackrabbitRelay modified: Base/Library/JRRmimic.py modified: Base/Library/JRRsupport.py modified: Base/Library/JackrabbitProxy.py modified: Base/Library/JackrabbitRelay.py modified: Extras/CodeProofs/readOHLCV new file: Extras/OliverTwist/ot2gb --- Base/JackrabbitLocker | 2 +- Base/JackrabbitOliverTwist | 89 ++++++++++++++------------ Base/JackrabbitRelay | 2 +- Base/Library/JRRmimic.py | 2 +- Base/Library/JRRsupport.py | 3 +- Base/Library/JackrabbitProxy.py | 2 +- Base/Library/JackrabbitRelay.py | 2 +- Extras/CodeProofs/readOHLCV | 3 +- Extras/OliverTwist/ot2gb | 107 ++++++++++++++++++++++++++++++++ 9 files changed, 164 insertions(+), 48 deletions(-) create mode 100755 Extras/OliverTwist/ot2gb diff --git a/Base/JackrabbitLocker b/Base/JackrabbitLocker index 5ce90ee..36340c3 100755 --- a/Base/JackrabbitLocker +++ b/Base/JackrabbitLocker @@ -28,7 +28,7 @@ import json import JRRsupport -Version="0.0.0.1.1000" +Version="0.0.0.1.1005" BaseDirectory='/home/JackrabbitRelay2/Base' ConfigDirectory='/home/JackrabbitRelay2/Config' LogDirectory="/home/JackrabbitRelay2/Logs" diff --git a/Base/JackrabbitOliverTwist b/Base/JackrabbitOliverTwist index 467c96b..eb793a4 100755 --- a/Base/JackrabbitOliverTwist +++ b/Base/JackrabbitOliverTwist @@ -5,16 +5,18 @@ # 2021 Copyright © Robert APM Darin # All rights reserved unconditionally. -# This is the Jackrabbit limit order manager. Its purpose is to track and maintain "orphaned" orders. Orphan -# orders are liit order that the parent or order initiator does not wat a response for. TradingView is an -# example, but the situation applies to any process that does not manage or maintain limit orders on their -# own. - -# IMPORTANT: Under no circumstances should THIS program manage an individual order. It is to be managed by a -# separate process, called an orphan manager. This is simply because there is no way on knowing how many -# exchanges/brokers might be used simultaneously. The memory of a single process managing a thousand orders -# would be absolutely thrashed. The orphan manager will follow the same conventions of the PlaceOrder -# program, ie: +# This is the Jackrabbit limit order manager. Its purpose is to track and maintain +# "orphaned" orders. Orphan orders are liit order that the parent or order +# initiator does not wat a response for. TradingView is an example, but the +# situation applies to any process that does not manage or maintain limit orders on +# their own. + +# IMPORTANT: Under no circumstances should THIS program manage an individual order. +# It is to be managed by a separate process, called an orphan manager. This is +# simply because there is no way on knowing how many exchanges/brokers might be +# used simultaneously. The memory of a single process managing a thousand orders +# would be absolutely thrashed. The orphan manager will follow the same conventions +# of the PlaceOrder program, ie: # Old: # Orphan.ccxt @@ -28,23 +30,28 @@ # OliverTwist-mimic # OliverTwist-oanda -# ANY order (direct orphan or conditional) is an ORPHAN at all times withis THIS framework. +# ANY order (direct orphan or conditional) is an ORPHAN at all times withis THIS +# framework. -# IMPORTANT: This fraework treats conditionals the same as orphans. The separation and distinction takes -# place in the transactor. While this has plenty of issues to be aware of, it is the most effecient way of -# handling the situation since at this level (dispatcher), they are syntactically the same. +# IMPORTANT: This fraework treats conditionals the same as orphans. The separation +# and distinction takes place in the transactor. While this has plenty of issues to +# be aware of, it is the most effecient way of handling the situation since at this +# level (dispatcher), they are syntactically the same. -# Framework identification, along with any other order verification, is done in the PlaceOrder +# Framework identification, along with any other order verification, is done in the +# PlaceOrder -# MIGRATE to checking the highest (take profit) and lowat prices (stop loss), rather then checking every -# order. This will optimize heavy usagevsystem to handle the most relevant orders first. Will need rethings -# on order list processing approach and methedology. +# MIGRATE to checking the highest (take profit) and lowat prices (stop loss), +# rather then checking every order. This will optimize heavy usagevsystem to handle +# the most relevant orders first. Will need rethings on order list processing +# approach and methedology. -# MOVE to reading storehouse every pass and write price data into main store house... Reading orderdetails -# is 3 minutes for order sweep, pre processing orders brings the time down to 0.00015 seconds. Be re-reading -# the storehouse each pass, we can update order as received to maintain sweepspeed. Reading the ticker will -# slow thing down, but breaking the storehouse down to exchange/account/asset with high/low extreme analysis -# will dramatically improve performance. +# MOVE to reading storehouse every pass and write price data into main store +# house... Reading orderdetails is 3 minutes for order sweep, pre processing orders +# brings the time down to 0.00015 seconds. Be re-reading the storehouse each pass, +# we can update order as received to maintain sweepspeed. Reading the ticker will +# slow thing down, but breaking the storehouse down to exchange/account/asset with +# high/low extreme analysis will dramatically improve performance. import sys sys.path.append('/home/JackrabbitRelay2/Base/Library') @@ -60,7 +67,7 @@ import subprocess import JRRsupport import JackrabbitRelay as JRR -Version="0.0.0.1.1000" +Version="0.0.0.1.1005" BaseDirectory='/home/JackrabbitRelay2/Base' DataDirectory='/home/JackrabbitRelay2/Data' ConfigDirectory='/home/JackrabbitRelay2/Config' @@ -118,9 +125,9 @@ def GetID(): pw+=oc return pw -# Split the complete list stored on disk, if it exists. Supports both orphans and conditionals. -# RUNS ONLY ONCE, no locking needed as this runs at the very start, if it runs at all. -# At some FUTURE point, this will be REMOVED. +# Split the complete list stored on disk, if it exists. Supports both orphans and +# conditionals. RUNS ONLY ONCE, no locking needed as this runs at the very start, +# if it runs at all. At some FUTURE point, this will be REMOVED. def SplitStorehouse(cltype=None): # Required as the blobals are modified @@ -160,8 +167,8 @@ def SplitStorehouse(cltype=None): JRLog.Write(f"Broken: {Entry}") continue - # I need to be able to extract Exchange, account, and asset to build the separate pair - # files. + # I need to be able to extract Exchange, account, and asset to + # build the separate pair files. if 'Order' in Orphan: if type(Orphan['Order'])==str: @@ -181,8 +188,9 @@ def SplitStorehouse(cltype=None): idx=f"{exchange}.{account}.{asset}" fname=f"{OliverTwistData}/{idx}.Storehouse" - # Write out the new storehouse. We already have the complete line from the original, just - # use it. This is not about preprocessing, only converting. + # Write out the new storehouse. We already have the complete line + # from the original, just use it. This is not about preprocessing, + # only converting. JRRsupport.AppendFile(fname,Entry+'\n') rc+=1 @@ -190,7 +198,8 @@ def SplitStorehouse(cltype=None): if rc>0: JRLog.Write(f"{rc} record(s) converted") -# Read the complete list stored on disk, if it exists. Supports both orphans and conditionals. +# Read the complete list stored on disk, if it exists. Supports both orphans and +# conditionals. def ReadReceiver(): # Required as the blobals are modified @@ -329,7 +338,8 @@ def WriteStorehouse(idx,OrphanList,deleteKey=None): interceptor.Critical(False) JRLog.Write(f"{idx}/{len(OrphanList)} order(s) written in {str(datetime.datetime.now()-StartTime)} seconds") -# Read the complete list stored on disk, if it exists. Supports both orphans and conditionals. +# Read the complete list stored on disk, if it exists. Supports both orphans and +# conditionals. def ReadStorehouse(idx=None,OrigOrphanList=None): # Required as the blobals are modified @@ -446,18 +456,19 @@ def ProcessOrphan(**kwargs): osh=kwargs State='Waiting' - # Massive headache: One of the most significant issues is that we don't know anything about any of the - # orders, so the processor needs to deal with everthing now in a layered approach. At the level we are - # at in this function, we only know the index and framework. Orphan and Conditional status is completely - # unknown... + # Massive headache: One of the most significant issues is that we don't know + # anything about any of the orders, so the processor needs to deal with + # everthing now in a layered approach. At the level we are at in this function, + # we only know the index and framework. Orphan and Conditional status is + # completely unknown... try: # We NEED the framework. idxList=osh['IDX'].split('.') relay=JRR.JackrabbitRelay(exchange=idxList[0],account=idxList[1],asset=idxList[2]) - # IMPORTANT: Modules loaded this way do NOT have access to anything of the parent. These - # modules are in complete and total isolation. + # IMPORTANT: Modules loaded this way do NOT have access to anything of the + # parent. These modules are in complete and total isolation. Processor=importlib.import_module(f"OliverTwist-{relay.Framework}") State=Processor.OrderProcessor(osh) diff --git a/Base/JackrabbitRelay b/Base/JackrabbitRelay index 7794d15..fc0a5bc 100755 --- a/Base/JackrabbitRelay +++ b/Base/JackrabbitRelay @@ -16,7 +16,7 @@ import json import JRRsupport -Version="0.0.0.1.1000" +Version="0.0.0.1.1005" BaseDirectory='/home/JackrabbitRelay2/Base' ConfigDirectory='/home/JackrabbitRelay2/Config' LogDirectory="/home/JackrabbitRelay2/Logs" diff --git a/Base/Library/JRRmimic.py b/Base/Library/JRRmimic.py index df749be..f530477 100755 --- a/Base/Library/JRRmimic.py +++ b/Base/Library/JRRmimic.py @@ -45,7 +45,7 @@ class mimic: # placed in init and released at exit. def __init__(self,Exchange,Config,Active,DataDirectory=None): - self.Version="0.0.0.1.1000" + self.Version="0.0.0.1.1005" self.StableCoinUSD=['USDT','USDC','BUSD','UST','DAI','FRAX','TUSD', \ 'USDP','LUSD','USDN','HUSD','FEI','TRIBE','RSR','OUSD','XSGD', \ diff --git a/Base/Library/JRRsupport.py b/Base/Library/JRRsupport.py index 7142d2a..5d9aabb 100755 --- a/Base/Library/JRRsupport.py +++ b/Base/Library/JRRsupport.py @@ -669,8 +669,7 @@ def list(self): # Create a directory def mkdir(fn): - if not os.path.isdir(fn): - os.makedirs(fn) + os.makedirs(fn,exist_ok=True) # Get Yesterday's date diff --git a/Base/Library/JackrabbitProxy.py b/Base/Library/JackrabbitProxy.py index 79c06a8..cda4e11 100755 --- a/Base/Library/JackrabbitProxy.py +++ b/Base/Library/JackrabbitProxy.py @@ -50,7 +50,7 @@ class JackrabbitProxy: def __init__(self,framework=None,payload=None,exchange=None,account=None,asset=None,Usage=None): # All the default locations - self.Version="0.0.0.1.1000" + self.Version="0.0.0.1.1005" self.BaseDirectory='/home/JackrabbitRelay2/Base' self.ConfigDirectory='/home/JackrabbitRelay2/Config' self.DataDirectory="/home/JackrabbitRelay2/Data" diff --git a/Base/Library/JackrabbitRelay.py b/Base/Library/JackrabbitRelay.py index 42aaf76..2b89b91 100755 --- a/Base/Library/JackrabbitRelay.py +++ b/Base/Library/JackrabbitRelay.py @@ -110,7 +110,7 @@ def Success(self,f,s): class JackrabbitRelay: def __init__(self,framework=None,payload=None,exchange=None,account=None,asset=None,secondary=None,NoIdentityVerification=False,Usage=None,RaiseError=False): # All the default locations - self.Version="0.0.0.1.1000" + self.Version="0.0.0.1.1005" self.NOhtml='NO!

NO!

' self.Directories={} self.Directories['Base']='/home/JackrabbitRelay2/Base' diff --git a/Extras/CodeProofs/readOHLCV b/Extras/CodeProofs/readOHLCV index 042552c..694e78c 100755 --- a/Extras/CodeProofs/readOHLCV +++ b/Extras/CodeProofs/readOHLCV @@ -36,5 +36,4 @@ markets=relay.GetMarkets() ohlcv=relay.GetOHLCV(symbol=asset,timeframe=tf,limit=count) for slice in ohlcv: - print(slice) - + print(','.join(f'{value:.5f}' for value in slice)) diff --git a/Extras/OliverTwist/ot2gb b/Extras/OliverTwist/ot2gb new file mode 100755 index 0000000..18d79e8 --- /dev/null +++ b/Extras/OliverTwist/ot2gb @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Jackrabbit Relay +# 2021 Copyright © Robert APM Darin +# All rights reserved unconditionally. + +import sys +sys.path.append('/home/JackrabbitRelay2/Base/Library') +import os +import time +import datetime +import json + +import JRRsupport +import JackrabbitRelay as JRR + +# Oliver Twist log +# 2024-08-01 06:35:32.669979 476486 88557 -> 88528 Prft short, 71.0: 1.08271 -> 1.081480/0.08730, 4:36:21.447434 + +chartDir='/home/JackrabbitRelay2/Data/Charts/' + +# Initialize global signal interceptor. prevent file trashing on CTRL-C + +Log=JRR.JackrabbitLog() +interceptor=JRRsupport.SignalInterceptor(Log=Log) + +### +### Main code base. +### + +if len(sys.argv)<2: + print("An least OliverTwist log file is required.") + sys.exit(1) + +entry={} +relay={} +lines={} +orderList={} + +# Copy arg list +gblog=sys.argv[1] + +# Remove sys arg list + +for i in range(1,len(sys.argv)): + sys.argv.remove(sys.argv[1]) + +# Read the log file + +data=gblog.split('.') + +account=data[2] +asset=data[3] + +# Add the / +pair=asset[:3]+'/'+asset[3:] +relay=JRR.JackrabbitRelay(framework='oanda',exchange=data[1],account=account,asset=pair) +lines=JRRsupport.ReadFile(gblog).strip().split('\n') + +# Get the order IDs and read them into a dictionary. + +oList=[] +nList=[] +nLines=[] + +for line in lines: + line=line.lower() + if line=='' or ('prft' not in line and 'rduc' not in line and 'loss' not in line): + continue + + data=line.split(' ') + while '' in data: + data.remove('') + + # Selling date/time + sdt=data[0]+' '+data[1] + pid=data[2] + id=data[3] + cid=data[5] + act=data[6] + dir=data[7].replace(',','') + amt=data[8].replace(':','') + bp=data[9] + sp,rpl=data[11].split('/') + dur=' '.join(data[12:]) + + oDetail=relay.GetOrderDetails(OrderID=id)[-1] + + # Buying date/time + parts=oDetail['time'].replace('T',' ').replace('Z','').split('.') + bdt=f"{parts[0]}.{parts[1][:6]}" + if act=='rduc': + amt=oDetail['units'] + bal=oDetail['accountBalance'] + + if dir=='shrt' or dir=='short': + dir='Short' + + nav=float(bp)*abs(float(amt)) + + if act=='prft' or act=='loss': + print(f"{bdt} {pid} OT {dir} - {id} Buy @{bp} -> {amt}") + print(f"{sdt} {pid} OT {id} {dir} - {cid} Sell {amt} @{sp} -> {rpl} {bal}/{nav:.5f} {dur}") + + if act=='rduc': + print(f"{sdt} {pid} OT {id} {dir} - {cid} ReduceBy {1} @{sp} -> {rpl} {bal}/{nav:.5f}")