Restructure and organize project files

IntersectMBO · Nov 26, 2024 · 31d8e68 · 31d8e68
1 parent 7598420
commit 31d8e68
Show file tree

Hide file tree

Showing 21 changed files with 64 additions and 294 deletions.
diff --git a/.buildkite/db_sync_full_sync.yml b/.buildkite/db_sync_full_sync.yml
@@ -1,9 +1,9 @@
 steps:
   - label: ':drum_with_drumsticks: Full sync test :drum_with_drumsticks:'
     commands:
-      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/full_sync_from_clean_state.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}"  -e "${environment}"
-      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/snapshot_creation.py -dv "${db_sync_version}" -e "${environment}" -rosc "${run_only_sync_test}"
-      - nix develop --accept-flake-config .#python --command python ./db_sync_tests/tests/local_snapshot_restoration.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}" -e "${environment}" -rosc "${run_only_sync_test}"
+      - nix develop --accept-flake-config .#python --command python ./sync_tests/tests/full_sync_from_clean_state.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}"  -e "${environment}"
+      - nix develop --accept-flake-config .#python --command python ./sync_tests/tests/snapshot_creation.py -dv "${db_sync_version}" -e "${environment}" -rosc "${run_only_sync_test}"
+      - nix develop --accept-flake-config .#python --command python ./sync_tests/tests/local_snapshot_restoration.py -nv "${node_version}" -dv "${db_sync_version}" -dsa "${db_sync_start_arguments}" -e "${environment}" -rosc "${run_only_sync_test}"
     timeout_in_minutes: 43200
     agents:
       system: x86_64-linux

diff --git a/db_sync_tests/utils/aws_db_utils.py b/db_sync_tests/utils/aws_db_utils.py
diff --git a/flake.nix b/flake.nix
@@ -37,7 +37,7 @@
               nativeBuildInputs = with pkgs; [ bash nix gnugrep gnumake gnutar coreutils git xz ];
             };
             python = pkgs.mkShell {
-              nativeBuildInputs = with pkgs; with python39Packages; [ python39Full virtualenv pip matplotlib pandas requests xmltodict psutil GitPython pymysql postgresql_14 wget curl psycopg2 assertpy ];
+              nativeBuildInputs = with pkgs; with python39Packages; [ python39Full virtualenv pip matplotlib pandas requests xmltodict psutil GitPython pymysql postgresql_14 wget curl psycopg2 assertpy colorama];
             };
             postgres = pkgs.mkShell {
               nativeBuildInputs = with pkgs; [ glibcLocales postgresql_14 lsof procps wget ];

diff --git a/db_sync_tests/scripts/db-sync-start.sh → sync_tests/scripts/db-sync-start.sh b/db_sync_tests/scripts/db-sync-start.sh → sync_tests/scripts/db-sync-start.sh
diff --git a/db_sync_tests/scripts/postgres-start.sh → sync_tests/scripts/postgres-start.sh b/db_sync_tests/scripts/postgres-start.sh → sync_tests/scripts/postgres-start.sh
diff --git a/db_sync_tests/tests/__init__.py → sync_tests/tests/__init__.py b/db_sync_tests/tests/__init__.py → sync_tests/tests/__init__.py
diff --git a/db_sync_tests/utils/db_tables_ddl.sql → sync_tests/tests/db_tables_ddl.sql b/db_sync_tests/utils/db_tables_ddl.sql → sync_tests/tests/db_tables_ddl.sql
diff --git a/...tests/tests/full_sync_from_clean_state.py → ...tests/tests/full_sync_from_clean_state.py b/...tests/tests/full_sync_from_clean_state.py → ...tests/tests/full_sync_from_clean_state.py
@@ -9,7 +9,7 @@
 
 sys.path.append(os.getcwd())
 
-from db_sync_tests.utils.utils import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
+from sync_tests.utils.utils_db_sync import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
     get_os_type, get_total_ram_in_GB, upload_artifact, clone_repo, zip_file, execute_command, \
     print_file, stop_process, write_data_as_json_to_file, get_node_config_files, \
     get_node_version, get_db_sync_version, start_node_in_cwd, wait_for_db_to_sync, \
@@ -25,7 +25,7 @@
     NODE_ARCHIVE, DB_SYNC_ARCHIVE, SYNC_DATA_ARCHIVE, EXPECTED_DB_SCHEMA, EXPECTED_DB_INDEXES, \
     ENVIRONMENT \
 
-from db_sync_tests.utils.aws_db_utils import get_identifier_last_run_from_table, \
+from sync_tests.utils.aws_db_utils import get_identifier_last_run_from_table, \
     add_bulk_rows_into_db, add_single_row_into_db
 
 

diff --git a/..._tests/tests/iohk_snapshot_restoration.py → ..._tests/tests/iohk_snapshot_restoration.py b/..._tests/tests/iohk_snapshot_restoration.py → ..._tests/tests/iohk_snapshot_restoration.py
@@ -8,7 +8,7 @@
 
 sys.path.append(os.getcwd())
 
-from db_sync_tests.utils.utils import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
+from sync_tests.utils.utils_db_sync import seconds_to_time, get_no_of_cpu_cores, get_current_date_time, \
     get_os_type, get_total_ram_in_GB, upload_artifact, clone_repo, wait, zip_file, \
     print_file, stop_process, copy_node_executables, write_data_as_json_to_file, \
     execute_command, get_node_config_files, are_errors_present_in_db_sync_logs, \
@@ -26,7 +26,7 @@
     DB_SYNC_PERF_STATS, NODE_LOG, DB_SYNC_LOG, EPOCH_SYNC_TIMES, PERF_STATS_ARCHIVE, \
     NODE_ARCHIVE, DB_SYNC_ARCHIVE, SYNC_DATA_ARCHIVE, ENVIRONMENT \
 
-from utils.aws_db_utils import get_identifier_last_run_from_table, \
+from sync_tests.utils.aws_db_utils import get_identifier_last_run_from_table, \
     add_bulk_rows_into_db, add_single_row_into_db
 
 

diff --git a/...tests/tests/local_snapshot_restoration.py → ...tests/tests/local_snapshot_restoration.py b/...tests/tests/local_snapshot_restoration.py → ...tests/tests/local_snapshot_restoration.py
@@ -7,7 +7,7 @@
 
 sys.path.append(os.getcwd())
 
-from db_sync_tests.utils.utils import seconds_to_time, get_no_of_cpu_cores, \
+from sync_tests.utils.utils_db_sync import seconds_to_time, get_no_of_cpu_cores, \
     get_current_date_time, get_os_type, get_total_ram_in_GB, \
     upload_artifact, print_file, stop_process, export_env_var, is_string_present_in_file, \
     zip_file, write_data_as_json_to_file, get_db_sync_version, start_node_in_cwd, \

diff --git a/sync_tests/node_sync_test.py → sync_tests/tests/node_sync_test.py b/sync_tests/node_sync_test.py → sync_tests/tests/node_sync_test.py
@@ -92,7 +92,7 @@ def check_string_format(input_string):
 
 
 def delete_node_files():
-    for p in Path(".").glob("cardano-*"):
+    for p in Path("..").glob("cardano-*"):
         print_info_warn(f"deleting file: {p}")
         p.unlink(missing_ok=True)
 
@@ -133,7 +133,7 @@ def disable_p2p_node_config():
 def rm_node_config_files() -> None:
     print_info_warn('Removing existing config files')
     os.chdir(Path(ROOT_TEST_PATH))
-    for gen in Path(".").glob("*-genesis.json"):
+    for gen in Path("..").glob("*-genesis.json"):
         Path(gen).unlink(missing_ok=True)
     for f in ('config.json', 'topology.json'):
         Path(f).unlink(missing_ok=True)

diff --git a/.../node_write_mainnet_tx_count_per_epoch.py → .../node_write_mainnet_tx_count_per_epoch.py b/.../node_write_mainnet_tx_count_per_epoch.py → .../node_write_mainnet_tx_count_per_epoch.py
diff --git a/sync_tests/node_write_sync_values_to_db.py → ...sts/tests/node_write_sync_values_to_db.py b/sync_tests/node_write_sync_values_to_db.py → ...sts/tests/node_write_sync_values_to_db.py
diff --git a/db_sync_tests/tests/snapshot_creation.py → sync_tests/tests/snapshot_creation.py b/db_sync_tests/tests/snapshot_creation.py → sync_tests/tests/snapshot_creation.py
@@ -8,7 +8,7 @@
 
 sys.path.append(os.getcwd())
 
-from db_sync_tests.utils.utils import seconds_to_time, get_no_of_cpu_cores, \
+from sync_tests.utils.utils_db_sync import seconds_to_time, get_no_of_cpu_cores, \
     get_current_date_time, get_os_type, get_total_ram_in_GB, \
     upload_artifact, print_file, create_db_sync_snapshot_stage_2, \
     write_data_as_json_to_file, set_buildkite_meta_data, \
@@ -17,7 +17,7 @@
     get_file_size, create_db_sync_snapshot_stage_1, print_color_log, \
     ROOT_TEST_PATH, ENVIRONMENT
 
-from db_sync_tests.utils.aws_db_utils import get_identifier_last_run_from_table, add_single_row_into_db
+from sync_tests.utils.aws_db_utils import get_identifier_last_run_from_table, add_single_row_into_db
 
 
 

diff --git a/db_sync_tests/utils/__init__.py → sync_tests/utils/__init__.py b/db_sync_tests/utils/__init__.py → sync_tests/utils/__init__.py