From a42f47cc2e07e38b0779207e8d294dc8d0eef26c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Redrejo?= Date: Thu, 13 Jun 2024 19:16:35 +0200 Subject: [PATCH 1/2] trying to fix integrity errors before migrating tables --- kolibri/utils/main.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/kolibri/utils/main.py b/kolibri/utils/main.py index 2c0cf0e38af..9d354d94078 100644 --- a/kolibri/utils/main.py +++ b/kolibri/utils/main.py @@ -10,6 +10,7 @@ from django.core.exceptions import ValidationError from django.core.management import call_command from django.core.management.base import handle_default_options +from django.db import connections from django.db.utils import DatabaseError import kolibri @@ -185,6 +186,27 @@ def _copy_preseeded_db(db_name, target=None): ) +def sqlite_check_foreign_keys(): + db_connection = connections["default"] + with db_connection.cursor() as cursor: + cursor.execute("PRAGMA foreign_key_check;") + result = cursor.fetchall() + if len(result) > 0: + logger.warning( + "Foreign key constraint failed. Trying to fix integrity errors..." + ) + for row in result: + bad_table = row[0] + rowid = row[1] + # for security, only fix automatically error integrities in loggers + if bad_table[:6] == "logger": + cursor.execute(f"DELETE FROM {bad_table} WHERE rowid = {rowid};") + else: + logger.error( + f"Foreign key constraint failed in {bad_table} table, rowid {rowid}. Please fix it manually." + ) + + def _upgrades_before_django_setup(updated, version): if version and updated: check_plugin_config_file_location(version) @@ -205,6 +227,7 @@ def _upgrades_before_django_setup(updated, version): check_default_options_exist() if OPTIONS["Database"]["DATABASE_ENGINE"] == "sqlite": + sqlite_check_foreign_keys() # If we are using sqlite, # we can shortcut migrations by using the preseeded databases # that we bundle in the Kolibri whl file. @@ -236,7 +259,7 @@ def _post_django_initialization(): process_cache.directory, settings.CACHES["process_cache"]["SHARDS"], settings.CACHES["process_cache"]["TIMEOUT"], - **settings.CACHES["process_cache"]["OPTIONS"] + **settings.CACHES["process_cache"]["OPTIONS"], ) From 132e31835dc56c10e6d88e2b0b4a2e87d99bb64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20L=2E=20Redrejo=20Rodr=C3=ADguez?= Date: Mon, 17 Jun 2024 10:43:33 +0200 Subject: [PATCH 2/2] do not use django connections and apply the check to all existing db files --- kolibri/utils/main.py | 56 ++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/kolibri/utils/main.py b/kolibri/utils/main.py index 9d354d94078..1f18328cb58 100644 --- a/kolibri/utils/main.py +++ b/kolibri/utils/main.py @@ -1,8 +1,8 @@ import logging.config import os import shutil +import sqlite3 import sys -from sqlite3 import DatabaseError as SQLite3DatabaseError import django from diskcache.fanout import FanoutCache @@ -10,7 +10,6 @@ from django.core.exceptions import ValidationError from django.core.management import call_command from django.core.management.base import handle_default_options -from django.db import connections from django.db.utils import DatabaseError import kolibri @@ -150,7 +149,7 @@ def _setup_django(): try: django.setup() - except (DatabaseError, SQLite3DatabaseError) as e: + except (DatabaseError, sqlite3.DatabaseError) as e: if "malformed" in str(e): logger.error( "Your database appears to be corrupted. If you encounter this," @@ -187,24 +186,37 @@ def _copy_preseeded_db(db_name, target=None): def sqlite_check_foreign_keys(): - db_connection = connections["default"] - with db_connection.cursor() as cursor: - cursor.execute("PRAGMA foreign_key_check;") - result = cursor.fetchall() - if len(result) > 0: - logger.warning( - "Foreign key constraint failed. Trying to fix integrity errors..." - ) - for row in result: - bad_table = row[0] - rowid = row[1] - # for security, only fix automatically error integrities in loggers - if bad_table[:6] == "logger": - cursor.execute(f"DELETE FROM {bad_table} WHERE rowid = {rowid};") - else: - logger.error( - f"Foreign key constraint failed in {bad_table} table, rowid {rowid}. Please fix it manually." - ) + DATABASE_NAMES = [ + os.path.join(KOLIBRI_HOME, OPTIONS["Database"]["DATABASE_NAME"] or "db.sqlite3") + ] + DATABASE_NAMES += [ + os.path.join(KOLIBRI_HOME, "{}.sqlite3".format(db)) + for db in ADDITIONAL_SQLITE_DATABASES + ] + + for name in DATABASE_NAMES: + db_connection = sqlite3.connect(name) + with sqlite3.connect(name) as db_connection: + cursor = db_connection.cursor() + cursor.execute("PRAGMA foreign_key_check;") + result = cursor.fetchall() + if len(result) > 0: + logger.warning( + "Foreign key constraint failed. Trying to fix integrity errors..." + ) + for row in result: + bad_table = row[0] + rowid = row[1] + # for security, in the default database, + # only fix automatically integrity errors in loggers + if bad_table[:6] == "logger" or name != DATABASE_NAMES[0]: + cursor.execute( + f"DELETE FROM {bad_table} WHERE rowid = {rowid};" + ) + else: + logger.error( + f"Foreign key constraint failed in {bad_table} table, rowid {rowid}. Please fix it manually." + ) def _upgrades_before_django_setup(updated, version): @@ -252,7 +264,7 @@ def _post_django_initialization(): if "DatabaseCache" not in CACHES["process_cache"]["BACKEND"]: try: process_cache.cull() - except SQLite3DatabaseError: + except sqlite3.DatabaseError: shutil.rmtree(process_cache.directory, ignore_errors=True) os.mkdir(process_cache.directory) process_cache._cache = FanoutCache(