✨(backends) granular data backends without template pattern

.env.dist

@@ -62,6 +62,12 @@ RALPH_BACKENDS__DATA__ES__TEST_HOSTS=http://elasticsearch:9200
 RALPH_BACKENDS__DATA__ES__TEST_INDEX=test-index-foo
 RALPH_BACKENDS__DATA__ES__TEST_FORWARDING_INDEX=test-index-foo-2
 
+# ES lrs backend
+
+# Same options as for the ES data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__ES__ instead. Example:
+# RALPH_BACKENDS__LRS__ES__HOSTS=http://elasticsearch:9200
+
 # MONGO data backend
 
 RALPH_BACKENDS__DATA__MONGO__CONNECTION_URI=mongodb://mongo:27017/
@@ -76,7 +82,13 @@ RALPH_BACKENDS__DATA__MONGO__TEST_FORWARDING_COLLECTION=foo-2
 RALPH_BACKENDS__DATA__MONGO__TEST_DATABASE=statements
 RALPH_BACKENDS__DATA__MONGO__TEST_CONNECTION_URI=mongodb://mongo:27017/
 
-# ClickHouse data backend
+# MONGO lrs backend
+
+# Same options as for the MONGO data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__MONGO__ instead. Example:
+# RALPH_BACKENDS__LRS__MONGO__CONNECTION_URI=mongodb://mongo:27017/
+
+# CLICKHOUSE data backend
 
 RALPH_BACKENDS__DATA__CLICKHOUSE__HOST=clickhouse
 RALPH_BACKENDS__DATA__CLICKHOUSE__PORT=8123
@@ -92,6 +104,30 @@ RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_HOST=clickhouse
 RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_PORT=8123
 RALPH_BACKENDS__DATA__CLICKHOUSE__TEST_TABLE_NAME=test_xapi_events_all
 
+# CLICKHOUSE lrs backend
+
+# Same options as for the CLICKHOUSE data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__CLICKHOUSE__ instead. Example:
+# RALPH_BACKENDS__LRS__CLICKHOUSE__HOST=clickhouse
+
+# Additional options specific to the CLICKHOUSE lrs backend:
+# RALPH_BACKENDS__LRS__CLICKHOUSE__IDS_CHUNK_SIZE=8123
+
+# FS data backend
+
+# RALPH_BACKENDS__DATA__FS__DEFAULT_CHUNK_SIZE=4096
+# RALPH_BACKENDS__DATA__FS__DEFAULT_DIRECTORY_PATH=.
+# RALPH_BACKENDS__DATA__FS__DEFAULT_QUERY_STRING=*
+# RALPH_BACKENDS__DATA__FS__LOCALE_ENCODING=utf8
+
+# FS lrs backend
+
+# Same options as for the FS data backend, however they are prefixed with
+# RALPH_BACKENDS__LRS__FS__ instead. Example:
+# RALPH_BACKENDS__LRS__FS__DEFAULT_DIRECTORY_PATH=.
+
+# Additional options specific to the FS lrs backend:
+# RALPH_BACKENDS__LRS__FS__DEFAULT_LRS_FILE=fs_lrs.jsonl
 
 # LRS HTTP backend
 

CHANGELOG.md

@@ -14,6 +14,10 @@ and this project adheres to
 - Implement xAPI LMS Profile statements validation
 - `EdX` to `xAPI` converters for enrollment events
 - Helm: Add variable ``ingress.hosts``
+- Backends: Add `Writable` and `Listable` interfaces to distinguish supported
+  functionalities among `data` backends
+- Backends: Add `get_backends` function to automatically discover backends
+  for CLI and LRS usage
 
 ### Changed
 

Makefile

@@ -85,7 +85,7 @@ arnold-bootstrap: \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a $(ARNOLD_APP) create_app_vaults && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a elasticsearch create_app_vaults && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) decrypt
-	sed -i 's/^# RALPH_BACKENDS__DATA__ES/RALPH_BACKENDS__DATA__ES/g' group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/secrets/$(ARNOLD_APP).vault.yml
+	sed -i 's/^# RALPH_BACKENDS__/RALPH_BACKENDS__/g' group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/secrets/$(ARNOLD_APP).vault.yml
 	source .k3d-cluster.env.sh && \
 	  $(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) encrypt
 	echo "skip_verification: True" > $(ARNOLD_APP_VARS)

src/helm/ralph/vault.yaml

@@ -1,3 +1,5 @@
+RALPH_BACKENDS__LRS__ES__HOSTS: http://elasticsearch:9200
+RALPH_BACKENDS__LRS__ES__INDEX: statements
 RALPH_BACKENDS__DATA__ES__HOSTS: http://elasticsearch:9200
 RALPH_BACKENDS__DATA__ES__INDEX: statements
 RALPH_SENTRY_DSN: https://fake@key.ingest.sentry.io/1234567

src/ralph/api/routers/health.py

@@ -6,19 +6,18 @@
 from fastapi import APIRouter, status
 from fastapi.responses import JSONResponse
 
-from ralph.backends.conf import backends_settings
+from ralph.backends.loader import get_lrs_backends
 from ralph.backends.lrs.base import BaseAsyncLRSBackend, BaseLRSBackend
 from ralph.conf import settings
-from ralph.utils import await_if_coroutine, get_backend_instance
+from ralph.utils import await_if_coroutine, get_backend_class
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter()
 
-BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_instance(
-    backend_type=backends_settings.BACKENDS.LRS,
-    backend_name=settings.RUNSERVER_BACKEND,
-)
+BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class(
+    backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND
+)()
 
 
 @router.get("/__lbheartbeat__")

src/ralph/api/routers/statements.py

@@ -27,9 +27,10 @@
 from ralph.api.auth.user import AuthenticatedUser
 from ralph.api.forwarding import forward_xapi_statements, get_active_xapi_forwardings
 from ralph.api.models import ErrorDetail, LaxStatement
-from ralph.backends.conf import backends_settings
+from ralph.backends.loader import get_lrs_backends
 from ralph.backends.lrs.base import (
     AgentParameters,
+    BaseAsyncLRSBackend,
     BaseLRSBackend,
     RalphStatementsQuery,
 )
@@ -45,7 +46,7 @@
 from ralph.models.xapi.base.common import IRI
 from ralph.utils import (
     await_if_coroutine,
-    get_backend_instance,
+    get_backend_class,
     now,
     statements_are_equivalent,
 )
@@ -58,10 +59,9 @@
 )
 
 
-BACKEND_CLIENT: BaseLRSBackend = get_backend_instance(
-    backend_type=backends_settings.BACKENDS.LRS,
-    backend_name=settings.RUNSERVER_BACKEND,
-)
+BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class(
+    backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND
+)()
 
 POST_PUT_RESPONSES = {
     400: {

src/ralph/backends/data/async_es.py

@@ -9,6 +9,8 @@
 from elasticsearch.helpers import BulkIndexError, async_streaming_bulk
 
 from ralph.backends.data.base import (
+    AsyncListable,
+    AsyncWritable,
     BaseAsyncDataBackend,
     BaseOperationType,
     DataBackendStatus,
@@ -23,7 +25,7 @@
 logger = logging.getLogger(__name__)
 
 
-class AsyncESDataBackend(BaseAsyncDataBackend):
+class AsyncESDataBackend(BaseAsyncDataBackend, AsyncWritable, AsyncListable):
     """Asynchronous Elasticsearch data backend."""
 
     name = "async_es"

src/ralph/backends/data/async_mongo.py

@@ -21,6 +21,8 @@
 from ralph.utils import parse_bytes_to_dict
 
 from ..data.base import (
+    AsyncListable,
+    AsyncWritable,
     BaseAsyncDataBackend,
     DataBackendStatus,
     async_enforce_query_checks,
@@ -29,7 +31,7 @@
 logger = logging.getLogger(__name__)
 
 
-class AsyncMongoDataBackend(BaseAsyncDataBackend):
+class AsyncMongoDataBackend(BaseAsyncDataBackend, AsyncWritable, AsyncListable):
     """Async MongoDB data backend."""
 
     name = "async_mongo"