diff --git a/.envrc b/.envrc index 2d2cb3033..3cbc6ebee 100644 --- a/.envrc +++ b/.envrc @@ -9,7 +9,7 @@ export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-api-client/" export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-sdk/" export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-pandas/" export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flight-server/" -export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flexfun/" +export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flexconnect/" export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-dbt/" export PATH="${PATH}:${SCRIPT_DIR}/gooddata-sdk/bin" diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 05fa27160..87f35c15e 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -29,7 +29,7 @@ jobs: - gooddata-sdk - gooddata-dbt - gooddata-flight-server - - gooddata-flexfun + - gooddata-flexconnect runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -83,7 +83,7 @@ jobs: - gooddata-sdk - gooddata-dbt - gooddata-flight-server - - gooddata-flexfun + - gooddata-flexconnect steps: - name: Obtain ${{ matrix.component }} artifacts uses: actions/download-artifact@v4 diff --git a/.github/workflows/dev-release.yaml b/.github/workflows/dev-release.yaml index ed703731f..db42df519 100644 --- a/.github/workflows/dev-release.yaml +++ b/.github/workflows/dev-release.yaml @@ -21,7 +21,7 @@ jobs: - gooddata-sdk - gooddata-dbt - gooddata-flight-server - - gooddata-flexfun + - gooddata-flexconnect steps: - name: Checkout Repository uses: actions/checkout@v4 diff --git a/.github/workflows/rw-collect-changes.yaml b/.github/workflows/rw-collect-changes.yaml index efb599ec9..518458b93 100644 --- a/.github/workflows/rw-collect-changes.yaml +++ b/.github/workflows/rw-collect-changes.yaml @@ -47,4 +47,4 @@ jobs: - 'gooddata-api-client/**' - 'gooddata-dbt/**' - 'gooddata-flight-server/**' - - 'gooddata-flexfun/**' + - 'gooddata-flexconnect/**' diff --git a/.github/workflows/rw-python-tests.yaml b/.github/workflows/rw-python-tests.yaml index 0f7e1a1f3..a3effd831 100644 --- a/.github/workflows/rw-python-tests.yaml +++ b/.github/workflows/rw-python-tests.yaml @@ -24,7 +24,7 @@ jobs: if: ${{ matrix.python_version == 'py312' }} uses: codecov/codecov-action@v3 with: - files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml,./gooddata-flexfun/coverage.xml + files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml,./gooddata-flexconnect/coverage.xml lint-and-format-check: runs-on: ubuntu-latest if: ${{inputs.changed-python-modules == 'true'}} diff --git a/.sonar.settings b/.sonar.settings index bc177f40e..53f46be57 100644 --- a/.sonar.settings +++ b/.sonar.settings @@ -1,4 +1,4 @@ # (C) 2021 GoodData Corporation -sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexfun +sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexconnect sonar.exclusions=gooddata-api-client/**/* sonar.python.version=3.9, 3.10, 3.11, 3.12 diff --git a/.sonarcloud.properties b/.sonarcloud.properties index d484c9e19..99a0c503e 100644 --- a/.sonarcloud.properties +++ b/.sonarcloud.properties @@ -1,4 +1,4 @@ # (C) 2022 GoodData Corporation -sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexfun +sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexconnect sonar.exclusions=gooddata-api-client/**/* sonar.python.version=3.9, 3.10, 3.11, 3.12 diff --git a/LICENSE.txt b/LICENSE.txt index 5242c3ae1..51ea274e4 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -7,4 +7,4 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ----------------------- -This repository includes source code for gooddata-api-client, gooddata-fdw, gooddata-pandas, gooddata-sdk, gooddata-flight-server, gooddata-flexfun, and gooddata-dbt, each of which is licensed under the MIT license above. For additional licensing information related to open source packages included in each of these projects, please see the OSS LICENSES directory. +This repository includes source code for gooddata-api-client, gooddata-fdw, gooddata-pandas, gooddata-sdk, gooddata-flight-server, gooddata-flexconnect, and gooddata-dbt, each of which is licensed under the MIT license above. For additional licensing information related to open source packages included in each of these projects, please see the OSS LICENSES directory. diff --git a/dev-requirements.txt b/dev-requirements.txt index 16d715f03..a183d6c1b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,11 +10,11 @@ pre-commit~=3.8.0 -r ./gooddata-fdw/requirements.txt -r ./gooddata-dbt/requirements.txt -r ./gooddata-flight-server/requirements.txt --r ./gooddata-flexfun/requirements.txt +-r ./gooddata-flexconnect/requirements.txt -r ./gooddata-sdk/test-requirements.txt -r ./gooddata-pandas/test-requirements.txt -r ./gooddata-fdw/test-requirements.txt -r ./gooddata-dbt/test-requirements.txt -r ./gooddata-flight-server/test-requirements.txt --r ./gooddata-flexfun/test-requirements.txt +-r ./gooddata-flexconnect/test-requirements.txt -r ./tests-support/requirements.txt diff --git a/gooddata-flexfun/.gitignore b/gooddata-flexconnect/.gitignore similarity index 100% rename from gooddata-flexfun/.gitignore rename to gooddata-flexconnect/.gitignore diff --git a/gooddata-flexfun/.readthedocs.yaml b/gooddata-flexconnect/.readthedocs.yaml similarity index 74% rename from gooddata-flexfun/.readthedocs.yaml rename to gooddata-flexconnect/.readthedocs.yaml index 2d13a3e2a..2a169e24c 100644 --- a/gooddata-flexfun/.readthedocs.yaml +++ b/gooddata-flexconnect/.readthedocs.yaml @@ -12,8 +12,8 @@ build: python: "3.11" sphinx: - configuration: gooddata-flexfun/docs/conf.py + configuration: gooddata-flexconnect/docs/conf.py python: install: - - requirements: gooddata-flexfun/docs/requirements.txt + - requirements: gooddata-flexconnect/docs/requirements.txt diff --git a/gooddata-flexfun/LICENSE.txt b/gooddata-flexconnect/LICENSE.txt similarity index 100% rename from gooddata-flexfun/LICENSE.txt rename to gooddata-flexconnect/LICENSE.txt diff --git a/gooddata-flexfun/Makefile b/gooddata-flexconnect/Makefile similarity index 100% rename from gooddata-flexfun/Makefile rename to gooddata-flexconnect/Makefile diff --git a/gooddata-flexconnect/README.md b/gooddata-flexconnect/README.md new file mode 100644 index 000000000..ecbd9330a --- /dev/null +++ b/gooddata-flexconnect/README.md @@ -0,0 +1,61 @@ +# GoodData FlexConnect + +GoodData FlexConnect allows you to build your own data source for GoodData Cloud or Cloud Native. + +FlexConnect works with a concept similar to 'table functions' that you may already know +from database technologies. + +- To build your own data source, you implement one or more FlexConnect functions. The + functions compute and return tabular data - how they do it is completely up to you. +- The functions are hosted and invoked inside a FlexConnect server (which is included in this package). +- A running FlexConnect server can be added as a data source to your GoodData Cloud or GoodData Cloud Native. +- The functions available on FlexConnect server will be mapped to data sets within GoodData's Semantic Model + and from then on can be used during report computation. + + +## Getting Started using the FlexConnect Template + +The easiest way to get started writing FlexConnect functions is to use [the template repository](https://github.com/gooddata/gooddata-flexconnect-template). +It provides a simple example of a FlexConnect function that can be used as a starting point for your own FlexConnect functions with all the necessary infrastructure in place. +It also has a README that explains how to get started with the template and some general tips on how to write FlexConnect functions. + +## Getting started using the FlexConnect package directly + +Install the package alongside the gooddata-flight-server using pip: + +```bash +pip install gooddata-flight-server gooddata-flexconnect +``` + +Next, update the GoodData Flight Server configuration to load the FlexConnect functions. + +```toml +[flexconnect] + +# specify one or more modules that contain your FlexConnect function implementations +# +functions = [ + "flexconnect.your_function" +] +``` + +Then when running the GoodData Flight Server, use the `--methods-provider` option to load the FlexConnect. +For example: + +```bash +#!/bin/bash + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +SERVER_CMD="${SCRIPT_DIR}/.venv/bin/gooddata-flight-server" + +export PYTHONPATH="${SCRIPT_DIR}/src" +export CONFIG_ENV="${1:-dev}" + +$SERVER_CMD start \ + --methods-provider gooddata_flexconnect \ + --config \ + config/${CONFIG_ENV}.server.toml \ + config/flexconnect.config.toml \ + --logging-config config/default.logging.ini \ + --dev-log +``` diff --git a/gooddata-flexfun/gooddata_flexfun/__init__.py b/gooddata-flexconnect/gooddata_flexconnect/__init__.py similarity index 64% rename from gooddata-flexfun/gooddata_flexfun/__init__.py rename to gooddata-flexconnect/gooddata_flexconnect/__init__.py index f6eec5ed7..d8e0a0003 100644 --- a/gooddata-flexfun/gooddata_flexfun/__init__.py +++ b/gooddata-flexconnect/gooddata_flexconnect/__init__.py @@ -1,7 +1,6 @@ # (C) 2024 GoodData Corporation -from gooddata_flexfun.flexfun.flex_fun import FlexFun -from gooddata_flexfun.flexfun.flex_fun_execution_context import ( +from gooddata_flexconnect.function.execution_context import ( ExecutionContext, ExecutionContextAbsoluteDateFilter, ExecutionContextAttribute, @@ -14,4 +13,5 @@ LabelElementsExecutionRequest, ReportExecutionRequest, ) -from gooddata_flexfun.flexfun.flight_methods import create_flexfun_flight_methods +from gooddata_flexconnect.function.flight_methods import create_flexconnect_flight_methods +from gooddata_flexconnect.function.function import FlexConnectFunction diff --git a/gooddata-flexfun/gooddata_flexfun/flexfun/__init__.py b/gooddata-flexconnect/gooddata_flexconnect/function/__init__.py similarity index 100% rename from gooddata-flexfun/gooddata_flexfun/flexfun/__init__.py rename to gooddata-flexconnect/gooddata_flexconnect/function/__init__.py diff --git a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_execution_context.py b/gooddata-flexconnect/gooddata_flexconnect/function/execution_context.py similarity index 96% rename from gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_execution_context.py rename to gooddata-flexconnect/gooddata_flexconnect/function/execution_context.py index c75dd2051..ba6d7136e 100644 --- a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_execution_context.py +++ b/gooddata-flexconnect/gooddata_flexconnect/function/execution_context.py @@ -236,7 +236,7 @@ class ExecutionContextAbsoluteDateFilter: @dataclass class ExecutionRequest: """ - Information about the execution request that is sent to the FlexFun. + Information about the execution request that is sent to the FlexConnect function. DEPRECATED: Use ReportExecutionRequest instead. """ @@ -481,7 +481,7 @@ def _dict_to_attributes(attributes: list[dict]) -> list[ExecutionContextAttribut @dataclass class ExecutionContext: """ - Execution context of the FlexFun + Execution context of the FlexConnect function """ execution_type: ExecutionType @@ -491,17 +491,17 @@ class ExecutionContext: organization_id: str """ - The ID of the organization that the FlexFun is executed in. + The ID of the organization that the FlexConnect function is executed in. """ workspace_id: str """ - The ID of the workspace that the FlexFun is executed in. + The ID of the workspace that the FlexConnect function is executed in. """ user_id: str """ - The ID of the user that invoked the FlexFun. + The ID of the user that invoked the FlexConnect function. """ timestamp: Optional[str] @@ -532,13 +532,13 @@ class ExecutionContext: report_execution_request: Optional[ReportExecutionRequest] """ - The report execution request that the FlexFun should process. + The report execution request that the FlexConnect function should process. Only present if the execution type is "REPORT". """ label_elements_execution_request: Optional[LabelElementsExecutionRequest] """ - The label elements execution request that the FlexFun should process. + The label elements execution request that the FlexConnect function should process. Only present if the execution type is "LABEL_ELEMENTS". """ @@ -568,8 +568,9 @@ def from_dict(d: dict) -> "ExecutionContext": @staticmethod def from_parameters(parameters: dict) -> Optional["ExecutionContext"]: """ - Create ExecutionContext from FlexFun parameters. - :param parameters: the parameters dictionary of the FlexFun invocation + Create ExecutionContext from FlexConnect function parameters. + + :param parameters: the parameters dictionary of the FlexConnect function invocation :return: None if the parameters do not contain the execution context, otherwise the execution context """ return ExecutionContext.from_dict(parameters["executionContext"]) if "executionContext" in parameters else None diff --git a/gooddata-flexfun/gooddata_flexfun/flexfun/flight_methods.py b/gooddata-flexconnect/gooddata_flexconnect/function/flight_methods.py similarity index 76% rename from gooddata-flexfun/gooddata_flexfun/flexfun/flight_methods.py rename to gooddata-flexconnect/gooddata_flexconnect/function/flight_methods.py index 363ecf31f..6ebab47fb 100644 --- a/gooddata-flexfun/gooddata_flexfun/flexfun/flight_methods.py +++ b/gooddata-flexconnect/gooddata_flexconnect/function/flight_methods.py @@ -16,16 +16,16 @@ flight_server_methods, ) -from gooddata_flexfun.flexfun.flex_fun import FlexFun -from gooddata_flexfun.flexfun.flex_fun_registry import FlexFunRegistry -from gooddata_flexfun.flexfun.flex_fun_task import FlexFunTask +from gooddata_flexconnect.function.function import FlexConnectFunction +from gooddata_flexconnect.function.function_registry import FlexConnectFunctionRegistry +from gooddata_flexconnect.function.function_task import FlexConnectFunctionTask -_LOGGER = structlog.get_logger("gooddata_flexfun.rpc") +_LOGGER = structlog.get_logger("gooddata_flexconnect.rpc") _DEFAULT_TASK_WAIT = 60.0 -class _FlexFunServerMethods(FlightServerMethods): - def __init__(self, ctx: ServerContext, registry: FlexFunRegistry) -> None: +class _FlexConnectServerMethods(FlightServerMethods): + def __init__(self, ctx: ServerContext, registry: FlexConnectFunctionRegistry) -> None: self._ctx = ctx self._registry = registry @@ -38,7 +38,7 @@ def _create_descriptor(fun_name: str, metadata: Optional[dict]) -> pyarrow.fligh return pyarrow.flight.FlightDescriptor.for_command(orjson.dumps(cmd)) - def _create_fun_info(self, fun: type[FlexFun]) -> pyarrow.flight.FlightInfo: + def _create_fun_info(self, fun: type[FlexConnectFunction]) -> pyarrow.flight.FlightInfo: # these are for type checker; the registry will only register functions # that have proper metadata on them assert fun.Name is not None @@ -57,18 +57,21 @@ def _extract_invocation_payload( ) -> tuple[str, dict, Optional[tuple[str, ...]]]: if descriptor.command is None or not len(descriptor.command): raise ErrorInfo.bad_argument( - "Incorrect FlexFun invocation. Flight descriptor must contain command with the invocation payload." + "Incorrect FlexConnect function invocation. Flight descriptor must contain command " + "with the invocation payload." ) try: payload = orjson.loads(descriptor.command) except Exception: - raise ErrorInfo.bad_argument("Incorrect FlexFun invocation. The invocation payload is not a valid JSON.") + raise ErrorInfo.bad_argument( + "Incorrect FlexConnect function invocation. The invocation payload is " "not a valid JSON." + ) fun = payload.get("functionName") if fun is None or not len(fun): raise ErrorInfo.bad_argument( - "Incorrect FlexFun invocation. The invocation payload does not specify 'functionName'." + "Incorrect FlexConnect function invocation. The invocation payload does not specify 'functionName'." ) parameters = payload.get("parameters") or {} @@ -80,13 +83,13 @@ def _prepare_task( self, context: pyarrow.flight.ServerCallContext, descriptor: pyarrow.flight.FlightDescriptor, - ) -> FlexFunTask: + ) -> FlexConnectFunctionTask: fun_name, parameters, columns = self._extract_invocation_payload(descriptor) headers = self.call_info_middleware(context).headers - flex_fun = self._registry.create_function(fun_name) + fun = self._registry.create_function(fun_name) - return FlexFunTask( - fun=flex_fun, + return FlexConnectFunctionTask( + fun=fun, parameters=parameters, columns=columns, headers=headers, @@ -100,7 +103,7 @@ def _prepare_flight_info(self, task_result: TaskExecutionResult) -> pyarrow.flig if task_result.cancelled: raise ErrorInfo.for_reason( ErrorCode.COMMAND_CANCELLED, - f"FlexFun invocation was cancelled. Invocation task was: '{task_result.task_id}'.", + f"FlexConnect function invocation was cancelled. Invocation task was: '{task_result.task_id}'.", ).to_server_error() result = task_result.result @@ -127,9 +130,9 @@ def list_flights( self, context: pyarrow.flight.ServerCallContext, criteria: bytes ) -> Generator[pyarrow.flight.FlightInfo, None, None]: structlog.contextvars.bind_contextvars(peer=context.peer()) - _LOGGER.info("list_flights", available_funs=self._registry.flex_funs_names) + _LOGGER.info("list_flights", available_funs=self._registry.function_names) - return (self._create_fun_info(fun) for fun in self._registry.flex_funs.values()) + return (self._create_fun_info(fun) for fun in self._registry.functions.values()) def get_flight_info( self, @@ -137,7 +140,7 @@ def get_flight_info( descriptor: pyarrow.flight.FlightDescriptor, ) -> pyarrow.flight.FlightInfo: structlog.contextvars.bind_contextvars(peer=context.peer()) - task: Optional[FlexFunTask] = None + task: Optional[FlexConnectFunctionTask] = None try: task = self._prepare_task(context, descriptor) @@ -163,7 +166,7 @@ def get_flight_info( if task is not None: _LOGGER.error("get_flight_info_failed", task_id=task.task_id, fun=task.fun_name, exc_info=True) else: - _LOGGER.error("flexfun_submit_failed", exc_info=True) + _LOGGER.error("flexconnect_fun_submit_failed", exc_info=True) raise @@ -190,23 +193,23 @@ def do_get( raise -_FLEXFUN_CONFIG_SECTION = "flexfun" -_FLEXFUN_FUNCTION_LIST = "functions" +_FLEX_CONNECT_CONFIG_SECTION = "flexconnect" +_FLEX_CONNECT_FUNCTION_LIST = "functions" @flight_server_methods -def create_flexfun_flight_methods(ctx: ServerContext) -> FlightServerMethods: +def create_flexconnect_flight_methods(ctx: ServerContext) -> FlightServerMethods: """ - This factory creates implementation of Flight RPC methods that realize the FlexFun server. + This factory creates implementation of Flight RPC methods that realize the FlexConnect server. - FlexFun Server hosts one or more functions developed externally, and linked to the server + FlexConnect Server hosts one or more functions developed externally, and linked to the server at runtime - during startup. :param ctx: server's context :return: new instance of Flight RPC server methods to integrate into the server """ - modules = list(ctx.settings.get(f"{_FLEXFUN_CONFIG_SECTION}.{_FLEXFUN_FUNCTION_LIST}") or []) - _LOGGER.info("flexfun_init", modules=modules) - registry = FlexFunRegistry().load(ctx, modules) + modules = list(ctx.settings.get(f"{_FLEX_CONNECT_CONFIG_SECTION}.{_FLEX_CONNECT_FUNCTION_LIST}") or []) + _LOGGER.info("flexconnect_init", modules=modules) + registry = FlexConnectFunctionRegistry().load(ctx, modules) - return _FlexFunServerMethods(ctx, registry) + return _FlexConnectServerMethods(ctx, registry) diff --git a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun.py b/gooddata-flexconnect/gooddata_flexconnect/function/function.py similarity index 70% rename from gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun.py rename to gooddata-flexconnect/gooddata_flexconnect/function/function.py index 5a72bf2d8..8929d57fd 100644 --- a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun.py +++ b/gooddata-flexconnect/gooddata_flexconnect/function/function.py @@ -6,7 +6,7 @@ from gooddata_flight_server import ArrowData, ServerContext -class FlexFun(abc.ABC): +class FlexConnectFunction(abc.ABC): """ Interface for pluggable functions which can generate flights based on a set of parameters sent by the GoodData Cloud and its FlexQuery. @@ -33,7 +33,7 @@ class FlexFun(abc.ABC): the function _must_ inspect the `parameters` to actually determine what the caller is interested in. - Programming detail: a new instance of the FlexFun will be created + Programming detail: a new instance of the FlexConnectFunction class will be created for every call using the `create` method. """ @@ -50,19 +50,19 @@ class FlexFun(abc.ABC): Metadata: Optional[dict] = None """ Function MAY provide additional metadata about themselves. These then - influence how the FlexFun is used by and called from GoodData Cloud & FlexQuery. + influence how the function is used by and called from GoodData Cloud & FlexQuery. """ @classmethod - def create(cls) -> "FlexFun": + def create(cls) -> "FlexConnectFunction": """ - This method is called by FlexFun server in order to create a new instance - of FlexFun which should be used to service the call. + This method is called by FlexConnect server in order to create a new instance + of function which should be used to service the call. - If your FlexFun implementation has default constructor with no parameters or + If your function implementation has default constructor with no parameters or variable parameters, then you do not have to implement this method. - Overriding this method is typically needed when the instance of FlexFun needs + Overriding this method is typically needed when the instance of function needs to receive dependencies created during the one-off initialization done in `on_load`. @@ -71,7 +71,7 @@ def create(cls) -> "FlexFun": should be done in `on_load` method; all expensive operations should be done within the `call` itself. - :return: an instance of concrete FlexFun + :return: an instance of concrete function """ return cls() @@ -86,9 +86,9 @@ def call( Function call. :param parameters: parameters sent from the GoodData Cloud / FlexQuery. - :param columns: hints which columns _should_ be returned; the FlexFun may decide to ignore + :param columns: hints which columns _should_ be returned; the function may decide to ignore this and always return all columns. The extraneous columns will be trimmed when received - by FlexQuery. See comments of FlexFun class to learn more. + by FlexQuery. See comments of `FlexConnectFunction` class to learn more. :param headers: Flight RPC headers :return: result of the call """ @@ -96,11 +96,11 @@ def call( def cancel(self) -> bool: """ - A FlexFun call may be cancelled by the server. It usually happens when the call takes - a long time to complete and the GoodData Cloud and its FlexQuery are no longer - interested in the result. + A FlexConnection function call may be cancelled by the server. It usually happens + when the call takes a long time to complete and the GoodData Cloud and its + FlexQuery are no longer interested in the result. - Implementing the cancellation is optional. If not implemented, the FlexFun server will + Implementing the cancellation is optional. If not implemented, the FlexConnect server will still pretend the entire call was cancelled - it's just that it will wait for the `call` to finish and then throw the results away. @@ -111,22 +111,22 @@ def cancel(self) -> bool: @staticmethod def on_load(ctx: ServerContext) -> None: """ - This method is called by FlexFun server at the time it discovers this FlexFun and + This method is called by FlexConnect server at the time it discovers this function and wants to load it and prepare it for calls. - This method is called exactly once for each discovered FlexFun; it is guaranteed that this - method will be called before first call to this FlexFun. If this method raises an error, + This method is called exactly once for each discovered function; it is guaranteed that this + method will be called before first call to this function. If this method raises an error, it will interrupt the server startup. - The intention of this method is to allow FlexFun perform one-off initialization of - some shared, reusable internals (usually some singletons). + The intention of this method is to allow the function implementation to perform + one-off initialization of some shared, reusable internals (usually some singletons). A very typical use of this method is to inspect the `settings` included in the context - this contains the settings loaded from config files/env variables provided to the server - at startup. If the FlexFun needs some external (custom) settings, the user can code + at startup. If the function needs some external (custom) settings, the user can code those into config files or env variables -> this method can act on them. - :param ctx: context of the server which hosts this FlexFun + :param ctx: context of the server which hosts this function :return: nothing """ return diff --git a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_registry.py b/gooddata-flexconnect/gooddata_flexconnect/function/function_registry.py similarity index 63% rename from gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_registry.py rename to gooddata-flexconnect/gooddata_flexconnect/function/function_registry.py index a5a87bb3a..7b4d530fe 100644 --- a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_registry.py +++ b/gooddata-flexconnect/gooddata_flexconnect/function/function_registry.py @@ -5,29 +5,29 @@ import structlog from gooddata_flight_server import ErrorInfo, ServerContext -from gooddata_flexfun.flexfun.flex_fun import FlexFun +from gooddata_flexconnect.function.function import FlexConnectFunction -class FlexFunRegistry: +class FlexConnectFunctionRegistry: """ - Registry for supported FlexFuns. + Registry for supported FlexConnect functions. """ def __init__(self) -> None: - self._logger = structlog.get_logger("gooddata_flexfun.registry") - self._fun_by_name: dict[str, type[FlexFun]] = {} + self._logger = structlog.get_logger("gooddata_flexconnect.registry") + self._fun_by_name: dict[str, type[FlexConnectFunction]] = {} self._fun_names: tuple[str, ...] = () self._loaded_modules: list[str] = [] @property - def flex_funs_names(self) -> tuple[str, ...]: + def function_names(self) -> tuple[str, ...]: """ :return: names of available functions """ return self._fun_names @property - def flex_funs(self) -> dict[str, type[FlexFun]]: + def functions(self) -> dict[str, type[FlexConnectFunction]]: """ :return: mapping of fun names to their classes """ @@ -40,24 +40,27 @@ def loaded_modules(self) -> list[str]: """ return self._loaded_modules - def _check_function(self, fun: type[FlexFun]) -> str: + def _check_function(self, fun: type[FlexConnectFunction]) -> str: # TODO: add name validation using regex if fun.Name is None or not len(fun.Name): raise ValueError( - f"FlexFun implemented in class {fun.__name__} does not specify name or specifies an empty name." + f"FlexConnect function implemented in class {fun.__name__} " + f"does not specify name or specifies an empty name." ) elif fun.Name in self._fun_by_name: raise ValueError( - f"FlexFun implemented in class {fun.__name__} specifies same name as " + f"FlexConnect function implemented in class {fun.__name__} specifies same name as " f"already registered fun from class {self._fun_by_name[fun.Name].__name__}." ) if fun.Schema is None: - raise ValueError(f"FlexFun '{fun.Name}' implemented in class {fun.__name__} does not specify schema.") + raise ValueError( + f"FlexConnect function '{fun.Name}' implemented in class {fun.__name__} d" f"oes not specify schema." + ) return fun.Name - def _initialize_and_register(self, ctx: ServerContext, fun: type[FlexFun]) -> None: + def _initialize_and_register(self, ctx: ServerContext, fun: type[FlexConnectFunction]) -> None: # this should be verified earlier and raise proper error assert fun.Name is not None @@ -65,9 +68,9 @@ def _initialize_and_register(self, ctx: ServerContext, fun: type[FlexFun]) -> No self._fun_by_name[fun.Name] = fun self._fun_names = tuple(self._fun_by_name.keys()) - def register(self, ctx: ServerContext, *funs: type[FlexFun]) -> "FlexFunRegistry": + def register(self, ctx: ServerContext, *funs: type[FlexConnectFunction]) -> "FlexConnectFunctionRegistry": """ - Register one or more FlexFuns. + Register one or more FlexConnect functions. :param ctx: server context to pass to the function's on_load :param funs: functions to register @@ -79,29 +82,29 @@ def register(self, ctx: ServerContext, *funs: type[FlexFun]) -> "FlexFunRegistry return self - def load(self, ctx: ServerContext, modules: Iterable[str]) -> "FlexFunRegistry": + def load(self, ctx: ServerContext, modules: Iterable[str]) -> "FlexConnectFunctionRegistry": """ - Loads FlexFuns from the provided python packages. The packages must be installed and + Loads FlexConnect functions from the provided python packages. The packages must be installed and importable. :param ctx: server context to pass to the function's on_load - :param modules: python modules that contain the FlexFun implementations + :param modules: python modules that contain the FlexConnect function implementations :return: self, for call chaining sakes :raises ModuleNotFoundError: when some package cannot be imported """ for module in modules: - self._logger.info("load_flex_funs", op_module=module) + self._logger.info("load_flexconnect_funs", op_module=module) op_module = importlib.import_module(module) loaded_funs: list[str] = [] for member in op_module.__dict__.values(): - if not isinstance(member, type) or not issubclass(member, FlexFun): + if not isinstance(member, type) or not issubclass(member, FlexConnectFunction): # filter out module members which are not classes that implement the - # FlexFun interface + # FlexConnectFunction interface continue - if member == FlexFun: - # the FlexFun class is likely imported in the module - + if member == FlexConnectFunction: + # the FlexConnectFunction class is likely imported in the module - # don't want that to interfere continue @@ -117,10 +120,10 @@ def load(self, ctx: ServerContext, modules: Iterable[str]) -> "FlexFunRegistry": if not len(loaded_funs): # log as warning just so that it stands out somewhat better - self._logger.warning("load_flex_funs_done", module=module, loaded_funs=[]) + self._logger.warning("load_flexconnect_funs_done", module=module, loaded_funs=[]) else: self._logger.info( - "load_flex_funs_done", + "load_flexconnect_funs_done", module=module, loaded_funs=loaded_funs, ) @@ -129,16 +132,16 @@ def load(self, ctx: ServerContext, modules: Iterable[str]) -> "FlexFunRegistry": return self - def create_function(self, name: str) -> FlexFun: + def create_function(self, name: str) -> FlexConnectFunction: """ - Creates a new instance of FlexFun with the provided name. + Creates a new instance of FlexConnect function with the provided name. If there is no function matching the name, this method will raise error containing ErrorInfo.bad_argument. - :return: an instance of FlexFun, ready to be called + :return: an instance of FlexConnect function, ready to be called """ fun = self._fun_by_name.get(name) if fun is None: - raise ErrorInfo.bad_argument(f"Unsupported FlexFun '{name}'.") + raise ErrorInfo.bad_argument(f"Unsupported FlexConnect function '{name}'.") return fun.create() diff --git a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_task.py b/gooddata-flexconnect/gooddata_flexconnect/function/function_task.py similarity index 73% rename from gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_task.py rename to gooddata-flexconnect/gooddata_flexconnect/function/function_task.py index a08d94b12..baf11bae5 100644 --- a/gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_task.py +++ b/gooddata-flexconnect/gooddata_flexconnect/function/function_task.py @@ -4,17 +4,17 @@ import structlog from gooddata_flight_server import FlightDataTaskResult, Task, TaskError, TaskResult -from gooddata_flexfun.flexfun.flex_fun import FlexFun +from gooddata_flexconnect.function.function import FlexConnectFunction -_LOGGER = structlog.get_logger("gooddata_flexfun.task") +_LOGGER = structlog.get_logger("gooddata_flexconnect.task") -class FlexFunTask(Task): +class FlexConnectFunctionTask(Task): __slots__ = ("_fun", "_parameters", "_columns", "_headers") def __init__( self, - fun: FlexFun, + fun: FlexConnectFunction, parameters: dict, columns: Optional[tuple[str, ...]], headers: dict[str, list[str]], @@ -29,7 +29,7 @@ def __init__( self._columns = columns self._headers = headers - _LOGGER.info("flexfun_task_created", fun=fun.Name, task_id=self._task_id) + _LOGGER.info("flexconnect_task_created", fun=fun.Name, task_id=self._task_id) @property def fun_name(self) -> Optional[str]: @@ -37,7 +37,7 @@ def fun_name(self) -> Optional[str]: def run(self) -> Union[TaskResult, TaskError]: structlog.contextvars.bind_contextvars(fun=self._fun.Name, task_id=self._task_id) - _LOGGER.info("flexfun_task_run") + _LOGGER.info("flexconnect_task_run") result = self._fun.call( parameters=self._parameters, @@ -48,6 +48,6 @@ def run(self) -> Union[TaskResult, TaskError]: return FlightDataTaskResult.for_data(result) def on_task_cancel(self) -> None: - _LOGGER.info("flexfun_task_cancel", fun=self._fun.Name, task_id=self._task_id) + _LOGGER.info("flexconnect_task_cancel", fun=self._fun.Name, task_id=self._task_id) self._fun.cancel() diff --git a/gooddata-flexconnect/gooddata_flexconnect/py.typed b/gooddata-flexconnect/gooddata_flexconnect/py.typed new file mode 100644 index 000000000..1d9f6f89d --- /dev/null +++ b/gooddata-flexconnect/gooddata_flexconnect/py.typed @@ -0,0 +1 @@ +Mark package as supporting typing. See https://www.python.org/dev/peps/pep-0561/ for details. diff --git a/gooddata-flexfun/json_schemas/execution-context/attribute.json b/gooddata-flexconnect/json_schemas/execution-context/attribute.json similarity index 100% rename from gooddata-flexfun/json_schemas/execution-context/attribute.json rename to gooddata-flexconnect/json_schemas/execution-context/attribute.json diff --git a/gooddata-flexfun/json_schemas/execution-context/date-granularity.json b/gooddata-flexconnect/json_schemas/execution-context/date-granularity.json similarity index 100% rename from gooddata-flexfun/json_schemas/execution-context/date-granularity.json rename to gooddata-flexconnect/json_schemas/execution-context/date-granularity.json diff --git a/gooddata-flexfun/json_schemas/execution-context/execution-context.json b/gooddata-flexconnect/json_schemas/execution-context/execution-context.json similarity index 93% rename from gooddata-flexfun/json_schemas/execution-context/execution-context.json rename to gooddata-flexconnect/json_schemas/execution-context/execution-context.json index a07d6995f..b46a66c44 100644 --- a/gooddata-flexfun/json_schemas/execution-context/execution-context.json +++ b/gooddata-flexconnect/json_schemas/execution-context/execution-context.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "execution-context/execution-context.json", "title": "Execution Context", - "description": "The context in which the FlexFun is executed. Contains all the information needed to execute the FlexFun.", + "description": "The context in which the FlexConnect function is executed. Contains all the information needed to execute the function.", "properties": { "executionType": { "enum": [ @@ -13,15 +13,15 @@ }, "organizationId": { "type": "string", - "description": "The ID of the organization that the FlexFun is executed in." + "description": "The ID of the organization that the FlexConnect function is executed in." }, "workspaceId": { "type": "string", - "description": "The ID of the workspace that the FlexFun is executed in." + "description": "The ID of the workspace that the FlexConnect function is executed in." }, "userId": { "type": "string", - "description": "The ID of the user that invoked the FlexFun." + "description": "The ID of the user that invoked the FlexConnect function." }, "timestamp": { "type": "string", diff --git a/gooddata-flexfun/json_schemas/execution-context/filter.json b/gooddata-flexconnect/json_schemas/execution-context/filter.json similarity index 100% rename from gooddata-flexfun/json_schemas/execution-context/filter.json rename to gooddata-flexconnect/json_schemas/execution-context/filter.json diff --git a/gooddata-flexfun/json_schemas/execution-context/label-elements/depends-on-date-filter.json b/gooddata-flexconnect/json_schemas/execution-context/label-elements/depends-on-date-filter.json similarity index 100% rename from gooddata-flexfun/json_schemas/execution-context/label-elements/depends-on-date-filter.json rename to gooddata-flexconnect/json_schemas/execution-context/label-elements/depends-on-date-filter.json diff --git a/gooddata-flexfun/json_schemas/execution-context/label-elements/depends-on.json b/gooddata-flexconnect/json_schemas/execution-context/label-elements/depends-on.json similarity index 100% rename from gooddata-flexfun/json_schemas/execution-context/label-elements/depends-on.json rename to gooddata-flexconnect/json_schemas/execution-context/label-elements/depends-on.json diff --git a/gooddata-flexfun/json_schemas/execution-context/label-elements/execution-request.json b/gooddata-flexconnect/json_schemas/execution-context/label-elements/execution-request.json similarity index 100% rename from gooddata-flexfun/json_schemas/execution-context/label-elements/execution-request.json rename to gooddata-flexconnect/json_schemas/execution-context/label-elements/execution-request.json diff --git a/gooddata-flexfun/mypy.ini b/gooddata-flexconnect/mypy.ini similarity index 100% rename from gooddata-flexfun/mypy.ini rename to gooddata-flexconnect/mypy.ini diff --git a/gooddata-flexfun/requirements.txt b/gooddata-flexconnect/requirements.txt similarity index 100% rename from gooddata-flexfun/requirements.txt rename to gooddata-flexconnect/requirements.txt diff --git a/gooddata-flexfun/setup.py b/gooddata-flexconnect/setup.py similarity index 89% rename from gooddata-flexfun/setup.py rename to gooddata-flexconnect/setup.py index 8c5b57c1d..b19b03331 100644 --- a/gooddata-flexfun/setup.py +++ b/gooddata-flexconnect/setup.py @@ -16,8 +16,8 @@ ] setup( - name="gooddata-flexfun", - description="Set of gooddata-flight-server methods to host custom functions for GoodData Cloud", + name="gooddata-flexconnect", + description="Build your own data source for GoodData Cloud and GoodData Cloud Native.", long_description=long_description, long_description_content_type="text/markdown", version="1.30.0", @@ -31,7 +31,7 @@ include_package_data=True, python_requires=">=3.9.0", project_urls={ - "Documentation": "https://gooddata-flexfun.readthedocs.io/en/v1.30.0", + "Documentation": "https://gooddata-flexconnect.readthedocs.io/en/v1.30.0", "Source": "https://github.com/gooddata/gooddata-python-sdk", }, classifiers=[ diff --git a/gooddata-flexfun/test-requirements.txt b/gooddata-flexconnect/test-requirements.txt similarity index 100% rename from gooddata-flexfun/test-requirements.txt rename to gooddata-flexconnect/test-requirements.txt diff --git a/gooddata-flexfun/tests/__init__.py b/gooddata-flexconnect/tests/__init__.py similarity index 100% rename from gooddata-flexfun/tests/__init__.py rename to gooddata-flexconnect/tests/__init__.py diff --git a/gooddata-flexfun/tests/assert_error_info.py b/gooddata-flexconnect/tests/assert_error_info.py similarity index 100% rename from gooddata-flexfun/tests/assert_error_info.py rename to gooddata-flexconnect/tests/assert_error_info.py diff --git a/gooddata-flexfun/tests/flexfun/__init__.py b/gooddata-flexconnect/tests/function/__init__.py similarity index 100% rename from gooddata-flexfun/tests/flexfun/__init__.py rename to gooddata-flexconnect/tests/function/__init__.py diff --git a/gooddata-flexfun/tests/flexfun/conftest.py b/gooddata-flexconnect/tests/function/conftest.py similarity index 100% rename from gooddata-flexfun/tests/flexfun/conftest.py rename to gooddata-flexconnect/tests/function/conftest.py diff --git a/gooddata-flexfun/tests/flexfun/test_flex_fun_execution_context.py b/gooddata-flexconnect/tests/function/test_flex_fun_execution_context.py similarity index 97% rename from gooddata-flexfun/tests/flexfun/test_flex_fun_execution_context.py rename to gooddata-flexconnect/tests/function/test_flex_fun_execution_context.py index aaca69892..04c0e1f56 100644 --- a/gooddata-flexfun/tests/flexfun/test_flex_fun_execution_context.py +++ b/gooddata-flexconnect/tests/function/test_flex_fun_execution_context.py @@ -1,5 +1,5 @@ # (C) 2024 GoodData Corporation -from gooddata_flexfun.flexfun.flex_fun_execution_context import ( +from gooddata_flexconnect.function.execution_context import ( ExecutionContext, ExecutionContextAttribute, ExecutionContextNegativeAttributeFilter, diff --git a/gooddata-flexfun/tests/flexfun/test_registry.py b/gooddata-flexconnect/tests/function/test_registry.py similarity index 73% rename from gooddata-flexfun/tests/flexfun/test_registry.py rename to gooddata-flexconnect/tests/function/test_registry.py index 6013eb70e..4e6b74a50 100644 --- a/gooddata-flexfun/tests/flexfun/test_registry.py +++ b/gooddata-flexconnect/tests/function/test_registry.py @@ -1,11 +1,11 @@ # (C) 2024 GoodData Corporation import pyarrow.flight import pytest -from gooddata_flexfun.flexfun.flex_fun_registry import FlexFunRegistry +from gooddata_flexconnect.function.function_registry import FlexConnectFunctionRegistry from gooddata_flight_server import ErrorCode, ServerContext from tests.assert_error_info import assert_error_code -from tests.flexfun.testing_funs import Fun1, Fun2 +from tests.function.testing_funs import Fun1, Fun2 @pytest.fixture(scope="module") @@ -24,22 +24,22 @@ def fake_ctx(): def test_registry1(fake_ctx): - r = FlexFunRegistry() + r = FlexConnectFunctionRegistry() r.register(fake_ctx, Fun1) - assert "fun1" in r.flex_funs + assert "fun1" in r.functions def test_load_from_modules(fake_ctx): - r = FlexFunRegistry() + r = FlexConnectFunctionRegistry() # reset the indicator (previous tests may have registered this fun) Fun2.OnLoadCalled = False - r.load(fake_ctx, modules=["tests.flexfun.testing_funs"]) + r.load(fake_ctx, modules=["tests.function.testing_funs"]) - assert "fun1" in r.flex_funs - assert "fun2" in r.flex_funs - assert len(r.flex_funs) == 2 + assert "fun1" in r.functions + assert "fun2" in r.functions + assert len(r.functions) == 2 assert Fun2.OnLoadCalled is True assert isinstance(r.create_function("fun1"), Fun1) @@ -47,7 +47,7 @@ def test_load_from_modules(fake_ctx): def test_create_when_missing(): - r = FlexFunRegistry() + r = FlexConnectFunctionRegistry() with pytest.raises(pyarrow.flight.FlightServerError) as e: r.create_function("fun1") diff --git a/gooddata-flexfun/tests/flexfun/testing_funs.py b/gooddata-flexconnect/tests/function/testing_funs.py similarity index 84% rename from gooddata-flexfun/tests/flexfun/testing_funs.py rename to gooddata-flexconnect/tests/function/testing_funs.py index 85740c283..e18a75b63 100644 --- a/gooddata-flexfun/tests/flexfun/testing_funs.py +++ b/gooddata-flexconnect/tests/function/testing_funs.py @@ -1,11 +1,11 @@ # (C) 2024 GoodData Corporation import pyarrow -from gooddata_flexfun.flexfun.flex_fun import FlexFun +from gooddata_flexconnect.function.function import FlexConnectFunction from gooddata_flight_server import ArrowData, ServerContext -class Fun1(FlexFun): +class Fun1(FlexConnectFunction): Name = "fun1" Schema = pyarrow.schema(fields=[pyarrow.field("col", pyarrow.int64())]) @@ -18,7 +18,7 @@ def call( pass -class Fun2(FlexFun): +class Fun2(FlexConnectFunction): Name = "fun2" Schema = pyarrow.schema(fields=[pyarrow.field("col", pyarrow.string())]) diff --git a/gooddata-flexfun/tests/json_schemas/__init__.py b/gooddata-flexconnect/tests/json_schemas/__init__.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/__init__.py rename to gooddata-flexconnect/tests/json_schemas/__init__.py diff --git a/gooddata-flexfun/tests/json_schemas/conftest.py b/gooddata-flexconnect/tests/json_schemas/conftest.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/conftest.py rename to gooddata-flexconnect/tests/json_schemas/conftest.py diff --git a/gooddata-flexfun/tests/json_schemas/test_attribute_schema.py b/gooddata-flexconnect/tests/json_schemas/test_attribute_schema.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/test_attribute_schema.py rename to gooddata-flexconnect/tests/json_schemas/test_attribute_schema.py diff --git a/gooddata-flexfun/tests/json_schemas/test_depends_on_date_filter_schema.py b/gooddata-flexconnect/tests/json_schemas/test_depends_on_date_filter_schema.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/test_depends_on_date_filter_schema.py rename to gooddata-flexconnect/tests/json_schemas/test_depends_on_date_filter_schema.py diff --git a/gooddata-flexfun/tests/json_schemas/test_depends_on_schema.py b/gooddata-flexconnect/tests/json_schemas/test_depends_on_schema.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/test_depends_on_schema.py rename to gooddata-flexconnect/tests/json_schemas/test_depends_on_schema.py diff --git a/gooddata-flexfun/tests/json_schemas/test_execution_context_schema.py b/gooddata-flexconnect/tests/json_schemas/test_execution_context_schema.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/test_execution_context_schema.py rename to gooddata-flexconnect/tests/json_schemas/test_execution_context_schema.py diff --git a/gooddata-flexfun/tests/json_schemas/test_filter_schema.py b/gooddata-flexconnect/tests/json_schemas/test_filter_schema.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/test_filter_schema.py rename to gooddata-flexconnect/tests/json_schemas/test_filter_schema.py diff --git a/gooddata-flexfun/tests/json_schemas/test_label_elements_execution_request_schema.py b/gooddata-flexconnect/tests/json_schemas/test_label_elements_execution_request_schema.py similarity index 100% rename from gooddata-flexfun/tests/json_schemas/test_label_elements_execution_request_schema.py rename to gooddata-flexconnect/tests/json_schemas/test_label_elements_execution_request_schema.py diff --git a/gooddata-flexfun/tests/server/__init__.py b/gooddata-flexconnect/tests/server/__init__.py similarity index 100% rename from gooddata-flexfun/tests/server/__init__.py rename to gooddata-flexconnect/tests/server/__init__.py diff --git a/gooddata-flexfun/tests/server/conftest.py b/gooddata-flexconnect/tests/server/conftest.py similarity index 91% rename from gooddata-flexfun/tests/server/conftest.py rename to gooddata-flexconnect/tests/server/conftest.py index 9433775ff..6b5e5ef05 100644 --- a/gooddata-flexfun/tests/server/conftest.py +++ b/gooddata-flexconnect/tests/server/conftest.py @@ -8,8 +8,8 @@ from typing import Union import pytest -from gooddata_flexfun.flexfun.flight_methods import ( - create_flexfun_flight_methods, +from gooddata_flexconnect.function.flight_methods import ( + create_flexconnect_flight_methods, ) from gooddata_flight_server import FlightServerMethods, FlightServerMethodsFactory, GoodDataFlightServer, create_server @@ -71,7 +71,7 @@ def server( @contextlib.contextmanager -def flexfun_server( +def flexconnect_server( modules: Iterable[str], tls: bool = False, mtls: bool = False, @@ -79,9 +79,9 @@ def flexfun_server( envvar = ", ".join([f'"{module}"' for module in modules]) envvar = f"[{envvar}]" - os.environ["GOODDATA_FLIGHT_FLEXFUN__FUNCTIONS"] = envvar + os.environ["GOODDATA_FLIGHT_FLEXCONNECT__FUNCTIONS"] = envvar - with server(create_flexfun_flight_methods, tls, mtls) as s: + with server(create_flexconnect_flight_methods, tls, mtls) as s: yield s diff --git a/gooddata-flexfun/tests/server/funs/__init__.py b/gooddata-flexconnect/tests/server/funs/__init__.py similarity index 100% rename from gooddata-flexfun/tests/server/funs/__init__.py rename to gooddata-flexconnect/tests/server/funs/__init__.py diff --git a/gooddata-flexfun/tests/server/funs/fun1.py b/gooddata-flexconnect/tests/server/funs/fun1.py similarity index 86% rename from gooddata-flexfun/tests/server/funs/fun1.py rename to gooddata-flexconnect/tests/server/funs/fun1.py index 8529e3e79..106be05b4 100644 --- a/gooddata-flexfun/tests/server/funs/fun1.py +++ b/gooddata-flexconnect/tests/server/funs/fun1.py @@ -1,11 +1,11 @@ # (C) 2024 GoodData Corporation import pyarrow -from gooddata_flexfun.flexfun.flex_fun import FlexFun +from gooddata_flexconnect.function.function import FlexConnectFunction from gooddata_flight_server import ArrowData -class _SimpleFun(FlexFun): +class _SimpleFun(FlexConnectFunction): Name = "SimpleFun" Schema = pyarrow.schema( fields=[ diff --git a/gooddata-flexfun/tests/server/funs/fun2.py b/gooddata-flexconnect/tests/server/funs/fun2.py similarity index 90% rename from gooddata-flexfun/tests/server/funs/fun2.py rename to gooddata-flexconnect/tests/server/funs/fun2.py index 09a335b9f..393e15999 100644 --- a/gooddata-flexfun/tests/server/funs/fun2.py +++ b/gooddata-flexconnect/tests/server/funs/fun2.py @@ -2,13 +2,13 @@ from typing import Optional import pyarrow -from gooddata_flexfun.flexfun.flex_fun import FlexFun +from gooddata_flexconnect.function.function import FlexConnectFunction from gooddata_flight_server import ArrowData, ServerContext _DATA: Optional[pyarrow.Table] = None -class _SimpleFun(FlexFun): +class _SimpleFun(FlexConnectFunction): Name = "SimpleFun" Schema = pyarrow.schema( fields=[ diff --git a/gooddata-flexfun/tests/server/test_flexfun_server.py b/gooddata-flexconnect/tests/server/test_flexconnect_server.py similarity index 92% rename from gooddata-flexfun/tests/server/test_flexfun_server.py rename to gooddata-flexconnect/tests/server/test_flexconnect_server.py index c563ed379..a18969bbc 100644 --- a/gooddata-flexfun/tests/server/test_flexfun_server.py +++ b/gooddata-flexconnect/tests/server/test_flexconnect_server.py @@ -2,11 +2,11 @@ import orjson import pyarrow.flight -from tests.server.conftest import flexfun_server +from tests.server.conftest import flexconnect_server def test_basic_function(): - with flexfun_server(["tests.server.funs.fun1"]) as s: + with flexconnect_server(["tests.server.funs.fun1"]) as s: c = pyarrow.flight.FlightClient(s.location) fun_infos = list(c.list_flights()) assert len(fun_infos) == 1 @@ -40,7 +40,7 @@ def test_function_with_on_load(): The testing `fun2` is implemented in a way that is sets up the data to serve in the on_load. If on_load does not happen, the GetFlightInfo->DoGet would fail """ - with flexfun_server(["tests.server.funs.fun2"]) as s: + with flexconnect_server(["tests.server.funs.fun2"]) as s: c = pyarrow.flight.FlightClient(s.location) descriptor = pyarrow.flight.FlightDescriptor.for_command( orjson.dumps( @@ -59,7 +59,7 @@ def test_function_with_on_load(): def test_basic_function_tls(tls_ca_cert): - with flexfun_server(["tests.server.funs.fun1"], tls=True) as s: + with flexconnect_server(["tests.server.funs.fun1"], tls=True) as s: c = pyarrow.flight.FlightClient(s.location, tls_root_certs=tls_ca_cert) fun_infos = list(c.list_flights()) assert len(fun_infos) == 1 diff --git a/gooddata-flexfun/tests/server/tls/ca-cert.pem b/gooddata-flexconnect/tests/server/tls/ca-cert.pem similarity index 100% rename from gooddata-flexfun/tests/server/tls/ca-cert.pem rename to gooddata-flexconnect/tests/server/tls/ca-cert.pem diff --git a/gooddata-flexfun/tests/server/tls/client-cert.pem b/gooddata-flexconnect/tests/server/tls/client-cert.pem similarity index 100% rename from gooddata-flexfun/tests/server/tls/client-cert.pem rename to gooddata-flexconnect/tests/server/tls/client-cert.pem diff --git a/gooddata-flexfun/tests/server/tls/client-key.pem b/gooddata-flexconnect/tests/server/tls/client-key.pem similarity index 100% rename from gooddata-flexfun/tests/server/tls/client-key.pem rename to gooddata-flexconnect/tests/server/tls/client-key.pem diff --git a/gooddata-flexfun/tests/server/tls/server-cert.pem b/gooddata-flexconnect/tests/server/tls/server-cert.pem similarity index 100% rename from gooddata-flexfun/tests/server/tls/server-cert.pem rename to gooddata-flexconnect/tests/server/tls/server-cert.pem diff --git a/gooddata-flexfun/tests/server/tls/server-key.pem b/gooddata-flexconnect/tests/server/tls/server-key.pem similarity index 100% rename from gooddata-flexfun/tests/server/tls/server-key.pem rename to gooddata-flexconnect/tests/server/tls/server-key.pem diff --git a/gooddata-flexfun/tox.ini b/gooddata-flexconnect/tox.ini similarity index 86% rename from gooddata-flexfun/tox.ini rename to gooddata-flexconnect/tox.ini index 98c97621b..1911726c0 100644 --- a/gooddata-flexfun/tox.ini +++ b/gooddata-flexconnect/tox.ini @@ -14,7 +14,7 @@ deps = setenv= PYTHONDONTWRITEBYTECODE=1 commands = - pytest -v --cov=gooddata_flexfun --cov-report=xml tests {posargs} + pytest -v --cov=gooddata_flexconnect --cov-report=xml tests {posargs} [testenv:mypy] basepython = python3.12 @@ -25,7 +25,7 @@ deps = -e../gooddata-sdk -e../gooddata-flight-server commands = - mypy gooddata_flexfun + mypy gooddata_flexconnect [testenv:docs] basepython = python3.12 diff --git a/gooddata-flexfun/type-requirements.txt b/gooddata-flexconnect/type-requirements.txt similarity index 100% rename from gooddata-flexfun/type-requirements.txt rename to gooddata-flexconnect/type-requirements.txt diff --git a/gooddata-flexfun/README.md b/gooddata-flexfun/README.md deleted file mode 100644 index 0dae373df..000000000 --- a/gooddata-flexfun/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# GoodData FlexFun - -The GoodData FlexFun package is a GoodData Flight Server-compatible extension -that provides a framework for hosting custom pluggable functions called FlexFuns. -These can be used to act as a dataset for GoodData FlightRPC data sources. - -## What is a FlexFun? - -In essence, FlexFun is a class that provides a set of methods that can be called by the GoodData Cloud when data is -requested from the corresponding FlightRPC data source dataset. - -Each FlexFun provides a `name` used to identify the FlexFun in the GoodData Cloud. -FlexFuns can provide a set of `metadata` that can further influence when and how they are called by GoodData Cloud. -They also provide a `schema` (defined in terms -of [pyarrow.Schema](https://arrow.apache.org/docs/python/generated/pyarrow.Schema.html)) -that describes the shape of data that the FlexFun can provide. -Finally, they provide a set of methods that can be called to provide data in response to queries: -* `call` - called to provide data in response to a query -* `cancel` - called to cancel a query if GoodData Cloud decides to stop requesting data (e.g. if there is a timeout) -* `on_load` - called when the FlexFun is created before any `call` or `cancel` methods are called - -## Getting Started using the FlexFun Template - -The easiest way to get started writing FlexFuns is to use [the template repository](https://github.com/gooddata/gooddata-flexfun-template). -It provides a simple example of a FlexFun that can be used as a starting point for your own FlexFun with all the necessary infrastructure in place. -It also has a README that explains how to get started with the template and some general tips on how to write FlexFuns. - -## Getting started using the FlexFun package directly - -Install the package alongside the gooddata-flight-server using pip: - -```bash -pip install gooddata-flight-server gooddata-flexfun -``` - -Next, update the GoodData Flight Server configuration to load the FlexFun methods. - -```toml -[flexfun] - -# specify one or more modules that contain your FlexFun implementations -# -functions = [ - "flexfun.your_function" -] -``` - -Then when running the GoodData Flight Server, use the `--methods-provider` option to load the FlexFun methods. -For example: - -```bash -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -SERVER_CMD="${SCRIPT_DIR}/.venv/bin/gooddata-flight-server" - -export PYTHONPATH="${SCRIPT_DIR}/src" -export CONFIG_ENV="${1:-dev}" - -$SERVER_CMD start \ - --methods-provider gooddata_flexfun \ - --config \ - config/${CONFIG_ENV}.server.toml \ - config/flexfun.config.toml \ - --logging-config config/default.logging.ini \ - --dev-log -``` - -This will start the GoodData Flight Server with the FlexFun methods loaded. diff --git a/gooddata-flight-server/README.md b/gooddata-flight-server/README.md index 106deb9d8..15373c292 100644 --- a/gooddata-flight-server/README.md +++ b/gooddata-flight-server/README.md @@ -581,8 +581,8 @@ server will expose two HTTP endpoints: - `/ready` - indicates whether the server is up and ready to serve requests The endpoint will respond with status `500` if the server is not ready. Otherwise will respond with - `202`. The server is deemed ready when all its modules (which includes your FlexFunctions) are - up and the Flight RPC server is 'unlocked' to handle requests. + `202`. The server is deemed ready when all its modules are up and the Flight RPC server is + 'unlocked' to handle requests. - `/live` - indicates whether the server is still alive and can be used. The liveness is determined from the status of the modules. diff --git a/gooddata-flight-server/gooddata_flight_server/server/auth/token_verifier_factory.py b/gooddata-flight-server/gooddata_flight_server/server/auth/token_verifier_factory.py index b8410d128..ff9bbbd55 100644 --- a/gooddata-flight-server/gooddata_flight_server/server/auth/token_verifier_factory.py +++ b/gooddata-flight-server/gooddata_flight_server/server/auth/token_verifier_factory.py @@ -22,7 +22,7 @@ def _import_verification_strategy(module_name: str) -> type[TokenVerificationStr continue if member == TokenVerificationStrategy: - # the FlexFun class is likely imported in the module - + # the TokenVerificationStrategy class is likely imported in the module - # don't want that to interfere continue diff --git a/gooddata-flight-server/gooddata_flight_server/server/flight_rpc/server_methods.py b/gooddata-flight-server/gooddata_flight_server/server/flight_rpc/server_methods.py index b65dbf75e..b83001506 100644 --- a/gooddata-flight-server/gooddata_flight_server/server/flight_rpc/server_methods.py +++ b/gooddata-flight-server/gooddata_flight_server/server/flight_rpc/server_methods.py @@ -91,7 +91,7 @@ def do_get_task_result( if task_result.cancelled: raise ErrorInfo.for_reason( ErrorCode.COMMAND_CANCELLED, - f"FlexFun invocation was cancelled. Invocation task was: '{task_result.task_id}'.", + f"FlexConnect function invocation was cancelled. Invocation task was: '{task_result.task_id}'.", ).to_server_error() result = task_result.result diff --git a/pyproject.toml b/pyproject.toml index 1c1f43943..753e8337e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ target-version = "py39" "gooddata-dbt/gooddata_dbt/__init__.py" = ["F401"] "gooddata-fdw/gooddata_fdw/__init__.py" = ["F401"] "gooddata-flight-server/gooddata_flight_server/__init__.py" = ["F401"] -"gooddata-flexfun/gooddata_flexfun/__init__.py" = ["F401"] +"gooddata-flexconnect/gooddata_flexconnect/__init__.py" = ["F401"] [tool.ruff.format] exclude = ['(gooddata-api-client|.*\.snapshot\..*)'] @@ -121,18 +121,18 @@ src = "gooddata-flight-server/setup.py" search = "version=\"{current_version}\"" [[tool.tbump.file]] -# gooddata-flexfun setup.py -src = "gooddata-flexfun/setup.py" +# gooddata-flexconnect setup.py +src = "gooddata-flexconnect/setup.py" search = "version=\"{current_version}\"" [[tool.tbump.file]] -# gooddata-flexfun setup.py dependency -src = "gooddata-flexfun/setup.py" +# gooddata-flexconnect setup.py dependency +src = "gooddata-flexconnect/setup.py" search = "gooddata-sdk~={current_version}" [[tool.tbump.file]] -# gooddata-flexfun setup.py dependency -src = "gooddata-flexfun/setup.py" +# gooddata-flexconnect setup.py dependency +src = "gooddata-flexconnect/setup.py" search = "gooddata-flight-server~={current_version}" [[tool.tbump.file]] @@ -176,8 +176,8 @@ src = "gooddata-flight-server/setup.py" search = '"Documentation": "https://gooddata-flight-server.readthedocs.io/en/v{current_version}"' [[tool.tbump.file]] -src = "gooddata-flexfun/setup.py" -search = '"Documentation": "https://gooddata-flexfun.readthedocs.io/en/v{current_version}"' +src = "gooddata-flexconnect/setup.py" +search = '"Documentation": "https://gooddata-flexconnect.readthedocs.io/en/v{current_version}"' # You can specify a list of commands to # run after the files have been patched