chore: remove blackbox testing (#1976)

README.md

@@ -1,5 +1,4 @@
 [![AsyncAPI Modelina](./docs/img/readme-banner.png)](https://www.modelina.org)
-[![blackbox pipeline status](<https://img.shields.io/github/actions/workflow/status/asyncapi/modelina/blackbox-testing.yml?label=blackbox%20testing>)](https://github.com/asyncapi/modelina/actions/workflows/blackbox-testing.yml?query=branch%3Amaster++)
 [![Coverage Status](https://coveralls.io/repos/github/asyncapi/modelina/badge.svg?branch=master)](https://coveralls.io/github/asyncapi/modelina?branch=master)
 [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
 ![Maintenance score](https://img.shields.io/npms-io/maintenance-score/@asyncapi/modelina)

docs/contributing.md

@@ -132,7 +132,6 @@ Time to adapt the tests, cause without tests, it's just an empty promise. The te
 4. Adapt the [generator tests](../test/generators/template/TemplateGenerator.spec.ts)
 5. Adapt the [renderer tests](../test/generators/template/TemplateRenderer.spec.ts)
 6. Add your generator to the [FileGenerators test](../test/generators/FileGenerators.spec.ts) to ensure the models are accurately written to files.
-7. Lastly, we have (arguably) the most important tests, [the BlackBox tests](./development.md#blackbox-testing). They are to ensure that real-world inputs generate usable models that do not contain syntax errors. You can read more about the BlackBox tests [here](./development.md#blackbox-testing).
 
 Lastly, we need to adapt some of the docs to showcase your new awesome generator! Cause if the users cant find it, it dont exist.
 1. Add your [generator specific documentation under languages](./languages/) and add it to the [list of generators](./README.md#languages)
@@ -163,7 +162,6 @@ That said, here is a general rundown on what's triggered by each PR:
 
 - We inherit all [AsyncAPI core GitHub workflows](https://github.com/asyncapi/.github/tree/master/.github/workflows), including the most important one:
     - [A standard PR workflow](https://github.com/asyncapi/.github/blob/master/.github/workflows/if-nodejs-pr-testing.yml) which ensures that the following commands need to succeed: `npm run test`, `npm run lint`, and `npm run generate:assets`.
-- [BlackBox testing](https://github.com/asyncapi/modelina/tree/master/test/blackbox) has its [own workflow](https://github.com/asyncapi/modelina/blob/master/.github/workflows/blackbox-testing.yml) which ensures that all supported inputs generate syntactically correct outputs to any of the output languages. This check takes a while (usually +5 minutes). Generally, you don't need to worry about this one, unless the code were to suddenly generate syntactically incorrect code (we will guide you if this happens). 
 - [Coverall](https://github.com/asyncapi/modelina/blob/master/.github/workflows/coverall.yml) ensures we get test coverage statistics in each PR, thus ensuring we see how it affects overall test coverage. It creates a comment on the PR with the coverage status.
 - [SonarCloud](https://sonarcloud.io/dashboard?id=asyncapi_generator-model-sdk) runs a code analysis to ensure no bugs, security concerns, code smells, or duplicated code blocks. Make sure you address any concerns found by this bot, because it generates a comment to the PR if it finds any issue.
 

docs/development.md

@@ -9,7 +9,6 @@ You can either build the image and run the needed commands manually or rather us
 
 - `npm run docker:build` builds the docker image with the tag `asyncapi/modelina` (the rest of the scripts run this one as well).
 - `npm run docker:test` runs the main test suite.
-- `npm run docker:test:blackbox` runs the BlackBox test suite.
 
 ## Environment setup
 
@@ -21,6 +20,3 @@ To setup the environment follow these steps:
    - You can update snapshots by running `npm run test -- -u`
 4. Make sure code is well formatted and secure with eslint by running `npm run lint`, you can also auto format your code with `npm run format`
 
-## BlackBox testing
-
-We have several BlackBox tests that are run separately from the `npm run test` script. Please refer to the [BlackBox documentation](../test/blackbox) for further information.

package.json

@@ -79,29 +79,17 @@
     "build:cli": "npm pack --pack-destination modelina-cli && node modelina-cli/scripts/renameModelinaPack.js && cd modelina-cli && npm i && npm run build",
     "docker:build": "docker compose run --build modelina npm install",
     "docker:test": "docker compose run modelina npm run test",
-    "docker:test:blackbox": "docker compose run modelina npm run test:blackbox",
     "test": "npm run test:library && npm run test:examples && npm run test:cli",
     "test:cli": "cd modelina-cli && npm run test",
     "test:update": "npm run test:library -- -u && npm run test:examples:update",
-    "test:library": "cross-env CI=true jest --coverage --testPathIgnorePatterns ./test/blackbox --testPathIgnorePatterns ./modelina-website --testPathIgnorePatterns ./test/runtime --testPathIgnorePatterns ./examples",
+    "test:library": "cross-env CI=true jest --coverage --testPathIgnorePatterns ./modelina-website --testPathIgnorePatterns ./test/runtime --testPathIgnorePatterns ./examples",
     "test:examples": "npm run test:examples:regular && npm run test:examples:websites",
     "test:examples:update": "npm run test:examples:regular -- -u && npm run test:examples:websites -- -u",
     "test:examples:regular": "cross-env CI=true jest ./examples --testPathIgnorePatterns ./examples/integrate-with-react --testPathIgnorePatterns ./examples/integrate-with-next",
     "test:examples:websites": "npm run test:examples:react && npm run test:examples:next",
     "test:examples:websites:update": "npm run test:examples:react -- -u && npm run test:examples:next -- -u",
     "test:examples:react": "cd ./examples/integrate-with-react && npm i && npm run test",
     "test:examples:next": "cd ./examples/integrate-with-next && npm i && npm run test",
-    "test:blackbox": "concurrently --group -n csharp,go,java,javascript,php,python,rust,typescript,kotlin,cplusplus \"npm run test:blackbox:csharp\" \"npm run test:blackbox:go\" \"npm run test:blackbox:java\" \"npm run test:blackbox:javascript\" \"npm run test:blackbox:php\" \"npm run test:blackbox:python\" \"npm run test:blackbox:rust\" \"npm run test:blackbox:typescript\" \"npm run test:blackbox:kotlin\" \"npm run test:blackbox:cplusplus\"",
-    "test:blackbox:csharp": "cross-env CI=true jest ./test/blackbox/blackbox-csharp.spec.ts",
-    "test:blackbox:go": "cross-env CI=true jest ./test/blackbox/blackbox-go.spec.ts",
-    "test:blackbox:javascript": "cross-env CI=true jest ./test/blackbox/blackbox-javascript.spec.ts",
-    "test:blackbox:php": "cross-env CI=true jest ./test/blackbox/blackbox-php.spec.ts",
-    "test:blackbox:python": "cross-env CI=true jest ./test/blackbox/blackbox-python.spec.ts",
-    "test:blackbox:rust": "cross-env CI=true jest ./test/blackbox/blackbox-rust.spec.ts",
-    "test:blackbox:typescript": "cross-env CI=true jest ./test/blackbox/blackbox-typescript.spec.ts",
-    "test:blackbox:kotlin": "cross-env CI=true jest ./test/blackbox/blackbox-kotlin.spec.ts",
-    "test:blackbox:cplusplus": "cross-env CI=true jest ./test/blackbox/blackbox-cplusplus.spec.ts",
-    "test:blackbox:java": "cross-env CI=true jest ./test/blackbox/blackbox-java.spec.ts",
     "test:runtime:java": "cross-env CI=true jest ./test/runtime/runtime-java.spec.ts",
     "test:runtime:python": "cross-env CI=true jest ./test/runtime/runtime-python.spec.ts",
     "generate:runtime:python": "cross-env CI=true ts-node ./test/runtime/runtime-python.ts",

src/helpers/FileHelpers.ts

@@ -37,7 +37,7 @@ export class FileHelpers {
         /**
          * It happens that the promise is resolved before the file is actually written to.
          *
-         * This often happen if the file system is swamped with write requests in either benchmarks or in our blackbox tests.
+         * This often happen if the file system is swamped with write requests in either benchmarks
          *
          * To avoid this we dont resolve until we are sure the file is written and exists.
          */

test/TestUtils/GeneralUtils.ts

@@ -0,0 +1,28 @@
+/* eslint-disable no-undef */
+import { promisify } from 'util';
+import { exec } from 'child_process';
+const promiseExec = promisify(exec);
+
+/**
+ * Execute a command and if any errors occur reject the promise.
+ *
+ * @param command
+ */
+export async function execCommand(
+  command: string,
+  allowStdError = false
+): Promise<void> {
+  try {
+    const { stderr } = await promiseExec(command);
+    if (stderr !== '') {
+      if (!allowStdError) {
+        return Promise.reject(stderr);
+      }
+      // eslint-disable-next-line no-console
+      console.error(stderr);
+    }
+    return Promise.resolve();
+  } catch (e: any) {
+    return Promise.reject(`${e.stack}; Stdout: ${e.stdout}`);
+  }
+}