diff --git a/Jenkinsfile b/Jenkinsfile index 232f960b321..e0d30fc9a65 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -158,7 +158,7 @@ pipeline { } } - stage('Integration Tests') { + stage('Integration Tests (without flaky tests)') { steps { sh './gradlew --console=plain integrationTest' } @@ -176,5 +176,26 @@ pipeline { } } } + + stage('Integration Tests (flaky tests only)') { + steps { + warnError("Integration Tests Failed") { // if this errs, mark the build unstable, not failed. + sh './gradlew --console=plain integrationTestFlaky' + } + } + post { + always { + // Gradle generates both a HTML report of the unit tests to `build/reports/tests/*` + // and XML reports to `build/test-results/*`. + // We need to upload the XML reports for visualization in Jenkins. + // + // See https://docs.gradle.org/current/userguide/java_testing.html#test_reporting + junit testResults: '**/build/test-results/integrationTestFlaky/*.xml', allowEmptyResults: true + // Jenkins truncates large test outputs, so archive it as well. + tar file: 'build/integrationTestFlaky-results.tgz', archive: true, compress: true, overwrite: true, + glob: '**/build/test-results/integrationTestFlaky/*.xml' + } + } + } } } diff --git a/engine-tests/build.gradle.kts b/engine-tests/build.gradle.kts index c5168a9f205..ce70c31ce6e 100644 --- a/engine-tests/build.gradle.kts +++ b/engine-tests/build.gradle.kts @@ -2,6 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 // Engine tests are split out due to otherwise quirky project dependency issues with module tests extending engine tests +// while locally all tests should be run, when building via github, tests can be run separated in the github pipeline +// file. For integration tests, a tag "flaky" was introduced to mark tests which frequently fail pipelines +// gradle test +// gradle --console=plain unitTest +// gradle --console=plain integrationTest +// gradle --console=plain integrationTestFlaky + plugins { id("java-library") id("org.jetbrains.gradle.plugin.idea-ext") @@ -105,7 +112,7 @@ tasks.register("unitTest") { group = "Verification" description = "Runs unit tests (fast)" useJUnitPlatform { - excludeTags = setOf("MteTest", "TteTest") + excludeTags("MteTest", "TteTest") } systemProperty("junit.jupiter.execution.timeout.default", "1m") } @@ -113,10 +120,22 @@ tasks.register("unitTest") { tasks.register("integrationTest") { dependsOn(tasks.getByPath(":extractNatives")) group = "Verification" - description = "Runs integration tests (slow) tagged with 'MteTest' or 'TteTest'" + description = "Runs integration tests (slow) tagged with 'MteTest' or 'TteTest', exclude tests tagged 'flaky'." + + useJUnitPlatform { + excludeTags("flaky") + includeTags("MteTest", "TteTest") + } + systemProperty("junit.jupiter.execution.timeout.default", "5m") +} + +tasks.register("integrationTestFlaky") { + dependsOn(tasks.getByPath(":extractNatives")) + group = "Verification" + description = "Runs integration tests tagged with 'flaky' and either 'MteTest' or 'TteTest'." useJUnitPlatform { - includeTags = setOf("MteTest", "TteTest") + includeTags("MteTest & flaky", "TteTest & flaky") } systemProperty("junit.jupiter.execution.timeout.default", "5m") } diff --git a/engine-tests/src/test/java/org/terasology/engine/integrationenvironment/ExampleTest.java b/engine-tests/src/test/java/org/terasology/engine/integrationenvironment/ExampleTest.java index adfae966a71..4dbdad5591d 100644 --- a/engine-tests/src/test/java/org/terasology/engine/integrationenvironment/ExampleTest.java +++ b/engine-tests/src/test/java/org/terasology/engine/integrationenvironment/ExampleTest.java @@ -5,6 +5,7 @@ import com.google.common.collect.Lists; import org.joml.Vector3i; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +39,7 @@ public class ExampleTest { private ModuleTestingHelper helper; @Test + @Tag("flaky") public void testClientCreation() { logger.info("Starting test 'testClientCreation'"); Assertions.assertDoesNotThrow(helper::createClient); @@ -45,6 +47,7 @@ public void testClientCreation() { } @Test + @Tag("flaky") public void testClientConnection() throws IOException { int currentClients = Lists.newArrayList(entityManager.getEntitiesWith(ClientComponent.class)).size();