update parser to find includes

add tests for $bitsAllClear

audits/src/main/kotlin/dev/morphia/audits/RstAuditor.kt

@@ -11,20 +11,21 @@ import kotlin.collections.Map.Entry
 import org.jboss.forge.roaster.model.source.MethodSource
 import org.jboss.forge.roaster.model.source.ParameterSource
 
-object RstAuditor {
-    val auditRoot by lazy {
-        var gitRoot = File(".").canonicalFile
-        while (!File(gitRoot, ".git").exists()) gitRoot = gitRoot.parentFile
-        File(gitRoot, "audits/target/mongodb-docs")
+class RstAuditor(val type: OperatorType) {
+    companion object {
+        val auditRoot by lazy {
+            var gitRoot = File(".").canonicalFile
+            while (!File(gitRoot, ".git").exists()) gitRoot = gitRoot.parentFile
+            File(gitRoot, "audits/target/mongodb-docs")
+        }
+        val coreTestRoot = File("../core/src/test/resources").absoluteFile
+        val coreTestSourceRoot = File("../core/src/test/java").absoluteFile
+        val includesRoot = File(auditRoot, "source")
     }
-    val coreTestRoot = File("../core/src/test/resources").absoluteFile
-    val coreTestSourceRoot = File("../core/src/test/java").absoluteFile
-    lateinit var operatorRoot: File
-    val includesRoot = File(auditRoot, "source")
 
-    fun audit(type: OperatorType): Results {
-        operatorRoot = File(auditRoot, "source/reference/operator/${type.root()}")
+    var operatorRoot = File(auditRoot, "source/reference/operator/${type.root()}")
 
+    fun audit(): Results {
         val methods = findMethods(type.taglet())
         val operators = emitExampleData(type)
         val created = updateGH(operators)

audits/src/main/kotlin/dev/morphia/audits/model/Operator.kt

@@ -4,51 +4,59 @@ import dev.morphia.audits.RstAuditor
 import dev.morphia.audits.model.OperatorType.EXPRESSION
 import dev.morphia.audits.model.OperatorType.FILTER
 import dev.morphia.audits.model.OperatorType.STAGE
-import dev.morphia.audits.rst.OperatorExample
 import dev.morphia.audits.rst.RstDocument
 import java.io.File
 
-class Operator(var type: OperatorType, var source: File) {
-    var versionAdded: String?
-    var name = source.nameWithoutExtension
-    var resourceFolder: File
-    var testSource: File
-    val implemented: Boolean
-    val testCaseExists: Boolean
-    val operator = "\$${name.substringBefore("-")}"
+class Operator private constructor(var type: OperatorType) {
+    val versionAdded by lazy {
+        source
+            .readLines()
+            .filter { it.contains(".. versionadded:: ") }
+            .firstOrNull()
+            ?.substringAfterLast(":")
+            ?.trim()
+    }
+    val resourceFolder by lazy {
+        File(
+                RstAuditor.coreTestRoot,
+                "dev/morphia/test/${type.root()}/${subpath()}/${name.substringBefore("-")}"
+            )
+            .canonicalFile
+    }
+    val testSource by lazy {
+        File(
+                RstAuditor.coreTestSourceRoot,
+                "dev/morphia/test/${type.root()}/${subpath()}/Test${name.substringBefore("-").titleCase()}.java"
+            )
+            .canonicalFile
+    }
+    val implemented by lazy { resourceFolder.exists() }
+    val testCaseExists by lazy { testSource.exists() }
+    val rstAuditor = RstAuditor(type)
+    val operator by lazy { "\$${name.substringBefore("-")}" }
+    val url: String by lazy {
+        "https://www.mongodb.com/docs/manual/reference/operator/${type.root()}/$name/"
+    }
+    val examples by lazy { RstDocument.read(operator, source).examples }
+    lateinit var source: File
+    lateinit var name: String
 
-    //    val type: OperatorType
-    val url: String = "https://www.mongodb.com/docs/manual/reference/operator/${type.root()}/$name/"
-    val examples: List<OperatorExample>
+    constructor(type: OperatorType, file: File) : this(type) {
+        source = file
+        name = file.nameWithoutExtension
+        updateType()
+    }
 
-    init {
+    constructor(type: OperatorType, name: String) : this(type) {
+        this.name = name
+        source = File("${rstAuditor.operatorRoot}/$name.txt")
+        updateType()
+    }
+
+    private fun updateType() {
         if (type == STAGE || type == EXPRESSION) {
             type = if (source.readText().contains(".. pipeline:: \$")) STAGE else EXPRESSION
         }
-        versionAdded =
-            source
-                .readLines()
-                .filter { it.contains(".. versionadded:: ") }
-                .firstOrNull()
-                ?.substringAfterLast(":")
-                ?.trim()
-
-        resourceFolder =
-            File(
-                    RstAuditor.coreTestRoot,
-                    "dev/morphia/test/${type.root()}/${subpath()}/${name.substringBefore("-")}"
-                )
-                .canonicalFile
-        testSource =
-            File(
-                    RstAuditor.coreTestSourceRoot,
-                    "dev/morphia/test/${type.root()}/${subpath()}/Test${name.substringBefore("-").titleCase()}.java"
-                )
-                .canonicalFile
-
-        implemented = resourceFolder.exists()
-        testCaseExists = testSource.exists()
-        examples = RstDocument.read(operator, source).examples
     }
 
     fun ignored() = File(resourceFolder, "ignored").exists()

audits/src/main/kotlin/dev/morphia/audits/rst/RstDocument.kt

@@ -1,6 +1,7 @@
 package dev.morphia.audits.rst
 
 import dev.morphia.audits.RstAuditor
+import dev.morphia.audits.RstAuditor.Companion.includesRoot
 import dev.morphia.audits.findIndent
 import dev.morphia.audits.rst.Separator.TILDE
 import java.io.File
@@ -23,12 +24,38 @@ class RstDocument(val operator: String, lines: MutableList<String>) {
                             val include = File(RstAuditor.includesRoot, line.substringAfter(":: "))
                             if (include.exists()) {
                                 include.readLines()
-                            } else listOf(line)
+                            } else {
+                                loadInclude(line.substringAfterLast("/").substringBefore("."))
+                                    ?: listOf(line)
+                            }
                         } else listOf(line)
                     }
                     .toMutableList()
             return RstDocument(operator, lines)
         }
+
+        private fun loadInclude(keyword: String): List<String>? {
+            return File(includesRoot, "includes")
+                .listFiles()
+                .filter { file -> file.isFile }
+                .mapNotNull { file -> loadReference(keyword, file) }
+                .firstOrNull()
+        }
+
+        private fun loadReference(keyword: String, file: File): List<String>? {
+            var lines =
+                file.readLines().dropWhile { line -> !line.contains("ref: $keyword") }.drop(1)
+            if (lines.isEmpty()) {
+                return null
+            }
+            lines = lines.dropWhile { !it.startsWith("content") && !it.startsWith("---") }
+            if (!lines[0].startsWith("content")) {
+                return null
+            }
+            val removeWhile =
+                lines.drop(1).toMutableList().removeWhile { it.isBlank() || it.startsWith("  ") }
+            return removeWhile
+        }
     }
 
     var examples: List<OperatorExample> = mutableListOf()

audits/src/test/kotlin/dev/morphia/audits/AggregationAuditTest.kt

@@ -1,6 +1,5 @@
 package dev.morphia.audits
 
-import dev.morphia.audits.RstAuditor.operatorRoot
 import dev.morphia.audits.model.Operator
 import dev.morphia.audits.model.OperatorType.EXPRESSION
 import dev.morphia.audits.model.OperatorType.STAGE
@@ -12,17 +11,17 @@ class AggregationAuditTest : BaseAuditTest() {
     @Test
     fun testOperator() {
         val name = "geoNear"
-        var operator = Operator(EXPRESSION, File("${operatorRoot}/$name.txt"))
+        val operator = Operator(EXPRESSION, name)
         operator.examples.forEach { it.output(File("target/testOperator-${name}/${it.name}")) }
     }
 
     @Test
     fun expressions() {
-        validate(RstAuditor.audit(EXPRESSION))
+        validate(RstAuditor(EXPRESSION).audit())
     }
 
     @Test
     fun stages() {
-        validate(RstAuditor.audit(STAGE))
+        validate(RstAuditor(STAGE).audit())
     }
 }

audits/src/test/kotlin/dev/morphia/audits/QueryAuditTest.kt

@@ -1,11 +1,20 @@
 package dev.morphia.audits
 
+import dev.morphia.audits.model.Operator
 import dev.morphia.audits.model.OperatorType.FILTER
+import java.io.File
 import org.testng.annotations.Test
 
 class QueryAuditTest : BaseAuditTest() {
     @Test
     fun filters() {
-        validate(RstAuditor.audit(FILTER))
+        validate(RstAuditor(FILTER).audit())
+    }
+
+    @Test
+    fun testOperator() {
+        val name = "bitsAllClear"
+        val operator = Operator(FILTER, name)
+        operator.examples.forEach { it.output(File("target/testOperator-${name}/${it.name}")) }
     }
 }

core/src/main/java/dev/morphia/query/filters/Filters.java

@@ -11,6 +11,7 @@
 import dev.morphia.aggregation.expressions.impls.Expression;
 import dev.morphia.query.Type;
 
+import org.bson.BsonBinary;
 import org.bson.Document;
 
 import static java.util.Arrays.asList;
@@ -72,6 +73,19 @@ public static Filter bitsAllClear(String field, int bitMask) {
         return new Filter("$bitsAllClear", field, bitMask);
     }
 
+    /**
+     * Matches numeric or binary values in which a set of bit positions all have a value of 0.
+     *
+     * @param field the field to check
+     * @param data  the data to use
+     * @return the filter
+     * @query.filter $bitsAllClear
+     * @since 3.0
+     */
+    public static Filter bitsAllClear(String field, byte[] data) {
+        return new Filter("$bitsAllClear", field, new BsonBinary(data));
+    }
+
     /**
      * Matches numeric or binary values in which a set of bit positions all have a value of 1.
      *

core/src/test/java/dev/morphia/test/query/FiltersTest.java

@@ -57,6 +57,7 @@
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
+@Deprecated
 @SuppressWarnings("resource")
 public class FiltersTest extends TemplatedTestBase {
     @AfterClass

core/src/test/java/dev/morphia/test/query/filters/TestBitsAllClear.java

@@ -0,0 +1,28 @@
+package dev.morphia.test.query.filters;
+
+import dev.morphia.test.ServerVersion;
+
+import org.testng.annotations.Test;
+
+import static dev.morphia.query.filters.Filters.bitsAllClear;
+
+public class TestBitsAllClear extends FilterTest {
+    @Test
+    public void testExample1() {
+        testQuery(ServerVersion.ANY, false, true, (query) -> query.filter(
+                bitsAllClear("a", new int[] { 1, 5 })));
+    }
+
+    @Test
+    public void testExample2() {
+        testQuery(ServerVersion.ANY, false, true, (query) -> query.filter(
+                bitsAllClear("a", 35)));
+    }
+
+    @Test
+    public void testExample3() {
+        testQuery(ServerVersion.ANY, false, true, (query) -> query.filter(
+                bitsAllClear("a", new byte[] { 32 })));
+    }
+
+}

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example1/action.json

@@ -0,0 +1 @@
+db.collection.find( { a: { $bitsAllClear: [ 1, 5 ] } } )

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example1/data.json

@@ -0,0 +1,4 @@
+{ _id: 1, a: 54, binaryValueofA: "00110110" },
+{ _id: 2, a: 20, binaryValueofA: "00010100" },
+{ _id: 3, a: 20.0, binaryValueofA: "00010100" },
+{ _id: 4, a: BinData(0, "Zg=="), binaryValueofA: "01100110" }

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example1/expected.json

@@ -0,0 +1,2 @@
+{ "_id" : 2, "a" : 20, "binaryValueofA" : "00010100" }
+{ "_id" : 3, "a" : 20, "binaryValueofA" : "00010100" }

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example1/name

@@ -0,0 +1 @@
+Bit Position Array

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example2/action.json

@@ -0,0 +1 @@
+db.collection.find( { a: { $bitsAllClear: 35 } } )

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example2/data.json

@@ -0,0 +1,4 @@
+{ _id: 1, a: 54, binaryValueofA: "00110110" },
+{ _id: 2, a: 20, binaryValueofA: "00010100" },
+{ _id: 3, a: 20.0, binaryValueofA: "00010100" },
+{ _id: 4, a: BinData(0, "Zg=="), binaryValueofA: "01100110" }

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example2/expected.json

@@ -0,0 +1,2 @@
+{ "_id" : 2, "a" : 20, "binaryValueofA" : "00010100" }
+{ "_id" : 3, "a" : 20, "binaryValueofA" : "00010100" }

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example2/name

@@ -0,0 +1 @@
+Integer Bitmask

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example3/action.json

@@ -0,0 +1 @@
+db.collection.find( { a: { $bitsAllClear: BinData(0, "IA==") } } )

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example3/data.json

@@ -0,0 +1,4 @@
+{ _id: 1, a: 54, binaryValueofA: "00110110" },
+{ _id: 2, a: 20, binaryValueofA: "00010100" },
+{ _id: 3, a: 20.0, binaryValueofA: "00010100" },
+{ _id: 4, a: BinData(0, "Zg=="), binaryValueofA: "01100110" }

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example3/expected.json

@@ -0,0 +1,2 @@
+{ "_id" : 2, "a" : 20, "binaryValueofA" : "00010100" }
+{ "_id" : 3, "a" : 20, "binaryValueofA" : "00010100" }

core/src/test/resources/dev/morphia/test/query/filters/bitsAllClear/example3/name

@@ -0,0 +1 @@
+BinData Bitmask