Support for XML and XPath and numpy/pandas libraries

.gitlab-ci.yml

@@ -46,8 +46,15 @@ FrontendTests:
 
 RuntimeTests:
   stage: tests3
+  artifacts:
+    name: "Runtime Tests log"
+    paths:
+      - target/runtime_test.log
+    when:
+      always
+    expire_in: 2 days
   script:
-    - mvn -Dtest=RuntimeTests test
+    - mvn -Dtest=RuntimeTests test --log-file target/runtime_test.log
 
 RuntimeTestsNoParallelism:
   stage: tests3

gen/XQueryParserScripting.interp

@@ -556,7 +556,7 @@ whileStatement
 pathExpr
 relativePathExpr
 stepExpr
-axisStep
+stepExpr
 forwardStep
 forwardAxis
 abbrevForwardStep

src/main/java/org/rumbledb/api/Item.java

@@ -1,11 +1,6 @@
 package org.rumbledb.api;
 
-import java.io.Serializable;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.List;
-import java.util.Map;
-
+import com.esotericsoftware.kryo.KryoSerializable;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.ml.Estimator;
 import org.apache.spark.ml.Transformer;
@@ -20,8 +15,14 @@
 import org.rumbledb.serialization.Serializer;
 import org.rumbledb.types.FunctionSignature;
 import org.rumbledb.types.ItemType;
+import org.w3c.dom.Node;
 
-import com.esotericsoftware.kryo.KryoSerializable;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 
 /**
@@ -297,6 +298,46 @@ default boolean isBase64Binary() {
         return false;
     }
 
+    /**
+     * Tests whether the item is an XML Element node.
+     *
+     * @return true if it is an XML Element node, false otherwise.
+     */
+    default boolean isElementNode() {
+        return false;
+    }
+
+    /**
+     * Tests whether the item is an XML Attribute node.
+     *
+     * @return true if it is an XML Attribute node, false otherwise.
+     */
+    default boolean isAttributeNode() {
+        return false;
+    }
+
+    /**
+     * Tests whether the item is an XML Text node.
+     *
+     * @return true if it is an XML Text node, false otherwise.
+     */
+    default boolean getContent() {
+        return false;
+    }
+
+    /**
+     * Tests whether the item is an XML Document node.
+     *
+     * @return true if it is an XML Document node, false otherwise.
+     */
+    default boolean isDocumentNode() {
+        return false;
+    }
+
+    default boolean isTextNode() {
+        return false;
+    }
+
     /**
      * Returns the members of the item if it is an array.
      *
@@ -747,4 +788,96 @@ default boolean isTransformer() {
     default Transformer getTransformer() {
         throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
     }
+
+    /**
+     * Returns the string value of the text item, if it is a text item.
+     *
+     * @return the string value.
+     */
+    default String getTextValue() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    /**
+     * Method sets the parent item for all descendents of the current item.
+     */
+    default void addParentToDescendants() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default List<Item> attributes() {
+        return new ArrayList<>();
+    }
+
+    default String baseUri() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default List<Item> children() {
+        return new ArrayList<>();
+    }
+
+    default String documentUri() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default boolean isId() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default boolean isIdRefs() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default List<Item> namespaceNodes() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default boolean nilled() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default String nodeKind() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default String nodeName() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default Item parent() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default String stringValue() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default String typeName() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default Item typedValue() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default String unparsedEntityPublicId() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default String unparsedEntityServerId() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default void setParent(Item parent) {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default int compareXmlNode(Item otherNode) {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
+    default Node getXmlNode() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
 }

src/main/java/org/rumbledb/compiler/CloneVisitor.java

@@ -1045,4 +1045,43 @@ public Node visitVariableDeclStatement(VariableDeclStatement statement, Node arg
         return result;
     }
     // end region scripting
+
+    // begin region xml
+
+    // @Override
+    // public Node visitPathExpr(PathExpr expression, Node argument) {
+    // List<Expression> intermediaryPaths = new ArrayList<>();
+    // for (IntermediaryPath path : expression.getRelativePathExpressions()) {
+    // Dash dash = null;
+    // if (path.getPreStepExprDash() != null) {
+    // dash = new Dash(
+    // path.getPreStepExprDash().requiresRoot(),
+    // (StepExpr) this.visitAxisStep(path.getPreStepExprDash().getAxisStep(), argument)
+    // );
+    // }
+    // StepExpr stepExpr = (StepExpr) this.visitStepExpr(path.getStepExpr(), argument);
+    // intermediaryPaths.add(new IntermediaryPath(dash, stepExpr));
+    // }
+    // PathExpr result = new PathExpr(intermediaryPaths, expression.getMetadata());
+    // result.setStaticSequenceType(expression.getStaticSequenceType());
+    // result.setStaticContext(expression.getStaticContext());
+    // return result;
+    // }
+
+    // @Override
+    // public Node visitStepExpr(StepExpr expression, Node argument) {
+    // StepExpr result;
+    // if (expression.getPostFixExpr() != null) {
+    // Expression postFixExpr = (Expression) this.visit(expression.getPostFixExpr(), argument);
+    // result = new StepExpr(postFixExpr, expression.getMetadata());
+    // } else {
+    // StepExpr stepExpr = (StepExpr) this.visitAxisStep(expression.getAxisStep(), argument);
+    // result = new StepExpr(stepExpr, expression.getMetadata());
+    // }
+    // result.setStaticSequenceType(expression.getStaticSequenceType());
+    // result.setStaticContext(expression.getStaticContext());
+    // return result;
+    // }
+
+    // end region xml
 }

src/main/java/org/rumbledb/compiler/InferTypeVisitor.java

@@ -103,6 +103,8 @@
 import org.rumbledb.expressions.update.RenameExpression;
 import org.rumbledb.expressions.update.ReplaceExpression;
 import org.rumbledb.expressions.update.TransformExpression;
+import org.rumbledb.expressions.xml.PathExpr;
+import org.rumbledb.expressions.xml.StepExpr;
 import org.rumbledb.runtime.functions.input.FileSystemUtil;
 import org.rumbledb.types.BuiltinTypesCatalogue;
 import org.rumbledb.types.FieldDescriptor;
@@ -2598,4 +2600,22 @@ public StaticContext visitBlockExpr(BlockExpression expression, StaticContext ar
     }
 
     // endregion
+
+    // region xml
+
+    @Override
+    public StaticContext visitPathExpr(PathExpr pathExpr, StaticContext argument) {
+        visitDescendants(pathExpr, argument);
+        pathExpr.setStaticSequenceType(SequenceType.ITEM_STAR);
+        return argument;
+    }
+
+    // TODO: Currently, step expressions are marked as string, but this type may differ. Update to relevant type.
+    @Override
+    public StaticContext visitStepExpr(StepExpr stepExpr, StaticContext argument) {
+        stepExpr.setStaticSequenceType(SequenceType.ITEM_STAR);
+        return argument;
+    }
+
+    // end xml
 }

src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java

@@ -32,6 +32,7 @@
 import org.rumbledb.exceptions.UnsupportedFeatureException;
 import org.rumbledb.expressions.AbstractNodeVisitor;
 import org.rumbledb.expressions.CommaExpression;
+import org.rumbledb.expressions.ExecutionMode;
 import org.rumbledb.expressions.Expression;
 import org.rumbledb.expressions.Node;
 import org.rumbledb.expressions.arithmetic.AdditiveExpression;
@@ -114,6 +115,9 @@
 import org.rumbledb.expressions.update.RenameExpression;
 import org.rumbledb.expressions.update.ReplaceExpression;
 import org.rumbledb.expressions.update.TransformExpression;
+import org.rumbledb.expressions.xml.PathExpr;
+import org.rumbledb.expressions.xml.StepExpr;
+import org.rumbledb.expressions.xml.node_test.NodeTest;
 import org.rumbledb.items.ItemFactory;
 import org.rumbledb.runtime.AtMostOneItemLocalRuntimeIterator;
 import org.rumbledb.runtime.CommaExpressionIterator;
@@ -190,6 +194,11 @@
 import org.rumbledb.runtime.update.expression.RenameExpressionIterator;
 import org.rumbledb.runtime.update.expression.ReplaceExpressionIterator;
 import org.rumbledb.runtime.update.expression.TransformExpressionIterator;
+import org.rumbledb.runtime.xml.AtomizationIterator;
+import org.rumbledb.runtime.xml.PathExprIterator;
+import org.rumbledb.runtime.xml.StepExprIterator;
+import org.rumbledb.runtime.xml.axis.AxisIterator;
+import org.rumbledb.runtime.xml.axis.AxisIteratorVisitor;
 import org.rumbledb.types.BuiltinTypesCatalogue;
 import org.rumbledb.types.SequenceType;
 
@@ -944,6 +953,13 @@ public RuntimeIterator visitRangeExpr(RangeExpression expression, RuntimeIterato
     public RuntimeIterator visitComparisonExpr(ComparisonExpression expression, RuntimeIterator argument) {
         RuntimeIterator left = this.visit(expression.getChildren().get(0), argument);
         RuntimeIterator right = this.visit(expression.getChildren().get(1), argument);
+        if (left instanceof PathExprIterator) {
+            // We potentially need to atomize
+            left = new AtomizationIterator(
+                    left,
+                    expression.getStaticContextForRuntime(this.config, this.visitorConfig)
+            );
+        }
         RuntimeIterator runtimeIterator = new ComparisonIterator(
                 left,
                 right,
@@ -1414,4 +1430,56 @@ public RuntimeIterator visitFlowrStatement(FlowrStatement statement, RuntimeIter
         runtimeIterator.setStaticContext(statement.getStaticContext());
         return runtimeIterator;
     }
+
+    @Override
+    public RuntimeIterator visitPathExpr(PathExpr pathExpr, RuntimeIterator argument) {
+        List<RuntimeIterator> stepExprIterators = new ArrayList<>();
+        pathExpr.getRelativePathExpressions()
+            .forEach(relativePathExpr -> stepExprIterators.add(this.visit(relativePathExpr, argument)));
+        RuntimeIterator getRootIterator = null;
+        if (pathExpr.needsRoot()) {
+            getRootIterator = this.visitFunctionCall(pathExpr.getFetchRootFunction(), argument);
+        }
+        RuntimeIterator runtimeIterator = new PathExprIterator(
+                stepExprIterators,
+                getRootIterator,
+                new RuntimeStaticContext(
+                        this.config,
+                        pathExpr.getStaticSequenceType(),
+                        pathExpr.getHighestExecutionMode(this.visitorConfig),
+                        pathExpr.getMetadata()
+                )
+        );
+        runtimeIterator.setStaticContext(pathExpr.getStaticContext());
+        return runtimeIterator;
+    }
+
+    @Override
+    public RuntimeIterator visitStepExpr(StepExpr stepExpr, RuntimeIterator argument) {
+        AxisIterator axisIterator = this.visitAxisStep(stepExpr, stepExpr.getMetadata());
+        NodeTest nodeTest = stepExpr.getNodeTest();
+        return new StepExprIterator(
+                axisIterator,
+                nodeTest,
+                new RuntimeStaticContext(
+                        this.config,
+                        SequenceType.ITEM,
+                        stepExpr.getHighestExecutionMode(this.visitorConfig),
+                        stepExpr.getMetadata()
+                )
+        );
+    }
+
+    private AxisIterator visitAxisStep(StepExpr stepExpr, ExceptionMetadata metadata) {
+        return stepExpr.accept(
+            new AxisIteratorVisitor(),
+            new RuntimeStaticContext(
+                    this.config,
+                    SequenceType.STRING,
+                    ExecutionMode.LOCAL,
+                    metadata
+            )
+        );
+    }
+
 }