Implement Support for Generation of Optional Record Fields and Proper Namespaces

misc/xml-to-record-converter/src/main/java/io/ballerina/xmltorecordconverter/util/ConverterUtils.java

@@ -20,6 +20,7 @@
 
 import io.ballerina.compiler.syntax.tree.AbstractNodeFactory;
 import io.ballerina.compiler.syntax.tree.ArrayTypeDescriptorNode;
+import io.ballerina.compiler.syntax.tree.Node;
 import io.ballerina.compiler.syntax.tree.ParenthesisedTypeDescriptorNode;
 import io.ballerina.compiler.syntax.tree.SyntaxInfo;
 import io.ballerina.compiler.syntax.tree.SyntaxKind;
@@ -71,6 +72,45 @@
         }
     }
 
+    /**
+     * This method returns the SyntaxToken corresponding to the JsonPrimitive.
+     *
+     * @param value JsonPrimitive that has to be classified.
+     * @return {@link Token} Classified Syntax Token.
+     */
+    public static Token getPrimitiveTypeName(String value) {
+        if (isBoolean(value)) {
+            return AbstractNodeFactory.createToken(SyntaxKind.BOOLEAN_KEYWORD);
+        } else if (isInteger(value)) {
+            return AbstractNodeFactory.createToken(SyntaxKind.INT_KEYWORD);
+        } else if (isDouble(value)) {
+            return AbstractNodeFactory.createToken(SyntaxKind.DECIMAL_KEYWORD);
+        }
+        return AbstractNodeFactory.createToken(SyntaxKind.STRING_KEYWORD);
+    }
+
+    /**
+     * This method extracts TypeDescriptorNodes within any UnionTypeDescriptorNodes or ParenthesisedTypeDescriptorNode.
+     *
+     * @param typeDescNodes List of Union and Parenthesised TypeDescriptorNodes
+     * @return {@link List<TypeDescriptorNode>} Extracted SimpleNameReferenceNodes.
+     */
+    public static List<TypeDescriptorNode> extractTypeDescriptorNodes(List<TypeDescriptorNode> typeDescNodes) {
+        List<TypeDescriptorNode> extractedTypeNames = new ArrayList<>();
+        for (TypeDescriptorNode typeDescNode : typeDescNodes) {
+            TypeDescriptorNode extractedTypeDescNode = extractParenthesisedTypeDescNode(typeDescNode);
+            if (extractedTypeDescNode instanceof UnionTypeDescriptorNode) {
+                List<TypeDescriptorNode> childTypeDescNodes =
+                        List.of(((UnionTypeDescriptorNode) extractedTypeDescNode).leftTypeDesc(),
+                                ((UnionTypeDescriptorNode) extractedTypeDescNode).rightTypeDesc());
+                addIfNotExist(extractedTypeNames, extractTypeDescriptorNodes(childTypeDescNodes));
+            } else {
+                addIfNotExist(extractedTypeNames, List.of(extractedTypeDescNode));
+            }
+        }
+        return extractedTypeNames;
+    }
+
     /**
      * This method returns the sorted TypeDescriptorNode list.
      *
@@ -82,6 +122,10 @@
                 .filter(node -> !(node instanceof ArrayTypeDescriptorNode)).collect(Collectors.toList());
         List<TypeDescriptorNode> arrayNodes = typeDescriptorNodes.stream()
                 .filter(node -> (node instanceof ArrayTypeDescriptorNode)).collect(Collectors.toList());
+        List<TypeDescriptorNode> membersOfArrayNodes = arrayNodes.stream()
+                .map(node -> extractArrayTypeDescNode((ArrayTypeDescriptorNode) node)).toList();
+        nonArrayNodes.removeIf(node ->
+                membersOfArrayNodes.stream().map(Node::toSourceCode).toList().contains(node.toSourceCode()));
         nonArrayNodes.sort(Comparator.comparing(TypeDescriptorNode::toSourceCode));
         arrayNodes.sort((node1, node2) -> {
             ArrayTypeDescriptorNode arrayNode1 = (ArrayTypeDescriptorNode) node1;
@@ -94,20 +138,6 @@
         return Stream.concat(nonArrayNodes.stream(), arrayNodes.stream()).collect(Collectors.toList());
     }
 
-    /**
-     * This method returns the number of dimensions of an ArrayTypeDescriptorNode.
-     *
-     * @param arrayNode ArrayTypeDescriptorNode for which the no. of dimensions has to be calculated.
-     * @return {@link Integer} The total no. of dimensions of the ArrayTypeDescriptorNode.
-     */
-    public static Integer getNumberOfDimensions(ArrayTypeDescriptorNode arrayNode) {
-        int totalDimensions = arrayNode.dimensions().size();
-        if (arrayNode.memberTypeDesc() instanceof ArrayTypeDescriptorNode) {
-            totalDimensions += getNumberOfDimensions((ArrayTypeDescriptorNode) arrayNode.memberTypeDesc());
-        }
-        return totalDimensions;
-    }
-
     /**
      * This method returns a list of TypeDescriptorNodes extracted from a UnionTypeDescriptorNode.
      *
@@ -132,29 +162,42 @@
         return extractedTypeDescNodes;
     }
 
+    /**
+     * This method returns the number of dimensions of an ArrayTypeDescriptorNode.
+     *
+     * @param arrayNode ArrayTypeDescriptorNode for which the no. of dimensions has to be calculated.
+     * @return {@link Integer} The total no. of dimensions of the ArrayTypeDescriptorNode.
+     */
+    public static Integer getNumberOfDimensions(ArrayTypeDescriptorNode arrayNode) {
+        int totalDimensions = arrayNode.dimensions().size();
+        if (arrayNode.memberTypeDesc() instanceof ArrayTypeDescriptorNode) {
+            totalDimensions += getNumberOfDimensions((ArrayTypeDescriptorNode) arrayNode.memberTypeDesc());
+        }
+        return totalDimensions;
+    }
+
+    private static TypeDescriptorNode extractArrayTypeDescNode(ArrayTypeDescriptorNode arrayTypeDescNode) {
+        if (arrayTypeDescNode.memberTypeDesc() instanceof ArrayTypeDescriptorNode) {
+            return extractArrayTypeDescNode((ArrayTypeDescriptorNode) arrayTypeDescNode.memberTypeDesc());
+        }
+        return arrayTypeDescNode.memberTypeDesc();
+    }
+
     private static TypeDescriptorNode extractParenthesisedTypeDescNode(TypeDescriptorNode typeDescNode) {
         if (typeDescNode instanceof ParenthesisedTypeDescriptorNode) {
             return extractParenthesisedTypeDescNode(((ParenthesisedTypeDescriptorNode) typeDescNode).typedesc());
-        } else {
-            return typeDescNode;
         }
+        return typeDescNode;
     }
 
-    /**
-     * This method returns the SyntaxToken corresponding to the JsonPrimitive.
-     *
-     * @param value JsonPrimitive that has to be classified.
-     * @return {@link Token} Classified Syntax Token.
-     */
-    public static Token getPrimitiveTypeName(String value) {
-        if (isBoolean(value)) {
-            return AbstractNodeFactory.createToken(SyntaxKind.BOOLEAN_KEYWORD);
-        } else if (isInteger(value)) {
-            return AbstractNodeFactory.createToken(SyntaxKind.INT_KEYWORD);
-        } else if (isDouble(value)) {
-            return AbstractNodeFactory.createToken(SyntaxKind.DECIMAL_KEYWORD);
+    private static void addIfNotExist(List<TypeDescriptorNode> typeDescNodes,
+                                      List<TypeDescriptorNode> typeDescNodesToBeInserted) {
+        for (TypeDescriptorNode typeDescNodeToBeInserted : typeDescNodesToBeInserted) {
+            if (typeDescNodes.stream().noneMatch(typeDescNode -> typeDescNode.toSourceCode()
+                    .equals(typeDescNodeToBeInserted.toSourceCode()))) {
+                typeDescNodes.add(typeDescNodeToBeInserted);
+            }
         }
-        return AbstractNodeFactory.createToken(SyntaxKind.STRING_KEYWORD);
     }
 
     private static boolean isBoolean(String value) {

misc/xml-to-record-converter/src/test/java/io/ballerina/xmltorecordconverter/XMLToRecordConverterTests.java

@@ -89,6 +89,41 @@ public class XMLToRecordConverterTests {
     private final Path sample9Bal = RES_DIR.resolve(BAL_DIR)
             .resolve("sample_9.bal");
 
+    private final Path sample10XML = RES_DIR.resolve(XML_DIR)
+            .resolve("sample_10.xml");
+    private final Path sample10Bal = RES_DIR.resolve(BAL_DIR)
+            .resolve("sample_10.bal");
+
+    private final Path sample11XML = RES_DIR.resolve(XML_DIR)
+            .resolve("sample_11.xml");
+    private final Path sample11Bal = RES_DIR.resolve(BAL_DIR)
+            .resolve("sample_11.bal");
+
+    private final Path sample12XML = RES_DIR.resolve(XML_DIR)
+            .resolve("sample_12.xml");
+    private final Path sample12Bal = RES_DIR.resolve(BAL_DIR)
+            .resolve("sample_12.bal");
+
+    private final Path sample13XML = RES_DIR.resolve(XML_DIR)
+            .resolve("sample_13.xml");
+    private final Path sample13Bal = RES_DIR.resolve(BAL_DIR)
+            .resolve("sample_13.bal");
+
+    private final Path sample14XML = RES_DIR.resolve(XML_DIR)
+            .resolve("sample_14.xml");
+    private final Path sample14Bal = RES_DIR.resolve(BAL_DIR)
+            .resolve("sample_14.bal");
+
+    private final Path sample15XML = RES_DIR.resolve(XML_DIR)
+            .resolve("sample_15.xml");
+    private final Path sample15Bal = RES_DIR.resolve(BAL_DIR)
+            .resolve("sample_15.bal");
+
+    private final Path sample16XML = RES_DIR.resolve(XML_DIR)
+            .resolve("sample_16.xml");
+    private final Path sample16Bal = RES_DIR.resolve(BAL_DIR)
+            .resolve("sample_16.bal");
+
     private static final String XMLToRecordServiceEP = "xmlToRecord/convert";
 
 
@@ -182,6 +217,69 @@ public void testComplexXMLWithNamespace() throws IOException {
         Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
     }
 
+    @Test(description = "testXMLWithOptionalFields1")
+    public void testXMLWithOptionalFields1() throws IOException {
+        String xmlFileContent = Files.readString(sample10XML);
+        String generatedCodeBlock = XMLToRecordConverter.convert(xmlFileContent, false, false, false)
+                .getCodeBlock().replaceAll("\\s+", "");
+        String expectedCodeBlock = Files.readString(sample10Bal).replaceAll("\\s+", "");
+        Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
+    }
+
+    @Test(description = "testXMLWithOptionalFields2")
+    public void testXMLWithOptionalFields2() throws IOException {
+        String xmlFileContent = Files.readString(sample11XML);
+        String generatedCodeBlock = XMLToRecordConverter.convert(xmlFileContent, false, false, false)
+                .getCodeBlock().replaceAll("\\s+", "");
+        String expectedCodeBlock = Files.readString(sample11Bal).replaceAll("\\s+", "");
+        Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
+    }
+
+    @Test(description = "testXMLWithOptionalFieldsInNestedNodes")
+    public void testXMLWithOptionalFieldsInNestedNodes() throws IOException {
+        String xmlFileContent = Files.readString(sample12XML);
+        String generatedCodeBlock = XMLToRecordConverter.convert(xmlFileContent, false, false, false)
+                .getCodeBlock().replaceAll("\\s+", "");
+        String expectedCodeBlock = Files.readString(sample12Bal).replaceAll("\\s+", "");
+        Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
+    }
+
+    @Test(description = "testXMLWithOptionalFieldsInMultipleNestedNodes")
+    public void testXMLWithOptionalFieldsInMultipleNestedNodes() throws IOException {
+        String xmlFileContent = Files.readString(sample13XML);
+        String generatedCodeBlock = XMLToRecordConverter.convert(xmlFileContent, false, false, false)
+                .getCodeBlock().replaceAll("\\s+", "");
+        String expectedCodeBlock = Files.readString(sample13Bal).replaceAll("\\s+", "");
+        Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
+    }
+
+    @Test(description = "testXMLWithMultipleNamespaces")
+    public void testXMLWithMultipleNamespaces() throws IOException {
+        String xmlFileContent = Files.readString(sample14XML);
+        String generatedCodeBlock = XMLToRecordConverter.convert(xmlFileContent, false, false, false)
+                .getCodeBlock().replaceAll("\\s+", "");
+        String expectedCodeBlock = Files.readString(sample14Bal).replaceAll("\\s+", "");
+        Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
+    }
+
+    @Test(description = "testComplexXMLWithMultipleNamespaces")
+    public void testComplexXMLWithMultipleNamespaces() throws IOException {
+        String xmlFileContent = Files.readString(sample15XML);
+        String generatedCodeBlock = XMLToRecordConverter.convert(xmlFileContent, false, false, false)
+                .getCodeBlock().replaceAll("\\s+", "");
+        String expectedCodeBlock = Files.readString(sample15Bal).replaceAll("\\s+", "");
+        Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
+    }
+
+    @Test(description = "testComplexXMLWithMultipleNamespacesAndRecurringNodes")
+    public void testComplexXMLWithMultipleNamespacesAndRecurringNodes() throws IOException {
+        String xmlFileContent = Files.readString(sample16XML);
+        String generatedCodeBlock = XMLToRecordConverter.convert(xmlFileContent, false, false, false)
+                .getCodeBlock().replaceAll("\\s+", "");
+        String expectedCodeBlock = Files.readString(sample16Bal).replaceAll("\\s+", "");
+        Assert.assertEquals(generatedCodeBlock, expectedCodeBlock);
+    }
+
     @Test(description = "testXMLToRecordService")
     public void testXMLToRecordService() throws IOException, ExecutionException, InterruptedException {
         Endpoint serviceEndpoint = TestUtil.initializeLanguageSever();

misc/xml-to-record-converter/src/test/resources/ballerina/sample_10.bal

@@ -0,0 +1,15 @@
+type Book record {
+    string author?;
+    string title;
+    decimal price;
+    (int|string) isbn;
+    @xmldata:Attribute
+    string id;
+};
+
+@xmldata:Name {
+    value: "catalog"
+}
+type Catalog record {
+    Book[] book;
+};

misc/xml-to-record-converter/src/test/resources/ballerina/sample_11.bal

@@ -0,0 +1,17 @@
+type Item record {
+    int id;
+    string name;
+    decimal cost?;
+    decimal price?;
+    int quantity?;
+    string code?;
+    string description?;
+    string discount?;
+};
+
+@xmldata:Name {
+    value: "root"
+}
+type Root record {
+    Item[] item;
+};

misc/xml-to-record-converter/src/test/resources/ballerina/sample_12.bal

@@ -0,0 +1,18 @@
+type ZASN_SDTXP record {
+    string TDFORMAT?;
+    string TDLINE;
+    @xmldata:Attribute
+    string SEGMENT;
+};
+
+type ZASN_SDTX record {
+    string TDID;
+    string TDSPRAS;
+    ZASN_SDTXP[] ZASN_SDTXP;
+    @xmldata:Attribute
+    string SEGMENT;
+};
+
+type ROOT record {
+    ZASN_SDTX[] ZASN_SDTX;
+};

misc/xml-to-record-converter/src/test/resources/ballerina/sample_13.bal

@@ -0,0 +1,72 @@
+type Task record {
+    string id;
+    string description;
+    @xmldata:Attribute
+    string name?;
+};
+
+type Tasks record {
+    Task[] task;
+};
+
+type Project record {
+    string id;
+    string name;
+    Tasks tasks?;
+};
+
+type Projects record {
+    Project[] project;
+};
+
+type Manager record {
+    string id;
+    string name;
+};
+
+type Employee record {
+    int id;
+    string name;
+    string position;
+    int phone?;
+    string dob?;
+    Projects projects;
+    Manager manager?;
+};
+
+type Employees record {
+    Employee[] employee;
+};
+
+type Members record {
+    string[] member;
+};
+
+type Team record {
+    string id;
+    string name;
+    Members members;
+};
+
+type Teams record {
+    Team[] team;
+};
+
+type Department record {
+    string id;
+    string name;
+    Teams teams;
+};
+
+type Departments record {
+    Department[] department;
+};
+
+@xmldata:Name {
+    value: "company"
+}
+type Company record {
+    string name;
+    Employees employees;
+    Departments departments;
+};