From 99e6575dab1129699f2c3e5894d1da3d0d6da5e9 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Wed, 12 Jul 2023 16:31:53 +0700 Subject: [PATCH 1/6] Add Javascript undeclared functions --- src/codetext/parser/javascript_parser.py | 17 +++++++- tests/test_parser/test_javascript.py | 55 +++++++++++++++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/codetext/parser/javascript_parser.py b/src/codetext/parser/javascript_parser.py index bd97b2f..c65c7c8 100644 --- a/src/codetext/parser/javascript_parser.py +++ b/src/codetext/parser/javascript_parser.py @@ -48,7 +48,12 @@ def get_comment_node(function_node): @staticmethod def get_function_list(node): - function_types = ['function_declaration', 'function', 'method_definition', 'generator_function_declaration'] + function_types = ['function_declaration', + 'function', + 'method_definition', + 'generator_function_declaration', + 'arrow_function', + 'generator_function'] res = get_node_by_kind(node, function_types) for node in res[:]: if not node.children: @@ -87,6 +92,16 @@ def get_function_metadata(function_node, blob: str=None) -> Dict[str, str]: return_statement = get_node_by_kind(function_node, ['return_statement']) if len(return_statement) > 0: metadata['return_type'] = '' + + if function_node.type in ["function", + "arrow_function", + "generator_function"]: + # function inside object property or variable declarator + identifier = function_node.prev_named_sibling + if identifier: + if identifier.type in ["identifier"]: + metadata["identifier"] = identifier.text.decode() + return metadata @staticmethod diff --git a/tests/test_parser/test_javascript.py b/tests/test_parser/test_javascript.py index 25c1483..aa4c1c9 100755 --- a/tests/test_parser/test_javascript.py +++ b/tests/test_parser/test_javascript.py @@ -76,8 +76,8 @@ class Car { def test_get_function_metadata(self): root = self.root_node - function = JavascriptParser.get_function_list(root)[1] - metadata = JavascriptParser.get_function_metadata(function) + _function = JavascriptParser.get_function_list(root)[1] + metadata = JavascriptParser.get_function_metadata(_function) for key in ['identifier', 'parameters', 'return_type']: self.assertTrue(key in metadata.keys()) @@ -109,6 +109,57 @@ def test_get_class_metadata(self): def test_extract_docstring(self): pass + + def test_metadata_with_arrow_function(self): + code_sample = ''' + export const parseModel = async (mesh) => + new Promise((resolve) => { + exporter.parse( + mesh, + (gltf) => { + const blob = new Blob([gltf], { type: "application/octet-stream" }); + resolve(blob); + return blob; + }, + (error) => { + console.log(error); + return error; + + } + ); + }); + ''' + root = parse_code(code_sample, 'javascript').root_node + fn = JavascriptParser.get_function_list(root)[0] + metadata = JavascriptParser.get_function_metadata(fn) + + identifier = metadata['identifier'] + self.assertEqual(identifier, 'parseModel') + + def test_metadata_with_undecleared_functions(self): + code_sample = """ + const asyncFunctionExpression = async function() { + // async function expression definition + return a + }; + + const generatorFunctionExpression = function*() { + // generator function expression definition + return b + }; + """ + root = parse_code(code_sample, 'javascript').root_node + fn1, fn2 = JavascriptParser.get_function_list(root) + + self.assertEqual(fn1.type, 'function') + self.assertEqual(fn2.type, 'generator_function') + + metadata1 = JavascriptParser.get_function_metadata(fn1) + metadata2 = JavascriptParser.get_function_metadata(fn2) + + self.assertEqual(metadata1['identifier'], 'asyncFunctionExpression') + self.assertEqual(metadata2['identifier'], 'generatorFunctionExpression') + if __name__ == '__main__': unittest.main() From b1fd37664f993c7fe51c2ab92de58dd08e8a7b2c Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Wed, 12 Jul 2023 17:43:25 +0700 Subject: [PATCH 2/6] Add PHP interface --- src/codetext/parser/php_parser.py | 4 +++- tests/test_parser/test_php.py | 18 ++++++++++----- .../test_sample/php_test_sample.php | 22 +++++++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/codetext/parser/php_parser.py b/src/codetext/parser/php_parser.py index b2a8ef2..9068f68 100644 --- a/src/codetext/parser/php_parser.py +++ b/src/codetext/parser/php_parser.py @@ -49,7 +49,9 @@ def get_comment_node(function_node): @staticmethod def get_class_list(node): - res = get_node_by_kind(node, ['class_declaration', 'trait_declaration']) + res = get_node_by_kind(node, ['class_declaration', + 'trait_declaration', + 'interface_declaration']) return res @staticmethod diff --git a/tests/test_parser/test_php.py b/tests/test_parser/test_php.py index 2d2d526..1e2058a 100755 --- a/tests/test_parser/test_php.py +++ b/tests/test_parser/test_php.py @@ -22,14 +22,14 @@ def test_get_function_list(self): function_list = PhpParser.get_function_list(root) - self.assertEqual(len(function_list), 3) + self.assertEqual(len(function_list), 5) def test_get_class_list(self): root = self.root_node class_list = PhpParser.get_class_list(root) - self.assertEqual(len(class_list), 1) + self.assertEqual(len(class_list), 3) def test_get_docstring(self): code_sample = """ @@ -104,11 +104,17 @@ def test_metadata_without_return_statement(self): def test_get_class_metadata(self): root = self.root_node - classes = list(PhpParser.get_class_list(root))[0] - metadata = PhpParser.get_class_metadata(classes) + _class, interface, trait = list(PhpParser.get_class_list(root)) + class_metadata = PhpParser.get_class_metadata(_class) - self.assertEqual(metadata['parameters'], {'AbstractSQLServerDriver': None}) - self.assertEqual(metadata['identifier'], 'Driver') + self.assertEqual(class_metadata['parameters'], {'AbstractSQLServerDriver': None}) + self.assertEqual(class_metadata['identifier'], 'Driver') + + interface_metadata = PhpParser.get_class_metadata(interface) + self.assertEqual(interface_metadata['identifier'], 'MyInterface') + + trait_metadata = PhpParser.get_class_metadata(trait) + self.assertEqual(trait_metadata['identifier'], 'MyTrait') if __name__ == '__main__': diff --git a/tests/test_parser/test_sample/php_test_sample.php b/tests/test_parser/test_sample/php_test_sample.php index 716ee0a..6301e17 100755 --- a/tests/test_parser/test_sample/php_test_sample.php +++ b/tests/test_parser/test_sample/php_test_sample.php @@ -88,3 +88,25 @@ private function getConnectionOptionsDsn(array $connectionOptions): string return $connectionOptionsDsn; } } + +interface MyInterface { + public function myMethod() { + // Method implementation + } + +} + +trait MyTrait { + + public function setBackgroundImage(Drawing $objDrawing): self + { + if (!array_key_exists($objDrawing->getType(), Drawing::IMAGE_TYPES_CONVERTION_MAP)) { + throw new PhpSpreadsheetException('Unsupported image type in comment background. Supported types: PNG, JPEG, BMP, GIF.'); + } + $this->backgroundImage = $objDrawing; + + return $this; + } + +} + From b8a87b64c79c21723d02c01f66ecd0479ce99f10 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Thu, 13 Jul 2023 10:17:19 +0700 Subject: [PATCH 3/6] Add singleton method --- src/codetext/parser/ruby_parser.py | 5 +++-- tests/test_parser/test_ruby.py | 14 +++++++++++--- tests/test_parser/test_sample/ruby_test_sample.rb | 6 ++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/codetext/parser/ruby_parser.py b/src/codetext/parser/ruby_parser.py index 93ee79d..bbb3cd1 100644 --- a/src/codetext/parser/ruby_parser.py +++ b/src/codetext/parser/ruby_parser.py @@ -20,7 +20,8 @@ class RubyParser(LanguageParser): @staticmethod def get_function_list(node): - res = get_node_by_kind(node, ['method']) + res = get_node_by_kind(node, ['method', + 'singleton_method']) return res @staticmethod @@ -88,7 +89,7 @@ def get_function_metadata(function_node, blob=None) -> Dict[str, str]: } assert type(function_node) == tree_sitter.Node - assert function_node.type == 'method' + assert function_node.type in ['method', 'singleton_method'] for child in function_node.children: if child.type == 'identifier': diff --git a/tests/test_parser/test_ruby.py b/tests/test_parser/test_ruby.py index 848d4a2..95fe92e 100755 --- a/tests/test_parser/test_ruby.py +++ b/tests/test_parser/test_ruby.py @@ -22,7 +22,7 @@ def test_get_function_list(self): function_list = RubyParser.get_function_list(root) - self.assertEqual(len(function_list), 1) + self.assertEqual(len(function_list), 2) def test_get_class_list(self): root = self.root_node @@ -76,8 +76,8 @@ def squeeze def test_get_function_metadata(self): root = self.root_node - function = RubyParser.get_function_list(root)[0] - metadata = RubyParser.get_function_metadata(function) + _function = RubyParser.get_function_list(root)[0] + metadata = RubyParser.get_function_metadata(_function) for key in ['identifier', 'parameters', 'return_type']: self.assertTrue(key in metadata.keys()) @@ -85,6 +85,14 @@ def test_get_function_metadata(self): self.assertEqual(metadata['parameters'], {'query': None, 'options': None}) self.assertEqual(metadata['return_type'], None) + _singleton = RubyParser.get_function_list(root)[1] + metadata = RubyParser.get_function_metadata(_singleton) + for key in ['identifier', 'parameters', 'return_type']: + self.assertTrue(key in metadata.keys()) + self.assertEqual(metadata['identifier'], 'my_method') + self.assertEqual(metadata['parameters'], {'a': None}) + self.assertEqual(metadata['return_type'], '') + def test_metadata_without_return_statement(self): code_sample = ''' diff --git a/tests/test_parser/test_sample/ruby_test_sample.rb b/tests/test_parser/test_sample/ruby_test_sample.rb index 1c82cc8..381dbf4 100755 --- a/tests/test_parser/test_sample/ruby_test_sample.rb +++ b/tests/test_parser/test_sample/ruby_test_sample.rb @@ -32,6 +32,12 @@ def search(query, options = {}) objects_from_response(:get, path, parameters) end + + def self.my_method(a) + # Method implementation + puts(a) + return a + end end end From 40c2a67ff6d2ac50afd787b1d058e7c66eb76d77 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Thu, 13 Jul 2023 11:27:27 +0700 Subject: [PATCH 4/6] Add Ruby actions with block parameters --- src/codetext/parser/ruby_parser.py | 36 +++++++++++++++++ tests/test_parser/test_ruby.py | 20 ++++++++++ .../test_sample/ruby_test_sample.rb | 40 +++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/src/codetext/parser/ruby_parser.py b/src/codetext/parser/ruby_parser.py index bbb3cd1..e9b5823 100644 --- a/src/codetext/parser/ruby_parser.py +++ b/src/codetext/parser/ruby_parser.py @@ -134,3 +134,39 @@ def get_class_metadata(class_node, blob=None): def get_comment_node(function_node): comment_node = get_node_by_kind(function_node, kind='comment') return comment_node + + @staticmethod + def get_action_list(action_node): + call_nodes = get_node_by_kind(action_node, ['call']) + res = [] + for call_node in call_nodes: + if get_node_by_kind(call_node, ["do_block"]): + res.append(call_node) + # print(res) + return res + + @staticmethod + def get_action_metadata(action_node): + metadata = { + 'identifier': '', + 'parameters': {}, + 'return_type': None, + } + + for child in action_node.children: + if child.type in ["identifier"]: + metadata['identifier'] = get_node_text(child) + if child.type in ["argument_list"]: + symbol = get_node_by_kind(child, ["simple_symbol"]) + if symbol: + metadata['identifier'] += get_node_text(symbol[0]) + + parameters = get_node_by_kind(action_node, ["block_parameters"]) + + if parameters: + for param in get_node_by_kind(parameters[0], ["identifier"]): + param_name = get_node_text(param) + metadata['parameters'].update({param_name : None}) + + return metadata + diff --git a/tests/test_parser/test_ruby.py b/tests/test_parser/test_ruby.py index 95fe92e..dc898a8 100755 --- a/tests/test_parser/test_ruby.py +++ b/tests/test_parser/test_ruby.py @@ -122,6 +122,26 @@ def test_get_class_metadata(self): self.assertEqual(metadata['identifier'], 'Client') self.assertEqual(metadata['parameters'], {'API': None}) + def test_get_action_list(self): + root = self.root_node + actions = RubyParser.get_action_list(root) + + self.assertEqual(len(actions), 5) + + def test_get_action_metadata(self): + root = self.root_node + actions = RubyParser.get_action_list(root) + metadatas = [ RubyParser.get_action_metadata(action) for action in actions] + self.assertEqual(metadatas[0]["identifier"], "load_current_value") + self.assertEqual(metadatas[1]["identifier"], "action:install") + self.assertEqual(metadatas[2]["identifier"], "converge_by") + + self.assertEqual(metadatas[3]["identifier"], "action:reinstall") + self.assertEqual(metadatas[4]["identifier"], "converge_by") + + self.assertEqual(metadatas[0]["parameters"]["new_resource"], None) + self.assertEqual(metadatas[0]["parameters"]["old_resource"], None) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_parser/test_sample/ruby_test_sample.rb b/tests/test_parser/test_sample/ruby_test_sample.rb index 381dbf4..acaf425 100755 --- a/tests/test_parser/test_sample/ruby_test_sample.rb +++ b/tests/test_parser/test_sample/ruby_test_sample.rb @@ -42,4 +42,44 @@ def self.my_method(a) end end end + +load_current_value do |new_resource, old_resource| + unless current_installed_version(new_resource).nil? + version(current_installed_version(new_resource)) + Chef::Log.debug("Current version is #{version}") if version + return a + end + end + + action :install do + build_essential + + install_version = new_resource.version unless new_resource.version.nil? || new_resource.version == current_resource.version + versions_match = candidate_version == current_installed_version(new_resource) + + if install_version || new_resource.version.nil? && !versions_match + converge_by("install package #{new_resource.package_name} #{install_version}") do + info_output = "Installing #{new_resource.package_name}" + info_output << " version #{install_version}" if install_version && !install_version.empty? + Chef::Log.info(info_output) + install_package(new_resource.package_name, install_version) + end + end + end + + action :reinstall do + build_essential + + install_version = new_resource.version unless new_resource.version.nil? + converge_by("reinstall package #{new_resource.package_name} #{install_version}") do + info_output = "Installing #{new_resource.package_name}" + info_output << " version #{install_version}" if install_version && !install_version.empty? + Chef::Log.info(info_output) + install_package(new_resource.package_name, install_version, force: true) + end + end + +a = 1 + +reinstall \ No newline at end of file From 35117b7d4f1aa5ec3030495a16304ace9530d36e Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Tue, 15 Aug 2023 01:16:12 +0700 Subject: [PATCH 5/6] Python: Handle class definitions with empty argument list class ABC() --- src/codetext/parser/python_parser.py | 4 +++- tests/test_parser/test_python.py | 26 +++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/codetext/parser/python_parser.py b/src/codetext/parser/python_parser.py index c54ff5c..8dd384e 100644 --- a/src/codetext/parser/python_parser.py +++ b/src/codetext/parser/python_parser.py @@ -114,7 +114,9 @@ def get_class_metadata(class_node, blob: str=None) -> Dict[str, str]: argument_list = get_node_text(child).split(',') for arg in argument_list: item = re.sub(r'[^a-zA-Z0-9\_]', ' ', arg).split() - metadata['parameters'][item[0].strip()] = None + # Handle class definitions with empty argument list class ABC() + if len(item) > 0: + metadata['parameters'][item[0].strip()] = None # get __init__ function return metadata diff --git a/tests/test_parser/test_python.py b/tests/test_parser/test_python.py index ae67239..43ca72a 100755 --- a/tests/test_parser/test_python.py +++ b/tests/test_parser/test_python.py @@ -59,21 +59,41 @@ def test_sample(arg1: str = "string", arg2 = "another_string"): def test_get_class_metadata(self): code_sample = ''' + class ABC() + pass + class Sample(ABC): def __init__(self): pass def test_sample(self, arg1: str = "string", arg2 = "another_string"): return NotImplement() + + class DEF(ABC, Sample): + pass ''' root = parse_code(code_sample, 'python').root_node - classes = list(PythonParser.get_class_list(root))[0] - metadata = PythonParser.get_class_metadata(classes) - + + classes = list(PythonParser.get_class_list(root)) + self.assertEqual(len(classes), 3) + + metadata = PythonParser.get_class_metadata(classes[0]) + self.assertEqual(metadata['parameters'], {}) + self.assertEqual(metadata['identifier'], 'ABC') + + + metadata = PythonParser.get_class_metadata(classes[1]) self.assertEqual(metadata['parameters'], {'ABC': None}) self.assertEqual(metadata['identifier'], 'Sample') + + metadata = PythonParser.get_class_metadata(classes[2]) + self.assertEqual(metadata['parameters'], [{'ABC': None}, {'Sample': None}]) + self.assertEqual(metadata['identifier'], 'DEF') + + + def test_get_comment_list(self): root = self.root_node From 0d16fb914d7f70a1ad4d8530679b5974ed347094 Mon Sep 17 00:00:00 2001 From: minhna1112 Date: Tue, 15 Aug 2023 01:32:35 +0700 Subject: [PATCH 6/6] Python: Refine testcase for class metadata tests --- tests/test_parser/test_python.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_parser/test_python.py b/tests/test_parser/test_python.py index 43ca72a..8d6156c 100755 --- a/tests/test_parser/test_python.py +++ b/tests/test_parser/test_python.py @@ -27,7 +27,6 @@ def test_get_class_list(self): root = self.root_node class_list = PythonParser.get_class_list(root) - self.assertEqual(len(class_list), 1) def test_get_docstring(self): @@ -59,7 +58,7 @@ def test_sample(arg1: str = "string", arg2 = "another_string"): def test_get_class_metadata(self): code_sample = ''' - class ABC() + class ABC(): pass class Sample(ABC): @@ -69,7 +68,7 @@ def __init__(self): def test_sample(self, arg1: str = "string", arg2 = "another_string"): return NotImplement() - class DEF(ABC, Sample): + class ThisIsalsoAclass(ABC, Sample): pass ''' root = parse_code(code_sample, 'python').root_node @@ -89,8 +88,8 @@ class DEF(ABC, Sample): metadata = PythonParser.get_class_metadata(classes[2]) - self.assertEqual(metadata['parameters'], [{'ABC': None}, {'Sample': None}]) - self.assertEqual(metadata['identifier'], 'DEF') + self.assertEqual(metadata['parameters'], {'ABC': None, 'Sample': None}) + self.assertEqual(metadata['identifier'], 'ThisIsalsoAclass')