Merge pull request #6 from Jkutkut/3-empty-line-with-content-line

bugfix: lexer cleaning logic refactored

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "osmia"
-version = "1.10.0"
+version = "1.10.1"
 edition = "2021"
 authors = ["Jkutkut"]
 

src/lexer/lexer.rs

@@ -124,39 +124,49 @@ impl<'a> Lexer<'a> {
 	}
 
 	fn clean_tokens(tokens: &mut Vec<Token>) {
-		let mut i = 0;
-		while i < tokens.len() - 1 {
-			i += 1;
-			if !Self::is_new_line_token(&tokens[i - 1]) {
-				continue;
+		let mut i: usize = tokens.len() - 1;
+		let mut end: usize;
+		while i > 0 {
+			end = i;
+			while i > 0 {
+				i -= 1;
+				if Self::is_new_line_token(&tokens[i]) {
+					break;
+				}
 			}
-			if !Self::is_whitespace_token(&tokens[i]) {
-				continue;
+			if Self::is_new_line_token(&tokens[i]) {
+				i += 1;
 			}
-			let mut j = 1;
-			let mut is_printable_token_in_row = false;
-			while i + j < tokens.len() - 1 {
-				if Self::is_new_line_token(&tokens[i + j]) {
-					break;
+			let line = &tokens[i..end + 1];
+			let mut row_has_printable_token = false;
+			let mut k = 0;
+			while k < line.len() - 1 && !row_has_printable_token {
+				match &line[k] {
+					Token::Raw(_) => row_has_printable_token = !Self::is_whitespace_token(&line[k]),
+					Token::DelimiterStart => {
+						match &line[k + 1] {
+							Token::Value(_) => row_has_printable_token = true,
+							_ => ()
+						}
+					},
+					_ => ()
 				}
-				else if let Token::Raw(_) = &tokens[i + j] {
-					is_printable_token_in_row = true;
-					break;
+				k += 1;
+			}
+			if !row_has_printable_token {
+				let mut ts: Vec<Token> = Vec::new();
+				let mut line = line.to_vec();
+				if Self::is_new_line_token(&tokens[end]) {
+					ts.push(tokens.remove(end));
+					line.remove(line.len() - 1);
 				}
-				else if let Token::DelimiterStart = &tokens[i + j] {
-					match &tokens[i + j + 1] {
-						Token::Value(_) => {
-							is_printable_token_in_row = true;
-							break;
-						},
-						_ => ()
-					}
+				if Self::is_whitespace_token(&tokens[i]) {
+					ts.push(tokens.remove(i));
+					line.remove(0);
 				}
-				j += 1;
 			}
-			if !is_printable_token_in_row {
-				tokens.drain(i - 1..i + 1);
-				i = i + j - 2;
+			if i > 0 {
+				i -= 1;
 			}
 		}
 	}

src/tests/lexer/lexer.rs

@@ -266,6 +266,7 @@ macro_tests!(
 		{{continue}}{{break}}
 		"#,
 		vec![
+			Token::Raw("\n".to_string()),
 			Token::DelimiterStart,
 			Token::Print,
 			Token::DelimiterEnd,

src/tests/tests.rs

@@ -138,7 +138,7 @@ macro_tests!(
 		"#),
 		"Hello, world!
 		<a href=\"https://example01.com\">Example01</a>
-		<a href=\"https://example02.org\">Example02</a>"
+		<a href=\"https://example02.org\">Example02</a>\n"
 	),
 	(
 		foreach02,
@@ -155,7 +155,7 @@ macro_tests!(
 		"Hello, world!
 		<li>Element 1</li>
 		<li>Element 2</li>
-		<li>Element 3</li>"
+		<li>Element 3</li>\n"
 	),
 	(
 		foreach03,
@@ -229,7 +229,7 @@ macro_tests!(
 		"#),
 		"Hey!
 			True!
-			var: value"
+			var: value\n"
 	),
 	(
 		spacing01,
@@ -416,5 +416,64 @@ Jack is an adult.
 			}
 		"#),
 		"\n\n\n\t\t7\t\t\t\t\n\t\t7"
+	),
+	(
+		gh_03_08,
+		r#"
+{{for page in pages}}
+  {{if page.disabled}}
+  {{fi}}
+  # {{page.endpoint}}
+{{done}}"#,
+		Some(r#"
+			{
+				"pages": [
+					{
+						"disabled": true,
+						"endpoint": "foo"
+					},
+					{
+						"disabled": false,
+						"endpoint": "bar"
+					},
+					{
+						"disabled": false,
+						"endpoint": "baz"
+					}
+				]
+			}
+		"#),
+		"\n  # foo\n  # bar\n  # baz\n"
+	),
+	(
+		gh_03_09,
+		r#"  {{for page in pages}}{{if page.disabled}}{{fi}}
+  # {{page.endpoint}}
+{{done}}"#,
+		Some(r#"
+			{
+				"pages": [
+					{
+						"disabled": true,
+						"endpoint": "foo"
+					},
+					{
+						"disabled": false,
+						"endpoint": "bar"
+					},
+					{
+						"disabled": false,
+						"endpoint": "baz"
+					}
+				]
+			}
+		"#),
+		"  # foo\n  # bar\n  # baz\n"
+	),
+	(
+		gh_03_10,
+		"\n\n\n\n\n    {{v}}",
+		Some(r#"{"v": "foo"}"#),
+		"\n\n\n\n\n    foo"
 	)
 );