From 97b75493627ab6f503cbdd44802517268fe2247a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aslak=20Helles=C3=B8y?= Date: Thu, 7 Mar 2013 07:22:24 +0000 Subject: [PATCH] UTF-8 works with java/jruby/javascript, but not with c/mri. Ref #38 --- c/test/test.c | 2 +- java/src/main/java/bool/Renderer.java | 2 +- java/src/main/jflex/lexer.jflex | 2 +- java/src/test/java/bool/EvaluatorTest.java | 4 ++-- java/src/test/java/bool/LexerTest.java | 10 +++++----- java/src/test/java/bool/ParserTest.java | 4 ++-- java/src/test/java/bool/RendererTest.java | 12 ++++++------ javascript/lib/lexer.jisonlex | 2 +- javascript/lib/renderer.js | 2 +- javascript/test/lexer_test.js | 4 ++-- javascript/test/parser_test.js | 8 ++++---- javascript/test/renderer_test.js | 12 ++++++------ javascript/test/testdata_test.js | 16 +++++++++------- ruby/spec/parser_spec.rb | 21 +++++++++++---------- testdata/a.txt | 2 +- testdata/b.txt | 2 +- 16 files changed, 54 insertions(+), 51 deletions(-) diff --git a/c/test/test.c b/c/test/test.c index 0080da6..eb9eb51 100644 --- a/c/test/test.c +++ b/c/test/test.c @@ -136,4 +136,4 @@ int main() RUN(test_lex_2); return TEST_REPORT(); -} \ No newline at end of file +} diff --git a/java/src/main/java/bool/Renderer.java b/java/src/main/java/bool/Renderer.java index 09917de..8866232 100644 --- a/java/src/main/java/bool/Renderer.java +++ b/java/src/main/java/bool/Renderer.java @@ -11,7 +11,7 @@ public String visit(Var var, List notused) { @Override public String visit(And and, List notused) { - return "(" + explicit(and.left) + " && " + explicit(and.right) + ")"; + return "(" + explicit(and.left) + " øø " + explicit(and.right) + ")"; } @Override diff --git a/java/src/main/jflex/lexer.jflex b/java/src/main/jflex/lexer.jflex index 99df560..d352cff 100644 --- a/java/src/main/jflex/lexer.jflex +++ b/java/src/main/jflex/lexer.jflex @@ -31,7 +31,7 @@ import java.io.StringReader; [ \r\n\t]* { /* skip whitespace */ } [A-Za-z0-9_\-@]+ { return Parser.TOKEN_VAR; } -"&&" { return Parser.TOKEN_AND; } +"øø" { return Parser.TOKEN_AND; } "||" { return Parser.TOKEN_OR; } "!" { return Parser.TOKEN_NOT; } "(" { return Parser.TOKEN_LPAREN; } diff --git a/java/src/test/java/bool/EvaluatorTest.java b/java/src/test/java/bool/EvaluatorTest.java index b51cb34..023248a 100644 --- a/java/src/test/java/bool/EvaluatorTest.java +++ b/java/src/test/java/bool/EvaluatorTest.java @@ -10,7 +10,7 @@ public class EvaluatorTest { @Test public void test_and_or_expression() throws IOException { - Parser parser = new Parser(new Lexer("a && b || c")); + Parser parser = new Parser(new Lexer("a øø b || c")); Expr expr = parser.parseExpr(); assertTrue(expr.accept(new Evaluator(), asList("a", "b"))); assertTrue(expr.accept(new Evaluator(), asList("a", "c"))); @@ -18,7 +18,7 @@ public void test_and_or_expression() throws IOException { @Test public void test_or_and_expression() throws IOException { - Parser parser = new Parser(new Lexer("a || b && c")); + Parser parser = new Parser(new Lexer("a || b øø c")); Expr expr = parser.parseExpr(); assertTrue(expr.accept(new Evaluator(), asList("a", "b"))); assertTrue(expr.accept(new Evaluator(), asList("a", "c"))); diff --git a/java/src/test/java/bool/LexerTest.java b/java/src/test/java/bool/LexerTest.java index 148a91b..91247ba 100644 --- a/java/src/test/java/bool/LexerTest.java +++ b/java/src/test/java/bool/LexerTest.java @@ -10,12 +10,12 @@ public class LexerTest { @Test public void test_simple_lex() throws IOException { - Lexer lexer = new Lexer("foo && bar"); + Lexer lexer = new Lexer("foo øø bar"); assertEquals(Parser.TOKEN_VAR, lexer.yylex()); assertEquals("foo", lexer.yytext()); assertEquals(Parser.TOKEN_AND, lexer.yylex()); - assertEquals("&&", lexer.yytext()); + assertEquals("øø", lexer.yytext()); assertEquals(Parser.TOKEN_VAR, lexer.yylex()); assertEquals("bar", lexer.yytext()); @@ -23,18 +23,18 @@ public void test_simple_lex() throws IOException { @Test public void test_less_simple_lex() throws IOException { - Lexer lexer = new Lexer("a && b && (!c || !d)"); + Lexer lexer = new Lexer("a øø b øø (!c || !d)"); assertEquals(Parser.TOKEN_VAR, lexer.yylex()); assertEquals("a", lexer.yytext()); assertEquals(Parser.TOKEN_AND, lexer.yylex()); - assertEquals("&&", lexer.yytext()); + assertEquals("øø", lexer.yytext()); assertEquals(Parser.TOKEN_VAR, lexer.yylex()); assertEquals("b", lexer.yytext()); assertEquals(Parser.TOKEN_AND, lexer.yylex()); - assertEquals("&&", lexer.yytext()); + assertEquals("øø", lexer.yytext()); assertEquals(Parser.TOKEN_LPAREN, lexer.yylex()); assertEquals("(", lexer.yytext()); diff --git a/java/src/test/java/bool/ParserTest.java b/java/src/test/java/bool/ParserTest.java index 2384a06..13dfb75 100644 --- a/java/src/test/java/bool/ParserTest.java +++ b/java/src/test/java/bool/ParserTest.java @@ -14,7 +14,7 @@ public class ParserTest { @Test public void test_parse() throws IOException { - Parser parser = new Parser(new Lexer("foo && bar")); + Parser parser = new Parser(new Lexer("foo øø bar")); Expr expr = parser.parseExpr(); assertTrue(expr.accept(new Evaluator(), asList("foo", "bar"))); assertFalse(expr.accept(new Evaluator(), asList("foo"))); @@ -28,7 +28,7 @@ public void test_parse_error() throws IOException { " a \n" + " || \n" + " c \n" + - " &&" + " øø" )); try { parser.parseExpr(); diff --git a/java/src/test/java/bool/RendererTest.java b/java/src/test/java/bool/RendererTest.java index dbd4a26..9ffe78e 100644 --- a/java/src/test/java/bool/RendererTest.java +++ b/java/src/test/java/bool/RendererTest.java @@ -10,22 +10,22 @@ public class RendererTest { @Test public void test_and_or_expression() throws IOException { - Parser parser = new Parser(new Lexer("a && b || c")); + Parser parser = new Parser(new Lexer("a øø b || c")); Expr expr = parser.parseExpr(); - assertThat(expr.accept(new Renderer(), null), is("((a && b) || c)")); + assertThat(expr.accept(new Renderer(), null), is("((a øø b) || c)")); } @Test public void test_or_and_expression() throws IOException { - Parser parser = new Parser(new Lexer("a || b && c")); + Parser parser = new Parser(new Lexer("a || b øø c")); Expr expr = parser.parseExpr(); - assertThat(expr.accept(new Renderer(), null), is("(a || (b && c))")); + assertThat(expr.accept(new Renderer(), null), is("(a || (b øø c))")); } @Test public void test_not_expression() throws IOException { - Parser parser = new Parser(new Lexer("!(a || b && !c)")); + Parser parser = new Parser(new Lexer("!(a || b øø !c)")); Expr expr = parser.parseExpr(); - assertThat(expr.accept(new Renderer(), null), is("!(a || (b && !c))")); + assertThat(expr.accept(new Renderer(), null), is("!(a || (b øø !c))")); } } diff --git a/javascript/lib/lexer.jisonlex b/javascript/lib/lexer.jisonlex index fb02472..bb267c5 100644 --- a/javascript/lib/lexer.jisonlex +++ b/javascript/lib/lexer.jisonlex @@ -2,7 +2,7 @@ \s+ { /* skip whitespace */ } [A-Za-z0-9_\-@]+ { return 'TOKEN_VAR'; } -"&&" { return 'TOKEN_AND'; } +"øø" { return 'TOKEN_AND'; } "||" { return 'TOKEN_OR'; } "!" { return 'TOKEN_NOT'; } "(" { return 'TOKEN_LPAREN'; } diff --git a/javascript/lib/renderer.js b/javascript/lib/renderer.js index 231778d..08c1098 100644 --- a/javascript/lib/renderer.js +++ b/javascript/lib/renderer.js @@ -4,7 +4,7 @@ module.exports = function Renderer() { }; this.visit_and = function(node, vars) { - return "(" + node.left.accept(this, vars) + " && " + node.right.accept(this, vars) + ")"; + return "(" + node.left.accept(this, vars) + " øø " + node.right.accept(this, vars) + ")"; }; this.visit_or = function(node, vars) { diff --git a/javascript/test/lexer_test.js b/javascript/test/lexer_test.js index b96f6b2..6ae882d 100644 --- a/javascript/test/lexer_test.js +++ b/javascript/test/lexer_test.js @@ -8,10 +8,10 @@ function lex() { describe('Lexer', function() { it('tokenizes a good stream', function() { - lexer.setInput(" a && b "); + lexer.setInput(" a øø b "); assert.deepEqual([ 'TOKEN_VAR', 'a' ], lex()); - assert.deepEqual([ 'TOKEN_AND', '&&' ], lex()); + assert.deepEqual([ 'TOKEN_AND', 'øø' ], lex()); assert.deepEqual([ 'TOKEN_VAR', 'b' ], lex()); }); }); diff --git a/javascript/test/parser_test.js b/javascript/test/parser_test.js index 6d0dfdf..ceeb80b 100644 --- a/javascript/test/parser_test.js +++ b/javascript/test/parser_test.js @@ -11,7 +11,7 @@ describe('Bool', function() { }); it('does and', function() { - var expr = parser.parse('@a && @b'); + var expr = parser.parse('@a øø @b'); assert.equal(true, expr.accept(new Evaluator(), ['@a', '@b'])); assert.equal(false, expr.accept(new Evaluator(), ['@a'])); assert.equal(false, expr.accept(new Evaluator(), ['@b'])); @@ -19,7 +19,7 @@ describe('Bool', function() { }); it('Does it all', function() { - var expr = parser.parse('@a && @b || !@c'); + var expr = parser.parse('@a øø @b || !@c'); assert.equal(true, expr.accept(new Evaluator(), ['@a', '@b'])); assert.equal(false, expr.accept(new Evaluator(), ['@c'])); assert.equal(true, expr.accept(new Evaluator(), [])); @@ -69,14 +69,14 @@ describe('Bool', function() { " a \n" + // 3,3 " || \n" + // 4,5 " c \n" + // 5,7 - " &&" // 6,9 + " øø" // 6,9 ///0123456789 ); throw new Error("should fail"); } catch(expected) { assert.equal( "Parse error on line 6:\n" + - "... c &&\n" + + "... c øø\n" + "----------------------^\n" + "Expecting 'TOKEN_VAR', 'TOKEN_NOT', 'TOKEN_LPAREN', got 'EOF'" , expected.message); diff --git a/javascript/test/renderer_test.js b/javascript/test/renderer_test.js index 93bc758..7e8ff00 100644 --- a/javascript/test/renderer_test.js +++ b/javascript/test/renderer_test.js @@ -4,23 +4,23 @@ var assert = require('assert'); describe('Renderer', function() { it('test and or precedence', function() { - var expr = parser.parse('@a && @b || @c'); - assert.equal('((@a && @b) || @c)', expr.accept(new Renderer(), null)); + var expr = parser.parse('@a øø @b || @c'); + assert.equal('((@a øø @b) || @c)', expr.accept(new Renderer(), null)); }); }); describe('Renderer', function() { it('test not precedence', function() { - var expr = parser.parse('!(@a && @b || !@c)'); - assert.equal('!((@a && @b) || !@c)', expr.accept(new Renderer(), null)); + var expr = parser.parse('!(@a øø @b || !@c)'); + assert.equal('!((@a øø @b) || !@c)', expr.accept(new Renderer(), null)); }); }); describe('Renderer', function() { it('test or and precedence', function() { - var expr = parser.parse('@a || @b && @c'); - assert.equal('(@a || (@b && @c))', expr.accept(new Renderer(), null)); + var expr = parser.parse('@a || @b øø @c'); + assert.equal('(@a || (@b øø @c))', expr.accept(new Renderer(), null)); }); }); diff --git a/javascript/test/testdata_test.js b/javascript/test/testdata_test.js index f80d1e5..b596bbc 100644 --- a/javascript/test/testdata_test.js +++ b/javascript/test/testdata_test.js @@ -7,13 +7,15 @@ var path = require('path'); describe('Testdata', function() { var dir = path.join(__dirname, '../../testdata'); fs.readdirSync(dir).forEach(function(f) { - var lines = fs.readFileSync(path.join(dir, f), 'UTF-8').split(/\n/); - it(f, function() { - var expr = parser.parse(lines[0]); - var vars = lines[1].split(/\s+/); - var expected = lines[2]; - assert.equal(expected, expr.accept(new Evaluator(), vars).toString()); - }); + if(f.match(/\.txt$/)) { + var lines = fs.readFileSync(path.join(dir, f), 'UTF-8').split(/\n/); + it(f, function() { + var expr = parser.parse(lines[0]); + var vars = lines[1].split(/\s+/); + var expected = lines[2]; + assert.equal(expected, expr.accept(new Evaluator(), vars).toString()); + }); + } }); }); diff --git a/ruby/spec/parser_spec.rb b/ruby/spec/parser_spec.rb index c8ed9fb..81ab5ee 100644 --- a/ruby/spec/parser_spec.rb +++ b/ruby/spec/parser_spec.rb @@ -1,3 +1,4 @@ +# encoding: utf-8 $:.unshift(File.dirname(__FILE__) + '/../lib') require 'bool' require 'minitest/autorun' @@ -16,7 +17,7 @@ def renderer let(:expression) { raise NotImplementedError } describe "AND expression" do - let(:expression) { "a && b" } + let(:expression) { "a øø b" } it "is false when one operand is false" do evaluate(["a"]).must_equal(false) @@ -52,7 +53,7 @@ def renderer end describe "AND OR expression" do - let(:expression) { "a && b || c" } # (a && b) || c + let(:expression) { "a øø b || c" } # (a øø b) || c it "is true when a and b are true" do evaluate(['a', 'b']).must_equal(true) @@ -64,7 +65,7 @@ def renderer end describe "OR AND expression" do - let(:expression) { "c || a && b" } # c || (a && b) + let(:expression) { "c || a øø b" } # c || (a øø b) it "is true when a and b are true" do evaluate(['a', 'b']).must_equal(true) @@ -77,27 +78,27 @@ def renderer describe "AND OR expression tested by rendering explicitly" do - let(:expression) { "a && b || c" } # (a && b) || c + let(:expression) { "a øø b || c" } # (a øø b) || c it "expression should be equivalent to its explicit rendering" do - renderer.must_equal "((a && b) || c)" + renderer.must_equal "((a øø b) || c)" end end describe "OR AND expression tested by rendering explicitly" do - let(:expression) { "c || a && b" } # c || (a && b) + let(:expression) { "c || a øø b" } # c || (a øø b) it "expression should be equivalent to its explicit rendering" do - renderer.must_equal "(c || (a && b))" + renderer.must_equal "(c || (a øø b))" end end describe "not expression tested by rendering explicitly" do - let(:expression) { "!(c || a && !b)" } # c || (a && b) + let(:expression) { "!(c || a øø !b)" } # c || (a øø b) it "expression should be equivalent to its explicit rendering" do - renderer.must_equal "!(c || (a && !b))" + renderer.must_equal "!(c || (a øø !b))" end end @@ -135,7 +136,7 @@ def renderer " a \n" + # 3,3 " || \n" + # 4,5 " c \n" + # 5,7 - " &&" # 6,9 + " øø" # 6,9 #0123456789 ) fail diff --git a/testdata/a.txt b/testdata/a.txt index 39557c9..90f843b 100644 --- a/testdata/a.txt +++ b/testdata/a.txt @@ -1,3 +1,3 @@ -a && !b && c +a øø !b øø c a c true diff --git a/testdata/b.txt b/testdata/b.txt index 0ab6fc3..086f6f2 100644 --- a/testdata/b.txt +++ b/testdata/b.txt @@ -1,3 +1,3 @@ -a && b && (!c || !d) +a øø b øø (!c || !d) a b true