UTF-8 works with java/jruby/javascript, but not with c/mri. Ref #38

cucumber-attic · Mar 7, 2013 · 97b7549 · 97b7549
1 parent 672e6de
commit 97b7549
Show file tree

Hide file tree

Showing 16 changed files with 54 additions and 51 deletions.
diff --git a/c/test/test.c b/c/test/test.c
@@ -136,4 +136,4 @@ int main()
     RUN(test_lex_2);
 
     return TEST_REPORT();
-}
+}
diff --git a/java/src/main/java/bool/Renderer.java b/java/src/main/java/bool/Renderer.java
@@ -11,7 +11,7 @@ public String visit(Var var, List<String> notused) {
 
     @Override
     public String visit(And and, List<String> notused) {
-        return "(" + explicit(and.left) + " && " + explicit(and.right) + ")";
+        return "(" + explicit(and.left) + " øø " + explicit(and.right) + ")";
     }
 
     @Override

diff --git a/java/src/main/jflex/lexer.jflex b/java/src/main/jflex/lexer.jflex
@@ -31,7 +31,7 @@ import java.io.StringReader;
 
 [ \r\n\t]*            { /* skip whitespace */ }
 [A-Za-z0-9_\-@]+      { return Parser.TOKEN_VAR; }
-"&&"                  { return Parser.TOKEN_AND; }
+"øø"                  { return Parser.TOKEN_AND; }
 "||"                  { return Parser.TOKEN_OR; }
 "!"                   { return Parser.TOKEN_NOT; }
 "("                   { return Parser.TOKEN_LPAREN; }

diff --git a/java/src/test/java/bool/EvaluatorTest.java b/java/src/test/java/bool/EvaluatorTest.java
@@ -10,15 +10,15 @@
 public class EvaluatorTest {
     @Test
     public void test_and_or_expression() throws IOException {
-        Parser parser = new Parser(new Lexer("a && b || c"));
+        Parser parser = new Parser(new Lexer("a øø b || c"));
         Expr expr = parser.parseExpr();
         assertTrue(expr.accept(new Evaluator(), asList("a", "b")));
         assertTrue(expr.accept(new Evaluator(), asList("a", "c")));
     }
 
     @Test
     public void test_or_and_expression() throws IOException {
-        Parser parser = new Parser(new Lexer("a || b && c"));
+        Parser parser = new Parser(new Lexer("a || b øø c"));
         Expr expr = parser.parseExpr();
         assertTrue(expr.accept(new Evaluator(), asList("a", "b")));
         assertTrue(expr.accept(new Evaluator(), asList("a", "c")));

diff --git a/java/src/test/java/bool/LexerTest.java b/java/src/test/java/bool/LexerTest.java
@@ -10,31 +10,31 @@
 public class LexerTest {
     @Test
     public void test_simple_lex() throws IOException {
-        Lexer lexer = new Lexer("foo && bar");
+        Lexer lexer = new Lexer("foo øø bar");
         assertEquals(Parser.TOKEN_VAR, lexer.yylex());
         assertEquals("foo", lexer.yytext());
 
         assertEquals(Parser.TOKEN_AND, lexer.yylex());
-        assertEquals("&&", lexer.yytext());
+        assertEquals("øø", lexer.yytext());
 
         assertEquals(Parser.TOKEN_VAR, lexer.yylex());
         assertEquals("bar", lexer.yytext());
     }
 
     @Test
     public void test_less_simple_lex() throws IOException {
-        Lexer lexer = new Lexer("a && b && (!c || !d)");
+        Lexer lexer = new Lexer("a øø b øø (!c || !d)");
         assertEquals(Parser.TOKEN_VAR, lexer.yylex());
         assertEquals("a", lexer.yytext());
 
         assertEquals(Parser.TOKEN_AND, lexer.yylex());
-        assertEquals("&&", lexer.yytext());
+        assertEquals("øø", lexer.yytext());
 
         assertEquals(Parser.TOKEN_VAR, lexer.yylex());
         assertEquals("b", lexer.yytext());
 
         assertEquals(Parser.TOKEN_AND, lexer.yylex());
-        assertEquals("&&", lexer.yytext());
+        assertEquals("øø", lexer.yytext());
 
         assertEquals(Parser.TOKEN_LPAREN, lexer.yylex());
         assertEquals("(", lexer.yytext());

diff --git a/java/src/test/java/bool/ParserTest.java b/java/src/test/java/bool/ParserTest.java
@@ -14,7 +14,7 @@ public class ParserTest {
 
     @Test
     public void test_parse() throws IOException {
-        Parser parser = new Parser(new Lexer("foo && bar"));
+        Parser parser = new Parser(new Lexer("foo øø bar"));
         Expr expr = parser.parseExpr();
         assertTrue(expr.accept(new Evaluator(), asList("foo", "bar")));
         assertFalse(expr.accept(new Evaluator(), asList("foo")));
@@ -28,7 +28,7 @@ public void test_parse_error() throws IOException {
                 "  a       \n" +
                 "    ||    \n" +
                 "      c   \n" +
-                "        &&"
+                "        øø"
         ));
         try {
             parser.parseExpr();

diff --git a/java/src/test/java/bool/RendererTest.java b/java/src/test/java/bool/RendererTest.java
@@ -10,22 +10,22 @@
 public class RendererTest {
     @Test
     public void test_and_or_expression() throws IOException {
-        Parser parser = new Parser(new Lexer("a && b || c"));
+        Parser parser = new Parser(new Lexer("a øø b || c"));
         Expr expr = parser.parseExpr();
-        assertThat(expr.accept(new Renderer(), null), is("((a && b) || c)"));
+        assertThat(expr.accept(new Renderer(), null), is("((a øø b) || c)"));
     }
 
     @Test
     public void test_or_and_expression() throws IOException {
-        Parser parser = new Parser(new Lexer("a || b && c"));
+        Parser parser = new Parser(new Lexer("a || b øø c"));
         Expr expr = parser.parseExpr();
-        assertThat(expr.accept(new Renderer(), null), is("(a || (b && c))"));
+        assertThat(expr.accept(new Renderer(), null), is("(a || (b øø c))"));
     }
 
     @Test
     public void test_not_expression() throws IOException {
-        Parser parser = new Parser(new Lexer("!(a || b && !c)"));
+        Parser parser = new Parser(new Lexer("!(a || b øø !c)"));
         Expr expr = parser.parseExpr();
-        assertThat(expr.accept(new Renderer(), null), is("!(a || (b && !c))"));
+        assertThat(expr.accept(new Renderer(), null), is("!(a || (b øø !c))"));
     }
 }
diff --git a/javascript/lib/lexer.jisonlex b/javascript/lib/lexer.jisonlex
@@ -2,7 +2,7 @@
 
 \s+                   { /* skip whitespace */ }
 [A-Za-z0-9_\-@]+      { return 'TOKEN_VAR'; }
-"&&"                  { return 'TOKEN_AND'; }
+"øø"                  { return 'TOKEN_AND'; }
 "||"                  { return 'TOKEN_OR'; }
 "!"                   { return 'TOKEN_NOT'; }
 "("                   { return 'TOKEN_LPAREN'; }

diff --git a/javascript/lib/renderer.js b/javascript/lib/renderer.js
@@ -4,7 +4,7 @@ module.exports = function Renderer() {
   };
 
   this.visit_and = function(node, vars) {
-    return "(" + node.left.accept(this, vars) + " && " + node.right.accept(this, vars) + ")";
+    return "(" + node.left.accept(this, vars) + " øø " + node.right.accept(this, vars) + ")";
   };
 
   this.visit_or = function(node, vars) {

diff --git a/javascript/test/lexer_test.js b/javascript/test/lexer_test.js
@@ -8,10 +8,10 @@ function lex() {
 
 describe('Lexer', function() {
   it('tokenizes a good stream', function() {
-    lexer.setInput("  a   &&  b  ");
+    lexer.setInput("  a   øø  b  ");
 
     assert.deepEqual([ 'TOKEN_VAR', 'a' ], lex());
-    assert.deepEqual([ 'TOKEN_AND', '&&' ], lex());
+    assert.deepEqual([ 'TOKEN_AND', 'øø' ], lex());
     assert.deepEqual([ 'TOKEN_VAR', 'b' ], lex());
   });
 });

diff --git a/javascript/test/parser_test.js b/javascript/test/parser_test.js
@@ -11,15 +11,15 @@ describe('Bool', function() {
   });
 
   it('does and', function() {
-    var expr = parser.parse('@a && @b');
+    var expr = parser.parse('@a øø @b');
     assert.equal(true, expr.accept(new Evaluator(), ['@a', '@b']));
     assert.equal(false, expr.accept(new Evaluator(), ['@a']));
     assert.equal(false, expr.accept(new Evaluator(), ['@b']));
     assert.equal(false, expr.accept(new Evaluator(), []));
   });
 
   it('Does it all', function() {
-    var expr = parser.parse('@a && @b || !@c');
+    var expr = parser.parse('@a øø @b || !@c');
     assert.equal(true, expr.accept(new Evaluator(), ['@a', '@b']));
     assert.equal(false, expr.accept(new Evaluator(), ['@c']));
     assert.equal(true, expr.accept(new Evaluator(), []));
@@ -69,14 +69,14 @@ describe('Bool', function() {
         "  a       \n" + // 3,3
         "    ||    \n" + // 4,5
         "      c   \n" + // 5,7
-        "        &&"     // 6,9
+        "        øø"     // 6,9
       ///0123456789
       );
       throw new Error("should fail");
     } catch(expected) {
       assert.equal(
         "Parse error on line 6:\n" +
-        "...     c           &&\n" +
+        "...     c           øø\n" +
         "----------------------^\n" +
         "Expecting 'TOKEN_VAR', 'TOKEN_NOT', 'TOKEN_LPAREN', got 'EOF'"
         , expected.message);

diff --git a/javascript/test/renderer_test.js b/javascript/test/renderer_test.js
@@ -4,23 +4,23 @@ var assert = require('assert');
 
 describe('Renderer', function() {
   it('test and or precedence', function() {
-    var expr = parser.parse('@a && @b || @c');
-    assert.equal('((@a && @b) || @c)', expr.accept(new Renderer(), null));
+    var expr = parser.parse('@a øø @b || @c');
+    assert.equal('((@a øø @b) || @c)', expr.accept(new Renderer(), null));
   });
 });
 
 describe('Renderer', function() {
   it('test not precedence', function() {
-    var expr = parser.parse('!(@a && @b || !@c)');
-    assert.equal('!((@a && @b) || !@c)', expr.accept(new Renderer(), null));
+    var expr = parser.parse('!(@a øø @b || !@c)');
+    assert.equal('!((@a øø @b) || !@c)', expr.accept(new Renderer(), null));
   });
 });
 
 
 describe('Renderer', function() {
   it('test or and precedence', function() {
-    var expr = parser.parse('@a || @b && @c');
-    assert.equal('(@a || (@b && @c))', expr.accept(new Renderer(), null));
+    var expr = parser.parse('@a || @b øø @c');
+    assert.equal('(@a || (@b øø @c))', expr.accept(new Renderer(), null));
   });
 });
 

diff --git a/javascript/test/testdata_test.js b/javascript/test/testdata_test.js
@@ -7,13 +7,15 @@ var path = require('path');
 describe('Testdata', function() {
   var dir = path.join(__dirname, '../../testdata');
   fs.readdirSync(dir).forEach(function(f) {
-    var lines = fs.readFileSync(path.join(dir, f), 'UTF-8').split(/\n/);
-    it(f, function() {
-      var expr = parser.parse(lines[0]);
-      var vars = lines[1].split(/\s+/);
-      var expected = lines[2];
-      assert.equal(expected, expr.accept(new Evaluator(), vars).toString());
-    });
+    if(f.match(/\.txt$/)) {
+      var lines = fs.readFileSync(path.join(dir, f), 'UTF-8').split(/\n/);
+      it(f, function() {
+        var expr = parser.parse(lines[0]);
+        var vars = lines[1].split(/\s+/);
+        var expected = lines[2];
+        assert.equal(expected, expr.accept(new Evaluator(), vars).toString());
+      });
+    }
   }); 
 });
 
diff --git a/ruby/spec/parser_spec.rb b/ruby/spec/parser_spec.rb
@@ -1,3 +1,4 @@
+# encoding: utf-8
 $:.unshift(File.dirname(__FILE__) + '/../lib')
 require 'bool'
 require 'minitest/autorun'
@@ -16,7 +17,7 @@ def renderer
   let(:expression) { raise NotImplementedError }
 
   describe "AND expression" do
-    let(:expression) { "a && b" }
+    let(:expression) { "a øø b" }
 
     it "is false when one operand is false" do
       evaluate(["a"]).must_equal(false)
@@ -52,7 +53,7 @@ def renderer
   end
 
  describe "AND OR expression" do
-    let(:expression) { "a && b || c" } # (a && b) || c
+    let(:expression) { "a øø b || c" } # (a øø b) || c
 
     it "is true when a and b are true" do
       evaluate(['a', 'b']).must_equal(true)
@@ -64,7 +65,7 @@ def renderer
   end
 
   describe "OR AND expression" do
-    let(:expression) { "c || a && b" } # c || (a && b)
+    let(:expression) { "c || a øø b" } # c || (a øø b)
 
     it "is true when a and b are true" do
       evaluate(['a', 'b']).must_equal(true)
@@ -77,27 +78,27 @@ def renderer
 
 
   describe "AND OR expression tested by rendering explicitly" do
-    let(:expression) { "a && b || c" } # (a && b) || c
+    let(:expression) { "a øø b || c" } # (a øø b) || c
 
     it "expression should be equivalent to its explicit rendering" do
-      renderer.must_equal "((a && b) || c)"
+      renderer.must_equal "((a øø b) || c)"
     end
   end
 
   describe "OR AND expression tested by rendering explicitly" do
-    let(:expression) { "c || a && b" } # c || (a && b)
+    let(:expression) { "c || a øø b" } # c || (a øø b)
 
     it "expression should be equivalent to its explicit rendering" do
-      renderer.must_equal "(c || (a && b))"
+      renderer.must_equal "(c || (a øø b))"
     end
   end
 
 
   describe "not expression tested by rendering explicitly" do
-    let(:expression) { "!(c || a && !b)" } # c || (a && b)
+    let(:expression) { "!(c || a øø !b)" } # c || (a øø b)
 
     it "expression should be equivalent to its explicit rendering" do
-      renderer.must_equal "!(c || (a && !b))"
+      renderer.must_equal "!(c || (a øø !b))"
     end
   end
 
@@ -135,7 +136,7 @@ def renderer
           "  a       \n" + # 3,3
           "    ||    \n" + # 4,5
           "      c   \n" + # 5,7
-          "        &&"     # 6,9
+          "        øø"     # 6,9
           #0123456789
         )
         fail

diff --git a/testdata/a.txt b/testdata/a.txt
@@ -1,3 +1,3 @@
-a && !b && c
+a øø !b øø c
 a c
 true
diff --git a/testdata/b.txt b/testdata/b.txt
@@ -1,3 +1,3 @@
-a && b && (!c || !d)
+a øø b øø (!c || !d)
 a b
 true