Merge pull request #43 from TrustNXT/feature/json-utf-8

JSONBox: Proper string encoding and decoding

src/jumbf/JSONBox.ts

@@ -3,18 +3,19 @@ import { Box } from './Box';
 import { BoxSchema } from './BoxSchema';
 import * as schemata from './schemata';
 
-// TODO: JSON is UTF-8, but we're reading bytes as if they were codepoints here
 class JSONBoxSchema extends BoxSchema<JSONBox> {
     readonly length = schemata.length;
     readonly type = schemata.type;
 
     readContent(input: bin.ISerialInput, type: string, length: number): JSONBox {
         if (type != JSONBox.typeCode) throw new Error(`JSONBox: Unexpected type ${type}`);
 
-        let json = '';
-        for (let i = 0; i < length - 8; i++) {
-            json += String.fromCharCode(input.readByte());
+        const payloadLength = length - 8;
+        const jsonBuffer = new Uint8Array(payloadLength);
+        for (let i = 0; i < payloadLength; i++) {
+            jsonBuffer[i] = input.readByte();
         }
+        const json = new TextDecoder().decode(jsonBuffer);
 
         const box = new JSONBox();
         try {
@@ -27,16 +28,21 @@ class JSONBoxSchema extends BoxSchema<JSONBox> {
         return box;
     }
 
-    writeContent(output: bin.ISerialOutput, value: JSONBox): void {
-        const json = value.content === undefined ? '' : JSON.stringify(value.content);
+    private encodeContent(value: JSONBox): Uint8Array {
+        if (!value.content) return new Uint8Array(0);
+        return new TextEncoder().encode(JSON.stringify(value.content));
+    }
 
-        for (let i = 0; i != json.length; i++) output.writeByte(json.charCodeAt(i));
+    writeContent(output: bin.ISerialOutput, value: JSONBox): void {
+        const jsonBuffer = this.encodeContent(value);
+        for (let i = 0; i != jsonBuffer.length; i++) output.writeByte(jsonBuffer[i]);
     }
 
     measureContent(value: JSONBox, measurer: bin.IMeasurer): bin.IMeasurer {
-        const json = value.content === undefined ? '' : JSON.stringify(value.content);
-
-        return measurer.add(json.length);
+        // We need to do the entire encoding twice (once to measure, once to write) which is
+        // not ideal but unavoidable without rather complicated caching and the resulting
+        // invalidation handling.
+        return measurer.add(this.encodeContent(value).length);
     }
 }
 

tests/jumbf/JSONBox.test.ts

@@ -88,4 +88,43 @@ describe('JSONBox Tests', function () {
             assert.equal(JSON.stringify(box.content), JSON.stringify({ a: 1 }));
         });
     });
+
+    describe('UTF-8', function () {
+        it('serialization', async function () {
+            const box = new JSONBox();
+            box.content = { shrug: '🤷‍♂️' };
+
+            // fetch schema from the box
+            const schema = box.schema;
+
+            // write the box to a buffer
+            const length = schema.measure(box).size;
+            const buffer = Buffer.alloc(length);
+            const writer = new bin.BufferWriter(buffer, { endianness: 'big' });
+            schema.write(writer, box);
+
+            // verify expected buffer contents
+            assert.equal(
+                BinaryHelper.toHexString(buffer),
+                '000000216a736f6e7b227368727567223a22f09fa4b7e2808de29982efb88f227d',
+            );
+        });
+
+        it('deserialization', async function () {
+            const buffer = BinaryHelper.fromHexString(
+                '000000216a736f6e7b227368727567223a22f09fa4b7e2808de29982efb88f227d',
+            );
+
+            // fetch schema from the box class
+            const schema = JSONBox.schema;
+
+            // read the box from the buffer
+            const reader = new bin.BufferReader(buffer, { endianness: 'big' });
+            const box = schema.read(reader);
+
+            // validate resulting box
+            if (!(box instanceof JSONBox)) assert.fail('resulting box has wrong type');
+            assert.equal((box.content as Record<string, string>).shrug, '🤷‍♂️');
+        });
+    });
 });