chore: remove bytebuffer dependency

This commit is contained in:
Szymon Lesisz
2023-12-12 19:06:07 +01:00
committed by martin
parent 4fe51472f3
commit 9b2f9def05
21 changed files with 104 additions and 151 deletions

View File

@@ -30,7 +30,6 @@
"prepublish": "yarn tsx ../../scripts/prepublish.js"
},
"dependencies": {
"bytebuffer": "^5.0.1",
"long": "^4.0.0",
"protobufjs": "7.2.5"
},

View File

@@ -1,4 +1,3 @@
import ByteBuffer from 'bytebuffer';
import { Type, Message, Field } from 'protobufjs/light';
import { isPrimitiveField } from './utils';
@@ -11,8 +10,7 @@ const transform = (field: Field, value: any) => {
}
if (field.type === 'bytes') {
return ByteBuffer.wrap(value).toString('hex');
// return value.toString('hex');
return Buffer.from(value).toString('hex');
}
// [compatibility]
@@ -65,10 +63,8 @@ export function messageToJSON(
return res;
}
export const decode = (MessageParam: Type, data: ByteBuffer) => {
const buff = data.toBuffer();
const a = new Uint8Array(buff);
const decoded = MessageParam.decode(a);
export const decode = (MessageParam: Type, data: Buffer) => {
const decoded = MessageParam.decode(new Uint8Array(data));
// [compatibility]: in the end it should be possible to get rid of messageToJSON method and call
// Message.toObject(decoded) to return result as plain javascript object. This method should be able to do

View File

@@ -1,4 +1,3 @@
import ByteBuffer from 'bytebuffer';
import { Type } from 'protobufjs/light';
import { isPrimitiveField } from './utils';
@@ -70,9 +69,6 @@ export const encode = (Message: Type, data: Record<string, unknown>) => {
const payload = patch(Message, data);
const message = Message.fromObject(payload);
// Encode a message to an Uint8Array (browser) or Buffer (node)
const buffer = Message.encode(message).finish();
const bytebuffer = new ByteBuffer(buffer.byteLength);
bytebuffer.append(buffer);
bytebuffer.reset();
return bytebuffer;
const bytes = Message.encode(message).finish();
return Buffer.from(bytes);
};

View File

@@ -232,8 +232,7 @@ describe('basic concepts', () => {
});
test('Different protobuf between receiving ends', () => {
/* eslint-disable-next-line @typescript-eslint/no-shadow */
const messages = {
const customMessages = (fields?: any) => ({
nested: {
messages: {
nested: {
@@ -247,23 +246,24 @@ describe('basic concepts', () => {
type: 'uint32',
id: 2,
},
...fields,
},
},
},
},
},
};
});
const SenderMessages = ProtoBuf.Root.fromJSON(messages);
const SenderMessages = ProtoBuf.Root.fromJSON(customMessages());
const senderEncoded = encode(SenderMessages.lookupType('messages.ButtonRequest'), {
type: 'foo',
pages: 123,
});
const receiverMessages = messages;
// now change field type from uint32 to string
receiverMessages.nested.messages.nested.ButtonRequest.fields.pages.type = 'string';
const ReceiverMessages = ProtoBuf.Root.fromJSON(receiverMessages);
const ReceiverMessages = ProtoBuf.Root.fromJSON(
customMessages({ pages: { type: 'string', id: 2 } }),
);
expect(() => {
decode(ReceiverMessages.lookupType('messages.ButtonRequest'), senderEncoded);

View File

@@ -24,9 +24,6 @@
"prepublishOnly": "yarn tsx ../../scripts/prepublishNPM.js",
"prepublish": "yarn tsx ../../scripts/prepublish.js"
},
"dependencies": {
"bytebuffer": "^5.0.1"
},
"devDependencies": {
"jest": "29.5.0",
"rimraf": "^5.0.5",

View File

@@ -0,0 +1 @@
export const HEADER_SIZE = 2 + 4; // messageType + dataLength. unlike protocol-v1 bridge (trezord) adds 2 bytes of MESSAGE_HEADER_BYTE to each message

View File

@@ -1,24 +1,25 @@
import ByteBuffer from 'bytebuffer';
import { HEADER_SIZE } from './constants';
import { TransportProtocolDecode } from '../types';
/**
* Reads meta information from buffer
*/
const readHeader = (buffer: ByteBuffer) => {
const typeId = buffer.readUint16();
const length = buffer.readUint32();
const readHeader = (buffer: Buffer) => {
// 2 bytes
const typeId = buffer.readUInt16BE();
// 4 bytes
const length = buffer.readUInt32BE(2);
return { typeId, length };
};
export const decode: TransportProtocolDecode = bytes => {
const byteBuffer = ByteBuffer.wrap(bytes, undefined, undefined, true);
const { typeId, length } = readHeader(byteBuffer);
const buffer = Buffer.from(bytes);
const { typeId, length } = readHeader(buffer);
return {
typeId,
length,
buffer: byteBuffer,
buffer: buffer.subarray(HEADER_SIZE),
};
};

View File

@@ -1,6 +1,4 @@
import ByteBuffer from 'bytebuffer';
import { HEADER_SIZE } from '../protocol-v1/constants';
import { HEADER_SIZE } from './constants';
import { TransportProtocolEncode } from '../types';
// this file is basically combination of "trezor v1 protocol" and "bridge protocol"
@@ -8,22 +6,17 @@ import { TransportProtocolEncode } from '../types';
// it is because bridge does some parts of the protocol itself (like chunking)
export const encode: TransportProtocolEncode = (data, options) => {
const { messageType } = options;
const fullSize = HEADER_SIZE - 2 + data.limit;
const encodedByteBuffer = new ByteBuffer(fullSize);
const encodedBuffer = Buffer.alloc(HEADER_SIZE + data.length);
// 2 bytes
encodedByteBuffer.writeUint16(messageType);
encodedBuffer.writeUInt16BE(messageType);
// 4 bytes
encodedByteBuffer.writeUint32(data.limit);
encodedBuffer.writeUInt32BE(data.length, 2);
// then put in the actual message
encodedByteBuffer.append(data.buffer);
data.copy(encodedBuffer, HEADER_SIZE);
encodedByteBuffer.reset();
// todo: it would be nicer to return Buffer instead of ByteBuffer. The problem is that ByteBuffer.Buffer.toString behaves differently in web and node.
// anyway, for now we can keep this legacy behavior
return [encodedByteBuffer];
return [encodedBuffer];
};

View File

@@ -1,21 +1,18 @@
import ByteBuffer from 'bytebuffer';
// Decode `trzd` protocol used for decoding of dynamically loaded `@trezor/protobuf` messages
// https://github.com/trezor/trezor-firmware/blob/087becd2caa5618eecab37ac3f2ca51172e52eb9/docs/common/ethereum-definitions.md#definition-format
export const decode = (bytes: ArrayBuffer) => {
const byteBuffer = ByteBuffer.wrap(bytes);
byteBuffer.LE(true); // use little endian byte order
const byteBuffer = Buffer.from(bytes);
// 5 bytes magic `trzd`
const magic = byteBuffer.readBytes(5).toUTF8();
const magic = byteBuffer.subarray(0, 5).toString('utf8');
// 1 byte
const definitionType = byteBuffer.readUint8();
const definitionType = byteBuffer.readUInt8(5);
// 4 bytes
const dataVersion = byteBuffer.readUint32();
const dataVersion = byteBuffer.readUInt32LE(6);
// 2 bytes
const protobufLength = byteBuffer.readUint16();
const protobufLength = byteBuffer.readUInt16LE(10);
// N bytes
const protobufPayload = byteBuffer.slice(12, 12 + protobufLength);
const protobufPayload = byteBuffer.subarray(12, 12 + protobufLength);
return {
magic,

View File

@@ -1,4 +1,4 @@
export const MESSAGE_MAGIC_HEADER_BYTE = 63;
export const MESSAGE_HEADER_BYTE = 0x23;
export const HEADER_SIZE = 1 + 1 + 4 + 2;
export const HEADER_SIZE = 1 + 1 + 2 + 4; // MESSAGE_HEADER_BYTE + MESSAGE_HEADER_BYTE + messageType + dataLength
export const BUFFER_SIZE = 64;

View File

@@ -1,18 +1,21 @@
import ByteBuffer from 'bytebuffer';
import * as ERRORS from '../errors';
import { MESSAGE_HEADER_BYTE, MESSAGE_MAGIC_HEADER_BYTE } from './constants';
import { HEADER_SIZE, MESSAGE_HEADER_BYTE, MESSAGE_MAGIC_HEADER_BYTE } from './constants';
import { TransportProtocolDecode } from '../types';
/**
* Reads meta information from chunked buffer
*/
const readHeaderChunked = (buffer: ByteBuffer) => {
const magic = buffer.readByte();
const sharp1 = buffer.readByte();
const sharp2 = buffer.readByte();
const typeId = buffer.readUint16();
const length = buffer.readUint32();
const readHeaderChunked = (buffer: Buffer) => {
// 1 byte
const magic = buffer.readUInt8();
// 1 byte
const sharp1 = buffer.readUInt8(1);
// 1 byte
const sharp2 = buffer.readUInt8(2);
// 2 bytes
const typeId = buffer.readUInt16BE(3);
// 4 bytes
const length = buffer.readUInt32BE(5);
return { magic, sharp1, sharp2, typeId, length };
};
@@ -20,10 +23,8 @@ const readHeaderChunked = (buffer: ByteBuffer) => {
// Parses first raw input that comes from Trezor and returns some information about the whole message.
// [compatibility]: accept Buffer just like decode does. But this would require changes in lower levels
export const decode: TransportProtocolDecode = bytes => {
// convert to ByteBuffer so it's easier to read
const byteBuffer = ByteBuffer.wrap(bytes, undefined, undefined, true);
const { magic, sharp1, sharp2, typeId, length } = readHeaderChunked(byteBuffer);
const buffer = Buffer.from(bytes);
const { magic, sharp1, sharp2, typeId, length } = readHeaderChunked(buffer);
if (
magic !== MESSAGE_MAGIC_HEADER_BYTE ||
@@ -34,5 +35,9 @@ export const decode: TransportProtocolDecode = bytes => {
throw new Error(ERRORS.PROTOCOL_MALFORMED);
}
return { length, typeId, buffer: byteBuffer };
return {
length,
typeId,
buffer: buffer.subarray(HEADER_SIZE + 1), // each chunk is prefixed by magic byte
};
};

View File

@@ -1,5 +1,3 @@
import ByteBuffer from 'bytebuffer';
import {
HEADER_SIZE,
MESSAGE_HEADER_BYTE,
@@ -10,47 +8,41 @@ import { TransportProtocolEncode } from '../types';
export const encode: TransportProtocolEncode = (data, options) => {
const { messageType } = options;
const fullSize = HEADER_SIZE + data.limit;
const fullSize = HEADER_SIZE + data.length;
const chunkSize = options.chunkSize || BUFFER_SIZE;
const encodedByteBuffer = new ByteBuffer(fullSize);
const encodedBuffer = Buffer.alloc(fullSize);
// 2*1 byte
encodedByteBuffer.writeByte(MESSAGE_HEADER_BYTE);
encodedByteBuffer.writeByte(MESSAGE_HEADER_BYTE);
encodedBuffer.writeUInt8(MESSAGE_HEADER_BYTE, 0);
encodedBuffer.writeUInt8(MESSAGE_HEADER_BYTE, 1);
// 2 bytes
encodedByteBuffer.writeUint16(messageType);
encodedBuffer.writeUInt16BE(messageType, 2);
// 4 bytes (so 8 in total)
encodedByteBuffer.writeUint32(data.limit);
encodedBuffer.writeUInt32BE(data.length, 4);
// then put in the actual message
encodedByteBuffer.append(data.buffer);
data.copy(encodedBuffer, HEADER_SIZE);
encodedByteBuffer.reset();
const size = chunkSize - 1; // chunkSize - 1 byte of MESSAGE_MAGIC_HEADER_BYTE
const size = chunkSize - 1;
const chunkCount = Math.ceil(encodedByteBuffer.limit / size) || 1;
const chunkCount = Math.ceil(encodedBuffer.length / size) || 1;
// size with one reserved byte for header
const result: ByteBuffer[] = [];
const result: Buffer[] = [];
// How many pieces will there actually be
// slice and dice
for (let i = 0; i < chunkCount; i++) {
const start = i * size;
const end = Math.min((i + 1) * size, encodedByteBuffer.limit);
const end = Math.min((i + 1) * size, encodedBuffer.length);
const buffer = new ByteBuffer(chunkSize);
const buffer = Buffer.alloc(chunkSize);
buffer.writeUInt8(MESSAGE_MAGIC_HEADER_BYTE);
encodedBuffer.copy(buffer, 1, start, end);
buffer.writeByte(MESSAGE_MAGIC_HEADER_BYTE);
const slice = encodedByteBuffer.slice(start, end);
slice.compact();
buffer.append(slice);
result.push(buffer);
}

View File

@@ -1,7 +1,7 @@
export type TransportProtocolDecode = (bytes: ArrayBuffer) => {
length: number;
typeId: number;
buffer: ByteBuffer;
buffer: Buffer;
};
export interface TransportProtocolEncodeOptions {
@@ -10,9 +10,9 @@ export interface TransportProtocolEncodeOptions {
}
export type TransportProtocolEncode = (
data: ByteBuffer,
data: Buffer,
options: TransportProtocolEncodeOptions,
) => ByteBuffer[];
) => Buffer[];
export interface TransportProtocol {
encode: TransportProtocolEncode;

View File

@@ -1,21 +1,19 @@
import ByteBuffer from 'bytebuffer';
import { bridge } from '../src/index';
describe('protocol-bridge', () => {
it('encode', () => {
let chunks;
// encode small chunk, message without data
chunks = bridge.encode(new ByteBuffer(0), { messageType: 55 });
chunks = bridge.encode(Buffer.alloc(0), { messageType: 55 });
expect(chunks.length).toEqual(1);
expect(chunks[0].limit).toEqual(6);
expect(chunks[0].length).toEqual(6);
// encode big chunk, message with data
chunks = bridge.encode(new ByteBuffer(371), { messageType: 55 });
chunks = bridge.encode(Buffer.alloc(371), { messageType: 55 });
expect(chunks.length).toEqual(1);
expect(chunks[0].slice(0, 6).toString('hex')).toEqual('003700000173');
expect(chunks[0].readUint32(2)).toEqual(371);
expect(chunks[0].buffer.length).toEqual(371 + 6);
expect(chunks[0].subarray(0, 6).toString('hex')).toEqual('003700000173');
expect(chunks[0].readUint32BE(2)).toEqual(371);
expect(chunks[0].length).toEqual(371 + 6);
});
it('decode', () => {

View File

@@ -13,7 +13,7 @@ describe('protocol-v1', () => {
expect(read.magic).toEqual('trzd1');
expect(read.definitionType).toEqual(0);
expect(read.protobufLength).toEqual(19);
expect(read.protobufPayload.toBuffer().toString('hex')).toEqual(
expect(read.protobufPayload.toString('hex')).toEqual(
'08011203455448183c2208457468657265756d',
);
@@ -26,7 +26,7 @@ describe('protocol-v1', () => {
expect(read.magic).toEqual('trzd1');
expect(read.definitionType).toEqual(1);
expect(read.protobufLength).toEqual(50);
expect(read.protobufPayload.toBuffer().toString('hex')).toEqual(
expect(read.protobufPayload.toString('hex')).toEqual(
'0a147af963cf6d228e564e2a0aa0ddbf06210b38615d10051a0354535420122a11676f65726c69205465737420746f6b656e',
);
@@ -38,9 +38,8 @@ describe('protocol-v1', () => {
`74727a643100585a6865${len.toString('hex')}${longName.toString('hex')}DEAD`,
'hex',
);
console.warn(len.toString('hex'));
read = trzd.decode(data);
expect(read.protobufLength).toEqual(400);
expect(read.protobufPayload.toBuffer()).toEqual(longName);
expect(read.protobufPayload).toEqual(longName);
});
});

View File

@@ -1,27 +1,25 @@
import ByteBuffer from 'bytebuffer';
import { v1 } from '../src/index';
describe('protocol-v1', () => {
it('encode', () => {
let chunks;
// encode only one chunk, message without data
chunks = v1.encode(new ByteBuffer(0), { messageType: 55 });
chunks = v1.encode(Buffer.alloc(0), { messageType: 55 });
expect(chunks.length).toEqual(1);
expect(chunks[0].limit).toEqual(64);
expect(chunks[0].length).toEqual(64);
// encode multiple chunks, message with data
chunks = v1.encode(new ByteBuffer(371), { messageType: 55 });
chunks = v1.encode(Buffer.alloc(371), { messageType: 55 });
expect(chunks.length).toEqual(7);
chunks.forEach((chunk, index) => {
expect(chunk.limit).toEqual(64);
expect(chunk.length).toEqual(64);
if (index === 0) {
// first chunk with additional data
expect(chunk.slice(0, 9).toString('hex')).toEqual('3f2323003700000173');
expect(chunk.readUint32(5)).toEqual(371);
expect(chunk.subarray(0, 9).toString('hex')).toEqual('3f2323003700000173');
expect(chunk.readUint32BE(5)).toEqual(371);
} else {
// following chunk starts with MESSAGE_MAGIC_HEADER_BYTE
expect(chunk.slice(0, 5).toString('hex')).toEqual('3f00000000');
expect(chunk.subarray(0, 5).toString('hex')).toEqual('3f00000000');
}
});
});

View File

@@ -57,7 +57,6 @@
"@trezor/protobuf": "workspace:*",
"@trezor/protocol": "workspace:*",
"@trezor/utils": "workspace:*",
"bytebuffer": "^5.0.1",
"cross-fetch": "^4.0.0",
"json-stable-stringify": "^1.0.2",
"long": "^4.0.0",

View File

@@ -192,7 +192,7 @@ export abstract class AbstractApiTransport extends AbstractTransport {
const { encode, decode } = protocol || v1Protocol;
const buffers = buildBuffers(this.messages, name, data, encode);
for (const chunk of buffers) {
this.api.write(path, chunk.buffer).then(result => {
this.api.write(path, chunk).then(result => {
if (!result.success) {
throw new Error(result.error);
}
@@ -244,7 +244,7 @@ export abstract class AbstractApiTransport extends AbstractTransport {
const { encode } = protocol || v1Protocol;
const buffers = buildBuffers(this.messages, name, data, encode);
for (const chunk of buffers) {
this.api.write(path, chunk.buffer).then(result => {
this.api.write(path, chunk).then(result => {
if (!result.success) {
throw new Error(result.error);
}

View File

@@ -1,15 +1,15 @@
import ByteBuffer from 'bytebuffer';
import { Root } from 'protobufjs/light';
import { decode as decodeProtobuf, createMessageFromType } from '@trezor/protobuf';
import { TransportProtocolDecode } from '@trezor/protocol';
async function receiveRest(
parsedInput: ByteBuffer,
result: Buffer,
receiver: () => Promise<ArrayBuffer>,
offset: number,
expectedLength: number,
): Promise<void> {
if (parsedInput.offset >= expectedLength) {
if (offset >= expectedLength) {
return;
}
const data = await receiver();
@@ -17,9 +17,10 @@ async function receiveRest(
if (data == null) {
throw new Error('Received no data.');
}
parsedInput.append(data.slice(1));
const length = offset + data.byteLength - 1;
Buffer.from(data).copy(result, offset, 1, length);
return receiveRest(parsedInput, receiver, expectedLength);
return receiveRest(result, receiver, length, expectedLength);
}
async function receiveBuffer(
@@ -28,13 +29,14 @@ async function receiveBuffer(
) {
const data = await receiver();
const { length, typeId, buffer } = decoder(data);
const decoded = new ByteBuffer(length);
const result = Buffer.alloc(length);
if (length) {
decoded.append(buffer);
buffer.copy(result);
}
await receiveRest(decoded, receiver, length);
return { received: decoded, typeId };
await receiveRest(result, receiver, buffer.length, length);
return { received: result, typeId };
}
export async function receiveAndParse(
@@ -44,7 +46,6 @@ export async function receiveAndParse(
) {
const { received, typeId } = await receiveBuffer(receiver, decoder);
const { Message, messageName } = createMessageFromType(messages, typeId);
received.reset();
const message = decodeProtobuf(Message, received);
return {
message,

View File

@@ -104,13 +104,13 @@ describe('encoding json -> protobuf -> json', () => {
const { length } = Buffer.from(f.in.source_account);
// chunk length cannot be less than message header/constant (28) + variable source_account length
// additional bytes are expected (encoded Uint32) if message length is greater
expect(chunk.buffer.length).toBeGreaterThanOrEqual(28 + length);
expect(chunk.length).toBeGreaterThanOrEqual(28 + length);
let i = -1;
const decoded = await receiveAndParse(
parsedMessages,
() => {
i++;
return Promise.resolve(result[i].buffer);
return Promise.resolve(result[i]);
},
bridgeProtocol.decode,
);
@@ -123,14 +123,14 @@ describe('encoding json -> protobuf -> json', () => {
const result = buildBuffers(parsedMessages, f.name, f.in, v1Protocol.encode);
// each protocol chunks are equal 64 bytes
result.forEach(chunk => {
expect(chunk.buffer.length).toEqual(64);
expect(chunk.length).toEqual(64);
});
let i = -1;
const decoded = await receiveAndParse(
parsedMessages,
() => {
i++;
return Promise.resolve(result[i].buffer);
return Promise.resolve(result[i]);
},
v1Protocol.decode,
);

View File

@@ -8833,7 +8833,6 @@ __metadata:
version: 0.0.0-use.local
resolution: "@trezor/protobuf@workspace:packages/protobuf"
dependencies:
bytebuffer: "npm:^5.0.1"
jest: "npm:29.5.0"
long: "npm:^4.0.0"
protobufjs: "npm:7.2.5"
@@ -8850,7 +8849,6 @@ __metadata:
version: 0.0.0-use.local
resolution: "@trezor/protocol@workspace:packages/protocol"
dependencies:
bytebuffer: "npm:^5.0.1"
jest: "npm:29.5.0"
rimraf: "npm:^5.0.5"
tsx: "npm:^4.6.2"
@@ -9451,7 +9449,6 @@ __metadata:
"@types/bytebuffer": "npm:^5.0.46"
"@types/sharedworker": "npm:^0.0.105"
"@types/w3c-web-usb": "npm:^1.0.9"
bytebuffer: "npm:^5.0.1"
cross-fetch: "npm:^4.0.0"
jest: "npm:29.5.0"
json-stable-stringify: "npm:^1.0.2"
@@ -13702,15 +13699,6 @@ __metadata:
languageName: node
linkType: hard
"bytebuffer@npm:^5.0.1":
version: 5.0.1
resolution: "bytebuffer@npm:5.0.1"
dependencies:
long: "npm:~3"
checksum: f3e9739ed9ab30e19d985fc3dadfdbd631d030874bbb313feefddac756f21ac10957257737e630fd9959744318e6e8b7d8c35b797519693bf1897be16c560970
languageName: node
linkType: hard
"bytes@npm:3.0.0":
version: 3.0.0
resolution: "bytes@npm:3.0.0"
@@ -23842,13 +23830,6 @@ __metadata:
languageName: node
linkType: hard
"long@npm:~3":
version: 3.2.0
resolution: "long@npm:3.2.0"
checksum: ffc685ec458ddf71a830d6deb62ff7dc551a736d47473350d9e077c22db96ec88c8a3554c11ffce7d7f2291b0c30da36629e4d0a97c29b5360dc977533c96d28
languageName: node
linkType: hard
"loose-envify@npm:^1.0.0, loose-envify@npm:^1.1.0, loose-envify@npm:^1.2.0, loose-envify@npm:^1.3.1, loose-envify@npm:^1.4.0":
version: 1.4.0
resolution: "loose-envify@npm:1.4.0"