feat(schema-utils): typebox with custom types and codegen

packages/schema-utils/jest.config.js

@@ -0,0 +1,18 @@
+module.exports = {
+    globals: {
+        'ts-jest': {
+            tsconfig: 'tsconfig.json',
+        },
+    },
+    testEnvironment: 'node',
+    testMatch: ['**/*.test.ts'],
+    coverageDirectory: './coverage/',
+    collectCoverage: true,
+    collectCoverageFrom: ['**/src/**/*.ts'],
+    modulePathIgnorePatterns: ['node_modules', '<rootDir>/lib', '<rootDir>/libDev'],
+    watchPathIgnorePatterns: ['<rootDir>/libDev', '<rootDir>/lib'],
+    testPathIgnorePatterns: ['<rootDir>/libDev/', '<rootDir>/lib/'],
+    transform: {
+        '^.+\\.ts$': 'ts-jest',
+    },
+};

packages/schema-utils/package.json

@@ -0,0 +1,26 @@
+{
+    "name": "@trezor/schema-utils",
+    "version": "1.0.0",
+    "license": "See LICENSE.md in repo root",
+    "sideEffects": false,
+    "main": "src/index",
+    "files": [
+        "src/"
+    ],
+    "scripts": {
+        "test:unit": "jest",
+        "lint:js": "yarn g:eslint '**/*.{ts,tsx,js}'",
+        "type-check": "tsc --build",
+        "codegen": "ts-node --skip-project ./src/codegen.ts"
+    },
+    "devDependencies": {
+        "jest": "^26.6.3",
+        "prettier": "^3.1.0",
+        "typescript": "5.3.2"
+    },
+    "dependencies": {
+        "@sinclair/typebox": "^0.31.28",
+        "@sinclair/typebox-codegen": "^0.8.13",
+        "ts-mixer": "^6.0.3"
+    }
+}

packages/schema-utils/src/codegen.ts

@@ -0,0 +1,72 @@
+import * as Codegen from '@sinclair/typebox-codegen/typescript';
+import fs from 'fs';
+
+export function generate(code: string) {
+    // Make some replacements to make the code processable by the generator
+    // Since there are some issues with typeof
+    code = code.replace(/typeof undefined/g, 'undefined');
+    code = code.replace(/keyof typeof/g, 'keyof');
+    // Ignore types added at end of message.ts, these are too complex for the generator
+    code = code.substring(0, code.indexOf('export type MessageKey = keyof MessageType'));
+    // Make generator aware of custom types
+    const customTypesMapping = {
+        ArrayBuffer: 'Type.ArrayBuffer()',
+        Buffer: 'Type.Buffer()',
+        UintType: 'Type.Uint()',
+    };
+    const customTypePlaceholder = Object.keys(customTypesMapping).map(t => `type ${t} = any;`);
+    // Run generator
+    let output = Codegen.TypeScriptToTypeBox.Generate(customTypePlaceholder + code, {
+        useTypeBoxImport: false,
+    });
+    // Remove placeholder declarations of custom types
+    const lastKey = Object.keys(customTypesMapping)[Object.keys(customTypesMapping).length - 1];
+    const index = output.lastIndexOf(`const ${lastKey} = `);
+    const blankLineIndex = output.indexOf('\n\n', index);
+    output = output.substring(blankLineIndex + 1);
+    // Replace placeholder types with custom types mapping
+    Object.entries(customTypesMapping).forEach(([key, value]) => {
+        output = output.replace(new RegExp(`\\b${key}\\b`, 'g'), value);
+    });
+    // Find enum occurences
+    const enums = [...output.matchAll(/enum Enum(\w+) {/g)].map(m => m[1]);
+    // Replace possible keyof for each enum
+    enums.forEach(e => {
+        // Replace all occurences of version without Enum prefix with version with Enum prefix
+        output = output.replace(new RegExp(`\\b${e}\\b`, 'g'), `Enum${e}`);
+        output = output.replace(
+            new RegExp(`type Enum${e} = Static\\<typeof Enum${e}\\>`, 'g'),
+            `type Enum${e} = Static<typeof Enum${e}>`,
+        );
+        output = output.replace(
+            new RegExp(`const Enum${e} = Type\\.Enum\\(Enum${e}\\)`, 'g'),
+            `const Enum${e} = Type.Enum(${e})`,
+        );
+        output = output.replace(new RegExp(`enum Enum${e} \\{`, 'g'), `enum ${e} {`);
+        output = output.replace(
+            new RegExp(`Type\\.KeyOf\\(Enum${e}\\)`, 'g'),
+            `Type.KeyOfEnum(${e})`,
+        );
+    });
+    // Add import of lib
+    output = `import { Type, Static } from '@trezor/schema-utils';\n\n${output}`;
+    // Add eslint ignore for camelcase, since some type names use underscores
+    output = `/* eslint-disable camelcase */\n${output}`;
+    return output;
+}
+
+export function generateForFile(fileName: string) {
+    const code = fs.readFileSync(fileName, 'utf-8');
+    return generate(code);
+}
+
+// If ran directly, output code for file passed as argument
+/* istanbul ignore next */
+if (require.main === module) {
+    const fileName = process.argv[2];
+    if (!fileName || !fs.existsSync(fileName)) {
+        throw new Error('File not found');
+    }
+    const output = generateForFile(fileName);
+    process.stdout.write(output);
+}

packages/schema-utils/src/custom-types/array-buffer.ts

@@ -0,0 +1,14 @@
+import { TypeRegistry, Kind, TSchema, JavaScriptTypeBuilder } from '@sinclair/typebox';
+
+export interface TArrayBuffer extends TSchema {
+    [Kind]: 'ArrayBuffer';
+    static: ArrayBuffer;
+    type: 'ArrayBuffer';
+}
+TypeRegistry.Set('ArrayBuffer', (_: TArrayBuffer, value: unknown) => value instanceof ArrayBuffer);
+
+export class ArrayBufferBuilder extends JavaScriptTypeBuilder {
+    ArrayBuffer(options?: TSchema): TArrayBuffer {
+        return this.Create({ ...options, [Kind]: 'ArrayBuffer', type: 'ArrayBuffer' });
+    }
+}

packages/schema-utils/src/custom-types/buffer.ts

@@ -0,0 +1,14 @@
+import { TypeRegistry, Kind, TSchema, JavaScriptTypeBuilder } from '@sinclair/typebox';
+
+export interface TBuffer extends TSchema {
+    [Kind]: 'Buffer';
+    static: Buffer;
+    type: 'Buffer';
+}
+TypeRegistry.Set('Buffer', (_: TBuffer, value: unknown) => value instanceof Buffer);
+
+export class BufferBuilder extends JavaScriptTypeBuilder {
+    Buffer(options?: TSchema): TBuffer {
+        return this.Create({ ...options, [Kind]: 'Buffer', type: 'Buffer' });
+    }
+}

packages/schema-utils/src/custom-types/index.ts

@@ -0,0 +1,4 @@
+export { ArrayBufferBuilder } from './array-buffer';
+export { BufferBuilder } from './buffer';
+export { KeyofEnumBuilder } from './keyof-enum';
+export { UintBuilder } from './uint';

packages/schema-utils/src/custom-types/keyof-enum.ts

@@ -0,0 +1,41 @@
+import { JavaScriptTypeBuilder, TUnion, Hint, SchemaOptions, TLiteral } from '@sinclair/typebox';
+
+// UnionToIntersection<A | B> = A & B
+type UnionToIntersection<U> = (U extends unknown ? (arg: U) => 0 : never) extends (
+    arg: infer I,
+) => 0
+    ? I
+    : never;
+
+// LastInUnion<A | B> = B
+type LastInUnion<U> = UnionToIntersection<U extends unknown ? (x: U) => 0 : never> extends (
+    x: infer L,
+) => 0
+    ? L
+    : never;
+
+// Build a tuple for the object
+// Strategy - take the last key, add it to the tuple, and recurse on the rest
+// Wrap the key in a TLiteral for Typebox
+type ObjectKeysToTuple<T, Last = LastInUnion<keyof T>> = [T] extends [never]
+    ? []
+    : [Last] extends [never]
+    ? []
+    : Last extends string | number
+    ? [...ObjectKeysToTuple<Omit<T, Last>>, TLiteral<Last>]
+    : [];
+
+export interface TKeyOfEnum<T extends Record<string, string | number>>
+    extends TUnion<ObjectKeysToTuple<T>> {
+    [Hint]: 'KeyOfEnum';
+}
+
+export class KeyofEnumBuilder extends JavaScriptTypeBuilder {
+    KeyOfEnum<T extends Record<string, string | number>>(
+        schema: T,
+        options?: SchemaOptions,
+    ): TKeyOfEnum<T> {
+        const keys = Object.keys(schema).map(key => this.Literal(key));
+        return this.Union(keys, { ...options, [Hint]: 'KeyOfEnum' }) as TKeyOfEnum<T>;
+    }
+}

packages/schema-utils/src/custom-types/uint.ts

@@ -0,0 +1,28 @@
+import { TypeRegistry, Kind, TSchema, JavaScriptTypeBuilder } from '@sinclair/typebox';
+
+export interface TUintOptions {
+    allowNegative?: boolean;
+}
+export interface TUint extends TUintOptions, TSchema {
+    [Kind]: 'Uint';
+    static: string | number;
+    type: 'Uint';
+}
+TypeRegistry.Set('Uint', (schema: TUint, value: unknown) => {
+    if (typeof value !== 'string' && typeof value !== 'number') {
+        return false;
+    }
+    if (
+        (typeof value === 'number' && !Number.isSafeInteger(value)) ||
+        !/^(?:[1-9]\d*|\d)$/.test(value.toString().replace(/^-/, schema.allowNegative ? '' : '-'))
+    ) {
+        return false;
+    }
+    return true;
+});
+
+export class UintBuilder extends JavaScriptTypeBuilder {
+    Uint(options?: TUintOptions): TUint {
+        return this.Create({ ...options, [Kind]: 'Uint', type: 'Uint' });
+    }
+}

packages/schema-utils/src/index.ts

@@ -0,0 +1,20 @@
+import { JavaScriptTypeBuilder, Static, TSchema } from '@sinclair/typebox';
+import { Value } from '@sinclair/typebox/value';
+import { Mixin } from 'ts-mixer';
+
+import { ArrayBufferBuilder, BufferBuilder, KeyofEnumBuilder, UintBuilder } from './custom-types';
+
+class CustomTypeBuilder extends Mixin(
+    JavaScriptTypeBuilder,
+    ArrayBufferBuilder,
+    BufferBuilder,
+    KeyofEnumBuilder,
+    UintBuilder,
+) {}
+
+export function Validate<T extends TSchema>(schema: T, value: unknown): value is Static<T> {
+    return Value.Check(schema, value);
+}
+
+export const Type = new CustomTypeBuilder();
+export type { Static };

packages/schema-utils/tests/codegen.test.ts

@@ -0,0 +1,8 @@
+import { generateForFile } from '../src/codegen';
+
+describe('codegen', () => {
+    it('should generate code for protobuf messages example', () => {
+        const output = generateForFile(`${__dirname}/__snapshots__/codegen.input.ts`);
+        expect(output).toMatchSnapshot();
+    });
+});

packages/schema-utils/tests/complex-example.test.ts

@@ -0,0 +1,79 @@
+import { Type, Validate } from '../src';
+
+describe('complex-example', () => {
+    it('should work with a schema like StellarSignTx', () => {
+        const schema = Type.Object({
+            address_n: Type.Array(Type.Uint()),
+            network_passphrase: Type.String(),
+            source_account: Type.String(),
+            fee: Type.Uint(),
+            sequence_number: Type.Uint(),
+            timebounds_start: Type.Uint(),
+            timebounds_end: Type.Uint(),
+            memo_type: Type.String(),
+            memo_text: Type.String(),
+            memo_id: Type.String(),
+            memo_hash: Type.Buffer(),
+            num_operations: Type.Uint(),
+        });
+        expect(schema.type).toEqual('object');
+
+        const value = {
+            address_n: [0],
+            network_passphrase: 'test',
+            source_account: 'GAA2J2KQV6J4LXQK2K3J2LQ3ZK7Q2K3J2K3J2K3J2K3J2K3J2K3J2K3J2',
+            fee: 100,
+            sequence_number: 4294967296,
+            timebounds_start: 0,
+            timebounds_end: 4294967296,
+            memo_type: 'MEMO_TEXT',
+            memo_text: 'test',
+            memo_id: '123',
+            memo_hash: Buffer.from('test'),
+            num_operations: 1,
+        };
+        expect(Validate(schema, value)).toBe(true);
+
+        const invalidValue = {
+            address_n: 'invalid',
+            network_passphrase: 'test',
+            source_account: 123456789,
+            fee: 100,
+            sequence_number: 4294967296,
+            timebounds_start: 0,
+            timebounds_end: 4294967296,
+            memo_type: 'MEMO_TEXT',
+            memo_text: 'test',
+            memo_id: '123',
+            memo_hash: Buffer.from('test'),
+            num_operations: 1,
+        };
+        expect(Validate(schema, invalidValue)).toBe(false);
+    });
+
+    it('should work with a schema like EthereumSignTypedHash', () => {
+        const schema = Type.Object({
+            address_n: Type.Array(Type.Uint()),
+            domain_separator_hash: Type.String(),
+            message_hash: Type.String(),
+            encoded_network: Type.ArrayBuffer(),
+        });
+        expect(schema.type).toEqual('object');
+
+        const value = {
+            address_n: [0],
+            domain_separator_hash: 'test',
+            message_hash: 'test',
+            encoded_network: new ArrayBuffer(10),
+        };
+        expect(Validate(schema, value)).toBe(true);
+
+        const invalidValue = {
+            address_n: 'invalid',
+            domain_separator_hash: 'test',
+            message_hash: 'test',
+            encoded_network: 'invalid',
+        };
+        expect(Validate(schema, invalidValue)).toBe(false);
+    });
+});

packages/schema-utils/tests/custom-types.test.ts

@@ -0,0 +1,50 @@
+import { Type, Validate } from '../src';
+
+describe('custom-types', () => {
+    it('should work with ArrayBuffer', () => {
+        const schema = Type.ArrayBuffer();
+        expect(schema.type).toEqual('ArrayBuffer');
+
+        const value = new ArrayBuffer(10);
+        expect(Validate(schema, value)).toBe(true);
+
+        const invalidValue = 'invalid';
+        expect(Validate(schema, invalidValue)).toBe(false);
+    });
+    it('should work with Buffer', () => {
+        const schema = Type.Buffer();
+        expect(schema.type).toEqual('Buffer');
+
+        const value = Buffer.from('test');
+        expect(Validate(schema, value)).toBe(true);
+
+        const invalidValue = 'invalid';
+        expect(Validate(schema, invalidValue)).toBe(false);
+    });
+    it('should work with Uint', () => {
+        const schema = Type.Uint();
+        expect(schema.type).toEqual('Uint');
+
+        const value = 10;
+        expect(Validate(schema, value)).toBe(true);
+
+        const valueAsString = '10';
+        expect(Validate(schema, valueAsString)).toBe(true);
+
+        const invalidInteger = 3.14;
+        expect(Validate(schema, invalidInteger)).toBe(false);
+
+        const invalidString = 'xxxx';
+        expect(Validate(schema, invalidString)).toBe(false);
+
+        const invalidValue = [123];
+        expect(Validate(schema, invalidValue)).toBe(false);
+    });
+    it('should work with Uint with allowNegative', () => {
+        const schema = Type.Uint({ allowNegative: true });
+        expect(schema.type).toEqual('Uint');
+
+        const valueAsString = '-10';
+        expect(Validate(schema, valueAsString)).toBe(true);
+    });
+});

packages/schema-utils/tests/guard.test.ts

@@ -0,0 +1,20 @@
+import { Type, Validate } from '../src';
+
+describe('type-guard', () => {
+    it('guard works when parsing unknown value', () => {
+        const schema = Type.Object({
+            foo: Type.String(),
+        });
+
+        const unknown = JSON.parse('{"foo": "bar"}') as unknown;
+
+        // @ts-expect-error
+        expect(unknown.foo).toBe('bar');
+
+        expect(Validate(schema, unknown)).toBe(true);
+        if (Validate(schema, unknown)) {
+            // @ts-expect-no-error
+            expect(unknown.foo).toBe('bar');
+        }
+    });
+});

packages/schema-utils/tests/keyof-enum.test.ts

@@ -0,0 +1,40 @@
+import { Static, Type, Validate } from '../src';
+
+describe('enum', () => {
+    it('should work with keyof enum', () => {
+        enum E {
+            A = 'a',
+            B = 'b',
+        }
+        const schema = Type.KeyOfEnum(E);
+        type T = Static<typeof schema>;
+
+        const x: T = 'A';
+        expect(x).toEqual('A');
+
+        // @ts-expect-error
+        const y: T = 'C';
+        expect(y).toEqual('C');
+
+        expect(Validate(schema, 'A')).toBe(true);
+        expect(Validate(schema, 'B')).toBe(true);
+        expect(Validate(schema, 'C')).toBe(false);
+    });
+
+    it('should work with keyof enum with exclude', () => {
+        enum E {
+            A = 'a',
+            B = 'b',
+        }
+        const schema = Type.KeyOfEnum(E);
+        type T1 = Static<typeof schema>;
+        const b: T1 = 'B';
+        expect(b).toEqual('B');
+
+        const excluded = Type.Exclude(schema, Type.Literal('B'));
+        type T2 = Static<typeof excluded>;
+        // @ts-expect-error
+        const x: T2 = 'B';
+        expect(x).toEqual('B');
+    });
+});

packages/schema-utils/tsconfig.json

@@ -0,0 +1,5 @@
+{
+    "extends": "../../tsconfig.json",
+    "compilerOptions": { "outDir": "libDev" },
+    "references": []
+}

yarn.lock

@@ -5691,6 +5691,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@sinclair/typebox-codegen@npm:^0.8.13":
+  version: 0.8.13
+  resolution: "@sinclair/typebox-codegen@npm:0.8.13"
+  dependencies:
+    "@sinclair/typebox": "npm:^0.31.28"
+    prettier: "npm:^2.8.7"
+    typescript: "npm:^5.1.6"
+  checksum: db9467313529b98373327c29d2a138e5e5f5834dafaa5ccc42be8a2aebe2c3e02ab568288b7c6147ff732417a43d832c5171d0bec9bb3371674f0cd0b048ba64
+  languageName: node
+  linkType: hard
+
 "@sinclair/typebox@npm:^0.27.8":
   version: 0.27.8
   resolution: "@sinclair/typebox@npm:0.27.8"
@@ -5698,6 +5709,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@sinclair/typebox@npm:^0.31.28":
+  version: 0.31.28
+  resolution: "@sinclair/typebox@npm:0.31.28"
+  checksum: 27c3af5539a12af9b3cda4432959c69fb500920f1dd3739700a1437cfa9de809a292398a0b3b871c7471e96e4088d58406105bed5407d089c91c56090c526013
+  languageName: node
+  linkType: hard
+
 "@sindresorhus/is@npm:^4.0.0":
   version: 4.6.0
   resolution: "@sindresorhus/is@npm:4.6.0"
@@ -8337,6 +8355,7 @@ __metadata:
     "@trezor/blockchain-link-types": "workspace:*"
     "@trezor/blockchain-link-utils": "workspace:*"
     "@trezor/e2e-utils": "workspace:*"
+    "@trezor/schema-utils": "workspace:*"
     "@trezor/type-utils": "workspace:*"
     "@trezor/utils": "workspace:*"
     "@trezor/utxo-lib": "workspace:*"
@@ -8889,6 +8908,19 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@trezor/schema-utils@workspace:*, @trezor/schema-utils@workspace:packages/schema-utils":
+  version: 0.0.0-use.local
+  resolution: "@trezor/schema-utils@workspace:packages/schema-utils"
+  dependencies:
+    "@sinclair/typebox": "npm:^0.31.28"
+    "@sinclair/typebox-codegen": "npm:^0.8.13"
+    jest: "npm:^26.6.3"
+    prettier: "npm:^3.1.0"
+    ts-mixer: "npm:^6.0.3"
+    typescript: "npm:5.3.2"
+  languageName: unknown
+  linkType: soft
+
 "@trezor/scripts@workspace:scripts":
   version: 0.0.0-use.local
   resolution: "@trezor/scripts@workspace:scripts"
@@ -32715,6 +32747,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"ts-mixer@npm:^6.0.3":
+  version: 6.0.3
+  resolution: "ts-mixer@npm:6.0.3"
+  checksum: ac9178bdac5e5f760472269ad4c461587a0f6793532ddbef1326bb01482425a6247be98f9bd11bf35a9fdd36b63b8c8dde393942b9b9ee52d154eef082fca39a
+  languageName: node
+  linkType: hard
+
 "ts-node@npm:^10.9.1":
   version: 10.9.1
   resolution: "ts-node@npm:10.9.1"