refactor!: migrate to @grpc/grpc-js

This migrates the `grpc` package to the pure javascript `grpc-js` implementation. `grpc-js` is the more actively maintained package and the preferred implementation by Google projects, and it reduces the dependency on and overhead from compilers like `node-pre-gyp`. `grpc` is now deprecated in npm. This also updates `grpc-tools` and tweaks the way grpc code is generated from `.proto` files to be compatible with `grpc-js`. Lastly this removes the grpc web proxy from xud. It is not compatible with `grpc-js` and it is currently not used for any purpose including testing or development. As the old approach relied on a custom fork of an old, unmaintained package to create the proxy, we plan to reimplement a web proxy with a new approach using an add-on or module that runs the proxy in a separate process. BREAKING CHANGE: Removes grpc web proxy from xud. Co-authored-by: LePremierHomme <[email protected]>
ExchangeUnion · Jan 4, 2021 · bcbb5d7 · bcbb5d7
1 parent e844594
commit bcbb5d7
Show file tree

Hide file tree

Showing 52 changed files with 19,938 additions and 13,868 deletions.
diff --git a/bin/xud b/bin/xud
@@ -188,16 +188,6 @@ const { argv } = require('yargs')
       type: 'number',
       alias: 'r',
     },
-    'webproxy.disable': {
-      describe: 'Disable web proxy server',
-      type: 'boolean',
-      default: undefined,
-    },
-    'webproxy.port': {
-      describe: 'Port for web proxy server',
-      type: 'number',
-      alias: 'w',
-    },
   })
   .coerce('lnd', (arg) => {
     delete arg['[currency]'];

diff --git a/lib/Config.ts b/lib/Config.ts
@@ -58,7 +58,6 @@ class Config {
   public lnd: { [currency: string]: LndClientConfig | undefined } = {};
   public connext: ConnextClientConfig;
   public orderthresholds: OrderBookThresholds;
-  public webproxy: { port: number; disable: boolean };
   public instanceid = 0;
   /** Whether to intialize a new database with default values. */
   public initdb = true;
@@ -133,10 +132,6 @@ class Config {
       host: 'localhost',
       port: this.getDefaultHttpPort(),
     };
-    this.webproxy = {
-      disable: true,
-      port: 8080,
-    };
     // TODO: add dynamic max/min price limits
     this.orderthresholds = { minQuantity: 0 }; // 0 = disabled
     this.lnd.BTC = {

diff --git a/lib/Xud.ts b/lib/Xud.ts
@@ -8,7 +8,6 @@ import Config from './Config';
 import { SwapClientType, XuNetwork } from './constants/enums';
 import DB from './db/DB';
 import GrpcServer from './grpc/GrpcServer';
-import GrpcWebProxyServer from './grpc/webproxy/GrpcWebProxyServer';
 import HttpServer from './http/HttpServer';
 import Logger from './Logger';
 import NodeKey from './nodekey/NodeKey';
@@ -40,7 +39,6 @@ class Xud extends EventEmitter {
   private orderBook!: OrderBook;
   private rpcServer?: GrpcServer;
   private httpServer?: HttpServer;
-  private grpcAPIProxy?: GrpcWebProxyServer;
   private swaps!: Swaps;
   private shuttingDown = false;
   private swapClientManager?: SwapClientManager;
@@ -102,16 +100,6 @@ class Xud extends EventEmitter {
           path.join(this.config.xudir, 'tls.cert'),
           path.join(this.config.xudir, 'tls.key'),
         );
-
-        if (!this.config.webproxy.disable) {
-          this.grpcAPIProxy = new GrpcWebProxyServer(loggers.rpc);
-          await this.grpcAPIProxy.listen(
-            this.config.webproxy.port,
-            this.config.rpc.port,
-            this.config.rpc.host,
-            path.join(this.config.xudir, 'tls.cert'),
-          );
-        }
       }
 
       this.db = new DB(loggers.db, this.config.dbpath);
@@ -298,9 +286,6 @@ class Xud extends EventEmitter {
     if (this.rpcServer) {
       closePromises.push(this.rpcServer.close());
     }
-    if (this.grpcAPIProxy) {
-      closePromises.push(this.grpcAPIProxy.close());
-    }
     if (this.swaps) {
       this.swaps.close();
     }

diff --git a/lib/cli/command.ts b/lib/cli/command.ts
@@ -1,9 +1,14 @@
+import * as grpc from '@grpc/grpc-js';
 import fs from 'fs';
-import grpc, { status } from 'grpc';
 import path from 'path';
 import { Arguments } from 'yargs';
 import Config from '../Config';
-import { XudClient, XudInitClient } from '../proto/xudrpc_grpc_pb';
+import * as xudGrpc from '../proto/xudrpc_grpc_pb';
+
+// @ts-ignore
+const XudClient = grpc.makeClientConstructor(xudGrpc['xudrpc.Xud'], 'XudService');
+// @ts-ignore
+const XudInitClient = grpc.makeClientConstructor(xudGrpc['xudrpc.XudInit'], 'XudInitService');
 
 /**
  * Attempts to load the xud configuration file to dynamically determine the
@@ -55,24 +60,24 @@ const getTlsCert = (certPath: string) => {
  * A generic function to instantiate an XU client.
  * @param argv the command line arguments
  */
-export const loadXudClient = async (argv: Arguments<any>) => {
+export const loadXudClient = async (argv: Arguments<any>): Promise<xudGrpc.XudClient> => {
   await loadXudConfig(argv);
 
   const certPath = argv.tlscertpath || path.join(argv.xudir, 'tls.cert');
   const cert = getTlsCert(certPath);
   const credentials = grpc.credentials.createSsl(cert);
 
-  return new XudClient(`${argv.rpchost}:${argv.rpcport}`, credentials);
+  return new XudClient(`${argv.rpchost}:${argv.rpcport}`, credentials) as any;
 };
 
-export const loadXudInitClient = async (argv: Arguments<any>) => {
+export const loadXudInitClient = async (argv: Arguments<any>): Promise<xudGrpc.XudInitClient> => {
   await loadXudConfig(argv);
 
   const certPath = argv.tlscertpath || path.join(argv.xudir, 'tls.cert');
   const cert = getTlsCert(certPath);
-  const credentials = grpc.credentials.createSsl(cert);
+  const grpcCredentials = grpc.credentials.createSsl(cert);
 
-  return new XudInitClient(`${argv.rpchost}:${argv.rpcport}`, credentials);
+  return new XudInitClient(`${argv.rpchost}:${argv.rpcport}`, grpcCredentials) as any;
 };
 
 interface GrpcResponse {
@@ -83,23 +88,23 @@ export const callback = (argv: Arguments, formatOutput?: Function, displayJson?:
   return (error: grpc.ServiceError | null, response: GrpcResponse) => {
     if (error) {
       process.exitCode = 1;
-      if (error.code === status.UNAVAILABLE && error.message.includes('xud is starting')) {
+      if (error.code === grpc.status.UNAVAILABLE && error.message.includes('xud is starting')) {
         console.error('xud is starting... try again in a few seconds');
       } else if (error.details === 'failed to connect to all addresses') {
         console.error(`could not connect to xud at ${argv.rpchost}:${argv.rpcport}, is xud running?`);
-      } else if (error.code === status.UNIMPLEMENTED && error.message.includes('xud is locked')) {
+      } else if (error.code === grpc.status.UNIMPLEMENTED && error.message.includes('xud is locked')) {
         console.error("xud is locked, run 'xucli unlock', 'xucli create', or 'xucli restore' then try again");
       } else if (
-        error.code === status.UNIMPLEMENTED &&
+        error.code === grpc.status.UNIMPLEMENTED &&
         error.message.includes('xud node cannot be created because it already exists')
       ) {
         console.error("an xud node already exists, try unlocking it with 'xucli unlock'");
       } else if (
-        error.code === status.UNIMPLEMENTED &&
+        error.code === grpc.status.UNIMPLEMENTED &&
         error.message.includes('xud node cannot be unlocked because it does not exist')
       ) {
         console.error("no xud node exists to unlock, try creating one with 'xucli create' or 'xucli restore'");
-      } else if (error.code === status.UNIMPLEMENTED && error.message.includes('xud init service is disabled')) {
+      } else if (error.code === grpc.status.UNIMPLEMENTED && error.message.includes('xud init service is disabled')) {
         console.error("xud is running and unlocked, try checking its status with 'xucli getinfo'");
       } else {
         console.error(`${error.name}: ${error.message}`);

diff --git a/lib/cli/commands/streamorders.ts b/lib/cli/commands/streamorders.ts
@@ -1,6 +1,6 @@
-import { ServiceError, status } from 'grpc';
+import { ServiceError, status } from '@grpc/grpc-js';
 import { Arguments, Argv } from 'yargs';
-import { XudClient } from '../../proto/xudrpc_grpc_pb';
+import { XudClient } from 'lib/proto/xudrpc_grpc_pb';
 import * as xudrpc from '../../proto/xudrpc_pb';
 import { setTimeoutPromise } from '../../utils/utils';
 import { loadXudClient } from '../command';
@@ -26,7 +26,7 @@ const ensureConnection = async (argv: Arguments, printError?: boolean) => {
   if (!client) {
     client = await loadXudClient(argv);
   }
-  client.waitForReady(Date.now() + 3000, (error: Error | null) => {
+  client.waitForReady(Date.now() + 3000, (error?: Error) => {
     if (error) {
       if (error.message === 'Failed to connect before the deadline') {
         console.error(`could not connect to xud at ${argv.rpchost}:${argv.rpcport}, is xud running?`);

diff --git a/lib/cli/placeorder.ts b/lib/cli/placeorder.ts
@@ -1,4 +1,5 @@
 import { Arguments, Argv } from 'yargs';
+import { XudClient } from 'lib/proto/xudrpc_grpc_pb';
 import {
   Order,
   OrderSide,
@@ -87,7 +88,7 @@ export const placeOrderHandler = async (argv: Arguments<any>, side: OrderSide) =
     request.setReplaceOrderId(argv.replace_order_id);
   }
 
-  const client = await loadXudClient(argv);
+  const client = ((await loadXudClient(argv)) as unknown) as XudClient;
   if (argv.sync) {
     client.placeOrderSync(request, callback(argv, formatPlaceOrderOutput));
   } else {

diff --git a/lib/grpc/GrpcInitService.ts b/lib/grpc/GrpcInitService.ts
@@ -1,9 +1,11 @@
-import grpc, { status } from 'grpc';
-import InitService from '../service/InitService';
+/* tslint:disable no-null-keyword */
+import * as grpc from '@grpc/grpc-js';
 import * as xudrpc from '../proto/xudrpc_pb';
+import InitService from '../service/InitService';
 import getGrpcError from './getGrpcError';
 
-class GrpcInitService {
+class GrpcInitService implements grpc.UntypedServiceImplementation {
+  [name: string]: any;
   private disabled = false;
   private initService?: InitService;
 
@@ -28,8 +30,8 @@ class GrpcInitService {
   ): initService is InitService => {
     if (!initService) {
       const err = this.disabled
-        ? { code: status.UNIMPLEMENTED, message: 'xud init service is disabled', name: 'DisabledError' }
-        : { code: status.UNAVAILABLE, message: 'xud is starting', name: 'NotReadyError' };
+        ? { code: grpc.status.UNIMPLEMENTED, message: 'xud init service is disabled', name: 'DisabledError' }
+        : { code: grpc.status.UNAVAILABLE, message: 'xud is starting', name: 'NotReadyError' };
       callback(err, null);
       return false;
     }

diff --git a/lib/grpc/GrpcServer.ts b/lib/grpc/GrpcServer.ts
@@ -1,27 +1,29 @@
+import { Server, ServerCredentials } from '@grpc/grpc-js';
 import assert from 'assert';
 import { promises as fs } from 'fs';
-import grpc from 'grpc';
 import { md, pki } from 'node-forge';
 import { hostname } from 'os';
 import Logger from '../Logger';
-import { XudInitService, XudService } from '../proto/xudrpc_grpc_pb';
+import * as xudGrpc from '../proto/xudrpc_grpc_pb';
 import errors from './errors';
 import GrpcInitService from './GrpcInitService';
 import GrpcService from './GrpcService';
-import serverProxy from './serverProxy';
+import { ServerProxy, serverProxy } from './serverProxy';
 
 class GrpcServer {
   public grpcService = new GrpcService();
   public grpcInitService = new GrpcInitService();
-  private server: any;
+  private server: ServerProxy;
 
   constructor(private logger: Logger) {
-    this.server = serverProxy(new grpc.Server());
+    this.server = serverProxy(new Server());
 
     this.grpcInitService = new GrpcInitService();
     this.grpcService = new GrpcService();
-    this.server.addService(XudInitService, this.grpcInitService);
-    this.server.addService(XudService, this.grpcService);
+    // @ts-ignore
+    this.server.addService(xudGrpc['xudrpc.XudInit'], this.grpcInitService);
+    // @ts-ignore
+    this.server.addService(xudGrpc['xudrpc.Xud'], this.grpcService);
 
     this.server.use(async (ctx: any, next: any) => {
       logger.trace(`received call ${ctx.service.path}`);
@@ -66,7 +68,8 @@ class GrpcServer {
       privateKey = Buffer.from(tlsKey);
     }
 
-    const credentials = grpc.ServerCredentials.createSsl(
+    // tslint:disable-next-line:no-null-keyword
+    const credentials = ServerCredentials.createSsl(
       null,
       [
         {
@@ -77,15 +80,17 @@ class GrpcServer {
       false,
     );
 
-    const bindCode = this.server.bind(`${host}:${port}`, credentials);
-    if (bindCode !== port) {
-      const error = errors.COULD_NOT_BIND(port.toString());
-      this.logger.error(error.message);
-      throw error;
-    }
-
-    this.server.start();
-    this.logger.info(`gRPC server listening on ${host}:${port}`);
+    return new Promise<void>((resolve, reject) => {
+      this.server.bindAsync(`${host}:${port}`, credentials, (err) => {
+        if (err) {
+          this.logger.error(err.message);
+          reject(errors.COULD_NOT_BIND(port.toString()));
+        }
+        this.server.start();
+        this.logger.info(`gRPC server listening on ${host}:${port}`);
+        resolve();
+      });
+    });
   };
 
   /**
@@ -111,11 +116,11 @@ class GrpcServer {
     tlsCertPath: string,
     tlsKeyPath: string,
   ): Promise<{ tlsCert: string; tlsKey: string }> => {
-    const keys = pki.rsa.generateKeyPair(1024);
+    const keys = pki.rsa.generateKeyPair(2048);
     const cert = pki.createCertificate();
 
     cert.publicKey = keys.publicKey;
-    cert.serialNumber = String(Math.floor(Math.random() * 1024) + 1);
+    cert.serialNumber = String(Math.floor(Math.random() * 2048) + 1);
 
     // TODO: handle expired certificates
     const date = new Date();