Add option to hide keys in log and print sha256(key) instead

api/librdb-api.h

@@ -436,20 +436,31 @@ _LIBRDB_API void RDB_IgnoreChecksum(RdbParser *p);
  * terminate its operation. The default threshold is unlimited. */
 _LIBRDB_API void RDB_setMaxRawSize(RdbParser *p, size_t maxSize);
 
-/* logger */
+/* Following function returns a hint for the total number of items in the current
+ * parsed key context - to assist with memory allocation or other optimizations.
+ * If hint is not available, then return -1. */
+_LIBRDB_API int64_t RDB_getNumItemsHint(RdbParser *p);
+
+/****************************************************************
+ * Logger
+ ****************************************************************/
+/* Set the logging level for the parser */
 _LIBRDB_API void RDB_setLogLevel(RdbParser *p, RdbLogLevel l);
+
+/* Set a custom logger callback function */
 _LIBRDB_API void RDB_setLogger(RdbParser *p, RdbLoggerCB f);
+
+/* Log a message with the specified log level */
 #ifdef __GNUC__
 _LIBRDB_API void RDB_log(RdbParser *p, RdbLogLevel lvl, const char *format, ...)
-                                          __attribute__((format(printf, 3, 4)));
+                         __attribute__((format(printf, 3, 4)));
 #else
 _LIBRDB_API void RDB_log(RdbParser *p, RdbLogLevel lvl, const char *format, ...);
 #endif
 
-/* Following function returns a hint for the total number of items in the current
- * parsed key context - to assist with memory allocation or other optimizations.
- * If hint is not available, then return -1. */
-_LIBRDB_API int64_t RDB_getNumItemsHint(RdbParser *p);
+/* To hide keys in logs by printing first 8 hex digits of SHA256(key) instead of 
+ * printing the key itself */
+_LIBRDB_API void RDB_hideKeysInLog(RdbParser *p);
 
 /****************************************************************
  * Pause the Parser

deps/redis/sha256.c

@@ -0,0 +1,163 @@
+/*********************************************************************
+* Filename:   sha256.c
+* Author:     Brad Conte (brad AT bradconte.com)
+* Copyright:
+* Disclaimer: This code is presented "as is" without any guarantees.
+* Details:    Implementation of the SHA-256 hashing algorithm.
+              SHA-256 is one of the three algorithms in the SHA2
+              specification. The others, SHA-384 and SHA-512, are not
+              offered in this implementation.
+              Algorithm specification can be found here:
+               * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf
+              This implementation uses little endian byte order.
+*********************************************************************/
+
+/*************************** HEADER FILES ***************************/
+#include <stdlib.h>
+#include <string.h>
+#include "sha256.h"
+
+/****************************** MACROS ******************************/
+#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b))))
+#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b))))
+
+#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z)))
+#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
+#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22))
+#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25))
+#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3))
+#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10))
+
+/**************************** VARIABLES *****************************/
+static const WORD k[64] = {
+        0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
+        0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
+        0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
+        0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
+        0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
+        0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
+        0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
+        0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+};
+
+/*********************** FUNCTION DEFINITIONS ***********************/
+void sha256_transform(SHA256_CTX *ctx, const BYTE data[])
+{
+    WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64];
+
+    for (i = 0, j = 0; i < 16; ++i, j += 4) {
+        m[i] = ((WORD) data[j + 0] << 24) |
+               ((WORD) data[j + 1] << 16) |
+               ((WORD) data[j + 2] << 8) |
+               ((WORD) data[j + 3]);
+    }
+
+    for ( ; i < 64; ++i)
+        m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16];
+
+    a = ctx->state[0];
+    b = ctx->state[1];
+    c = ctx->state[2];
+    d = ctx->state[3];
+    e = ctx->state[4];
+    f = ctx->state[5];
+    g = ctx->state[6];
+    h = ctx->state[7];
+
+    for (i = 0; i < 64; ++i) {
+        t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i];
+        t2 = EP0(a) + MAJ(a,b,c);
+        h = g;
+        g = f;
+        f = e;
+        e = d + t1;
+        d = c;
+        c = b;
+        b = a;
+        a = t1 + t2;
+    }
+
+    ctx->state[0] += a;
+    ctx->state[1] += b;
+    ctx->state[2] += c;
+    ctx->state[3] += d;
+    ctx->state[4] += e;
+    ctx->state[5] += f;
+    ctx->state[6] += g;
+    ctx->state[7] += h;
+}
+
+void sha256_init(SHA256_CTX *ctx)
+{
+    ctx->datalen = 0;
+    ctx->bitlen = 0;
+    ctx->state[0] = 0x6a09e667;
+    ctx->state[1] = 0xbb67ae85;
+    ctx->state[2] = 0x3c6ef372;
+    ctx->state[3] = 0xa54ff53a;
+    ctx->state[4] = 0x510e527f;
+    ctx->state[5] = 0x9b05688c;
+    ctx->state[6] = 0x1f83d9ab;
+    ctx->state[7] = 0x5be0cd19;
+}
+
+void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len)
+{
+    WORD i;
+
+    for (i = 0; i < len; ++i) {
+        ctx->data[ctx->datalen] = data[i];
+        ctx->datalen++;
+        if (ctx->datalen == 64) {
+            sha256_transform(ctx, ctx->data);
+            ctx->bitlen += 512;
+            ctx->datalen = 0;
+        }
+    }
+}
+
+void sha256_final(SHA256_CTX *ctx, BYTE hash[])
+{
+    WORD i;
+
+    i = ctx->datalen;
+
+    // Pad whatever data is left in the buffer.
+    if (ctx->datalen < 56) {
+        ctx->data[i++] = 0x80;
+        while (i < 56)
+            ctx->data[i++] = 0x00;
+    }
+    else {
+        ctx->data[i++] = 0x80;
+        while (i < 64)
+            ctx->data[i++] = 0x00;
+        sha256_transform(ctx, ctx->data);
+        memset(ctx->data, 0, 56);
+    }
+
+    // Append to the padding the total message's length in bits and transform.
+    ctx->bitlen += ctx->datalen * 8;
+    ctx->data[63] = ctx->bitlen;
+    ctx->data[62] = ctx->bitlen >> 8;
+    ctx->data[61] = ctx->bitlen >> 16;
+    ctx->data[60] = ctx->bitlen >> 24;
+    ctx->data[59] = ctx->bitlen >> 32;
+    ctx->data[58] = ctx->bitlen >> 40;
+    ctx->data[57] = ctx->bitlen >> 48;
+    ctx->data[56] = ctx->bitlen >> 56;
+    sha256_transform(ctx, ctx->data);
+
+    // Since this implementation uses little endian byte ordering and SHA uses big endian,
+    // reverse all the bytes when copying the final state to the output hash.
+    for (i = 0; i < 4; ++i) {
+        hash[i]      = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 4]  = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 8]  = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff;
+        hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff;
+    }
+}

deps/redis/sha256.h

@@ -0,0 +1,35 @@
+/*********************************************************************
+* Filename:   sha256.h
+* Author:     Brad Conte (brad AT bradconte.com)
+* Copyright:
+* Disclaimer: This code is presented "as is" without any guarantees.
+* Details:    Defines the API for the corresponding SHA256 implementation.
+*********************************************************************/
+
+#ifndef SHA256_H
+#define SHA256_H
+
+/*************************** HEADER FILES ***************************/
+#include <stddef.h>
+#include <stdint.h>
+
+/****************************** MACROS ******************************/
+#define SHA256_BLOCK_SIZE 32            // SHA256 outputs a 32 byte digest
+
+/**************************** DATA TYPES ****************************/
+typedef uint8_t BYTE;   // 8-bit byte
+typedef uint32_t WORD;  // 32-bit word
+
+typedef struct {
+    BYTE data[64];
+    WORD datalen;
+    unsigned long long bitlen;
+    WORD state[8];
+} SHA256_CTX;
+
+/*********************** FUNCTION DECLARATIONS **********************/
+void sha256_init(SHA256_CTX *ctx);
+void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len);
+void sha256_final(SHA256_CTX *ctx, BYTE hash[]);
+
+#endif   // SHA256_H

src/cli/rdb-cli.c

@@ -342,8 +342,8 @@ int matchRdbDataType(const char *dataTypeStr) {
     if (!strcmp(dataTypeStr, "func")) return RDB_DATA_TYPE_FUNCTION;
 
     loggerWrap(RDB_LOG_ERR,
-               "Invalid TYPE argument (%s). Valid values: str, list, set, zset, hash, module, stream, func",
-               dataTypeStr);
+        "Invalid TYPE argument (%s). Valid values: str, list, set, zset, hash, module, stream, func",
+        dataTypeStr);
     exit(1);
 }
 

src/ext/common.c → src/ext/extCommon.c

@@ -1,4 +1,4 @@
-#include "common.h"
+#include "extCommon.h"
 #include "../../deps/redis/util.h"
 
 /* Example:: Input: length=123  return: buf="\r\n$123\r\n" */

src/ext/common.h → src/ext/extCommon.h

@@ -50,4 +50,7 @@ static inline void iov_plain(struct iovec *iov, const char *s, size_t l) {
     iov->iov_len = l;
 }
 
+/*** hidden LIB API function (not declared in librdb-api.h) ***/
+_LIBRDB_API char *__RDB_key(RdbParser *p, char *key, char buf[9]);
+
 #endif /*define RDBX_COMMON_H*/

src/ext/handlersFilter.c

@@ -1,7 +1,7 @@
 #include <regex.h>
 #include <string.h>
 #include "../lib/defines.h" /* valid include since it brings only RDB_* defines */
-#include "common.h"
+#include "extCommon.h"
 
 struct RdbxFilter {
     regex_t regex_compiled;

src/ext/handlersToJson.c

@@ -4,7 +4,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <ctype.h>
-#include "common.h"
+#include "extCommon.h"
 #include "../../deps/redis/util.h"
 
 struct RdbxToJson;

src/ext/handlersToPrint.c

@@ -2,7 +2,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <ctype.h>
-#include "common.h"
+#include "extCommon.h"
 #include "../../deps/redis/util.h"
 
 struct RdbxToPrint;

src/ext/handlersToResp.c

@@ -2,7 +2,7 @@
 #include <assert.h>
 #include <sys/uio.h>
 #include <inttypes.h>
-#include "common.h"
+#include "extCommon.h"
 
 #include "../../deps/redis/crc64.h"
 #include "../../deps/redis/util.h"

src/ext/readerFile.c

@@ -1,7 +1,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
-#include "common.h"
+#include "extCommon.h"
 
 struct RdbxReaderFile {
     RdbParser *parser;

src/ext/readerFileDesc.c

@@ -5,7 +5,7 @@
 #include <string.h>
 #include <errno.h>
 #include <unistd.h>
-#include "common.h"
+#include "extCommon.h"
 
 
 struct RdbxReaderFileDesc {

src/ext/respToFileWriter.c

@@ -1,5 +1,5 @@
 #include <stdio.h>
-#include "common.h"
+#include "extCommon.h"
 #include <string.h>
 #include <errno.h>
 

src/ext/respToRedisLoader.c

@@ -6,7 +6,7 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <errno.h>
-#include "common.h"
+#include "extCommon.h"
 #include "readerResp.h"
 
 #ifdef USE_OPENSSL
@@ -71,10 +71,11 @@ static int onReadRepliesErrorCb(void *context, char *msg) {
         (strstr(msg, "not found")))              /* error includes "not found" */
         return 0; /* mask error */
 
+    char buf[9];
     RDB_reportError(ctx->p, (RdbRes) RDBX_ERR_RESP_WRITE,
                     "\nerror from dst '-%s' on key '%s' on command '%s' (RESP Command #%zu)\n",
                     msg,
-                    ctx->pendingCmds.key[currIdx],
+                    __RDB_key(ctx->p, ctx->pendingCmds.key[currIdx], buf),
                     ctx->pendingCmds.cmd[currIdx],
                     ctx->respReader.countReplies);