netSerial.c - netSerial - serializer producing compact bitstreams

netSerial.c (145543B)
      1 
      2 #include "libsheepyObject.h"
      3 #include "netSerial.h"
      4 #include "netSerialInternal.h"
      5 
      6 #include <stdlib.h>
      7 #include <string.h>
      8 #include <stdio.h>
      9 
     10 #define lv logVarG
     11 
     12 void initiateNetSerialLevel0(smallJsont *self);
     13 void initiateNetSerialLevel1(smallJsont *self);
     14 void initiateNetSerialLevel2(smallJsont *self);
     15 void initiateNetSerial(smallJsont *self);
     16 void registerMethodsNetSerialLevel0(smallJsonFunctionst *f);
     17 void registerMethodsNetSerialLevel1(smallJsonFunctionst *f);
     18 void registerMethodsNetSerialLevel2(smallJsonFunctionst *f);
     19 void registerMethodsNetSerial(smallJsonFunctionst *f);
     20 void initiateAllocateNetSerialLevel0(smallJsont **self);
     21 void initiateAllocateNetSerialLevel1(smallJsont **self);
     22 void initiateAllocateNetSerialLevel2(smallJsont **self);
     23 void initiateAllocateNetSerial(smallJsont **self);
     24 void finalizeNetSerial(void);
     25 smallJsont* allocNetSerialLevel0(void);
     26 smallJsont* allocNetSerialLevel1(void);
     27 smallJsont* allocNetSerialLevel2(void);
     28 smallJsont* allocNetSerial(void);
     29 internal const char* helpNetSerial(smallJsont *self);
     30 internal smallBytest* serialNetSerialLevel0(smallJsont *self);
     31 internal smallBytest* serialNetSerialLevel1(smallJsont *self);
     32 internal smallBytest* serialNetSerialLevel2(smallJsont *self);
     33 internal smallBytest* serialNetSerial(smallJsont *self);
     34 internal smallJsont* deserialNetSerialLevel0(smallJsont *self, smallBytest *data);
     35 internal smallJsont* deserialNetSerialLevel1(smallJsont *self, smallBytest *data);
     36 internal smallJsont* deserialNetSerialLevel2(smallJsont *self, smallBytest *data);
     37 internal smallJsont* deserialNetSerial(smallJsont *self, smallBytest *data);
     38 
     39 internal void uintToNetTypeVarint(sBytest **buf, u8 type, u64 value);
     40 internal void uintToVarint(sBytest **buf, u64 value);
     41 internal u64 netTypeVarintToUint(u8 **buf);
     42 internal u64 varintToUint(u8 **buf);
     43 
     44 internal sBytest* netSerialLevel0(smallt *o);
     45 internal void dictNetSerialLevel0(sBytest **r, sDictt *dict);
     46 internal void arrayNetSerialLevel0(sBytest **r, sArrayt *array);
     47 internal sBytest* netSerialLevel1(smallt *o);
     48 internal void dictNetSerialLevel1(sBytest **r, sDictt *dict, contextt *ctx);
     49 internal void arrayNetSerialLevel1(sBytest **r, sArrayt *array, contextt *ctx);
     50 internal sBytest* netSerialLevel2(smallt *o);
     51 internal void dictNetSerialLevel2(sBytest **r, sDictt *dict, contextt *ctx, bool packed);
     52 internal void arrayNetSerialLevel2(sBytest **r, sArrayt *array, contextt *ctx, bool packed);
     53 internal sBytest* netSerial(smallt *o);
     54 internal void dictNetSerial(sBytest **r, sDictt *dict, contextt *ctx, packingT packing);
     55 internal void arrayNetSerial(sBytest **r, sArrayt *array, contextt *ctx, packingT packing);
     56 
     57 internal smallt* netDeserialLevel0(sBytest *obj);
     58 internal void dictNetDeserialLevel0(sDictt **dict, char **data);
     59 internal void arrayNetDeserialLevel0(sArrayt **array, char **data);
     60 internal smallt* netDeserialLevel1(sBytest *obj);
     61 internal void dictNetDeserialLevel1(sDictt **dict, u8 **data, contextt *ctx);
     62 internal void arrayNetDeserialLevel1(sArrayt **array, u8 **data, contextt *ctx);
     63 internal smallt* netDeserialLevel2(sBytest *obj);
     64 internal void dictNetDeserialLevel2(sDictt **dict, u8 **data, contextt *ctx, bool packed);
     65 internal u8 isDictUniform(sDictt *dict);
     66 internal u8 isArrayUniform(sArrayt *array);
     67 internal void uniformDictNetDeserialLevel2(sDictt **dict, u8 **data, contextt *ctx, bool packed);
     68 internal void arrayNetDeserialLevel2(sArrayt **array, u8 **data, contextt *ctx, bool packed);
     69 internal void uniformArrayNetDeserialLevel2(sArrayt **array, u8 **data, contextt *ctx, bool packed);
     70 internal smallt* netDeserial(sBytest *obj);
     71 internal void dictNetDeserial(sDictt **dict, u8 **data, contextt *ctx, bool packed);
     72 internal void uniformDictNetDeserial(sDictt **dict, u8 **data, contextt *ctx, bool packed);
     73 internal void arrayNetDeserial(sArrayt **array, u8 **data, contextt *ctx, bool packed);
     74 internal void uniformArrayNetDeserial(sArrayt **array, u8 **data, contextt *ctx, bool packed);
     75 
     76 /* enable/disable logging */
     77 /* #undef pLog */
     78 /* #define pLog(...) */
     79 
     80 void initiateNetSerialLevel0(smallJsont *self) {
     81 
     82   initiateSmallJson(self);
     83 
     84   self->type = "netSerial";
     85   if (!netSerialF) {
     86     netSerialF            = malloc(sizeof(smallJsonFunctionst));
     87     registerMethodsNetSerialLevel0(netSerialF);
     88     pErrorNot0(atexit(finalizeNetSerial));
     89   }
     90   self->f = netSerialF;
     91 }
     92 
     93 void initiateNetSerialLevel1(smallJsont *self) {
     94 
     95   initiateSmallJson(self);
     96 
     97   self->type = "netSerial";
     98   if (!netSerialF) {
     99     netSerialF            = malloc(sizeof(smallJsonFunctionst));
    100     registerMethodsNetSerialLevel1(netSerialF);
    101     pErrorNot0(atexit(finalizeNetSerial));
    102   }
    103   self->f = netSerialF;
    104 }
    105 
    106 void initiateNetSerialLevel2(smallJsont *self) {
    107 
    108   initiateSmallJson(self);
    109 
    110   self->type = "netSerial";
    111   if (!netSerialF) {
    112     netSerialF            = malloc(sizeof(smallJsonFunctionst));
    113     registerMethodsNetSerialLevel2(netSerialF);
    114     pErrorNot0(atexit(finalizeNetSerial));
    115   }
    116   self->f = netSerialF;
    117 }
    118 
    119 void initiateNetSerial(smallJsont *self) {
    120 
    121   initiateSmallJson(self);
    122 
    123   self->type = "netSerial";
    124   if (!netSerialF) {
    125     netSerialF            = malloc(sizeof(smallJsonFunctionst));
    126     registerMethodsNetSerial(netSerialF);
    127     pErrorNot0(atexit(finalizeNetSerial));
    128   }
    129   self->f = netSerialF;
    130 }
    131 
    132 void registerMethodsNetSerialLevel0(smallJsonFunctionst *f) {
    133 
    134   registerMethodsSmallJson(f);
    135   f->help      = helpNetSerial;
    136   f->serial    = serialNetSerialLevel0;
    137   f->deserial  = deserialNetSerialLevel0;
    138 }
    139 
    140 void registerMethodsNetSerialLevel1(smallJsonFunctionst *f) {
    141 
    142   registerMethodsSmallJson(f);
    143   f->help      = helpNetSerial;
    144   f->serial    = serialNetSerialLevel1;
    145   f->deserial  = deserialNetSerialLevel1;
    146 }
    147 
    148 void registerMethodsNetSerialLevel2(smallJsonFunctionst *f) {
    149 
    150   registerMethodsSmallJson(f);
    151   f->help      = helpNetSerial;
    152   f->serial    = serialNetSerialLevel2;
    153   f->deserial  = deserialNetSerialLevel2;
    154 }
    155 
    156 void registerMethodsNetSerial(smallJsonFunctionst *f) {
    157 
    158   registerMethodsSmallJson(f);
    159   f->help      = helpNetSerial;
    160   f->serial    = serialNetSerial;
    161   f->deserial  = deserialNetSerial;
    162 }
    163 
    164 void initiateAllocateNetSerialLevel0(smallJsont **self) {
    165 
    166   if (self) {
    167     initiateG(self); // call initiateAllocateSmallJson to initialize the container recycling system
    168     if (*self) {
    169       initiateNetSerialLevel0(*self);
    170     }
    171   }
    172 }
    173 
    174 void initiateAllocateNetSerialLevel1(smallJsont **self) {
    175 
    176   if (self) {
    177     initiateG(self); // call initiateAllocateSmallJson to initialize the container recycling system
    178     if (*self) {
    179       initiateNetSerialLevel1(*self);
    180     }
    181   }
    182 }
    183 
    184 void initiateAllocateNetSerialLevel2(smallJsont **self) {
    185 
    186   if (self) {
    187     initiateG(self); // call initiateAllocateSmallJson to initialize the container recycling system
    188     if (*self) {
    189       initiateNetSerialLevel2(*self);
    190     }
    191   }
    192 }
    193 
    194 void initiateAllocateNetSerial(smallJsont **self) {
    195 
    196   if (self) {
    197     initiateG(self); // call initiateAllocateSmallJson to initialize the container recycling system
    198     if (*self) {
    199       initiateNetSerial(*self);
    200     }
    201   }
    202 }
    203 
    204 void finalizeNetSerial(void) {
    205 
    206   if (netSerialF) {
    207     free(netSerialF);
    208     netSerialF = NULL;
    209   }
    210 }
    211 
    212 smallJsont* allocNetSerialLevel0(void) {
    213   smallJsont *r = NULL;
    214 
    215   initiateAllocateNetSerialLevel0(&r);
    216   ret r;
    217 }
    218 
    219 smallJsont* allocNetSerialLevel1(void) {
    220   smallJsont *r = NULL;
    221 
    222   initiateAllocateNetSerialLevel1(&r);
    223   ret r;
    224 }
    225 
    226 smallJsont* allocNetSerialLevel2(void) {
    227   smallJsont *r = NULL;
    228 
    229   initiateAllocateNetSerialLevel2(&r);
    230   ret r;
    231 }
    232 
    233 smallJsont* allocNetSerial(void) {
    234   smallJsont *r = NULL;
    235 
    236   initiateAllocateNetSerial(&r);
    237   ret r;
    238 }
    239 
    240 
    241 internal const char* helpNetSerial(smallJsont UNUSED *self) {
    242   ret "TODO helpNetSerial \n" helpTextSmallJson;
    243 }
    244 
    245 /**
    246  * encode type in lower nibble, uint value in high nibble and next bytes as varuint
    247  */
    248 internal void uintToNetTypeVarint(sBytest **buf, u8 type, u64 value) {
    249   u64 c = value;
    250   /* encode b0..2 */
    251   u8  b = type + ((c & 0x7) << 4);
    252   if (c & 0xFFFFFFFFFFFFFFF8) {
    253     b |= 0x80;
    254     sBytesPush(buf, b);
    255     c >>=3;
    256     /* encode b3..9 ...*/
    257     while(c) {
    258       b = c & 0x7F;
    259       if (c & 0xFFFFFFFFFFFFFF80)
    260         b |= 0x80;
    261       sBytesPush(buf, b);
    262       c >>=7;
    263     }
    264   }
    265   else
    266     sBytesPush(buf, b);
    267 }
    268 
    269 /**
    270  * encode uint as varuint
    271  */
    272 internal void uintToVarint(sBytest **buf, u64 value) {
    273   u64 c = value;
    274   u8  b = c & 0x7F;
    275   if (c & 0xFFFFFFFFFFFFFF80) {
    276     b |= 0x80;
    277     sBytesPush(buf, b);
    278     c >>=7;
    279     /* encode b7..14 ...*/
    280     while(c) {
    281       b = c & 0x7F;
    282       if (c & 0xFFFFFFFFFFFFFF80)
    283         b |= 0x80;
    284       sBytesPush(buf, b);
    285       c >>=7;
    286     }
    287   }
    288   else
    289     sBytesPush(buf, b);
    290 }
    291 
    292 /**
    293  * decode type and varuint to uint
    294  */
    295 internal u64 netTypeVarintToUint(u8 **buf) {
    296   u64 r = 0;
    297 
    298   r = (**buf >> 4) & 0x7;
    299 
    300   u8 c = 0;
    301   while (**buf & 0x80) {
    302     (*buf)++;
    303     // note: keep 0x7FUL to use 64 bit shift operation, without UL a 32 bit shift operation is use and then casted to 64 bit
    304     r |= (**buf & 0x7FUL) << (7*c+3);
    305     c++;
    306   }
    307   (*buf)++;
    308   ret r;
    309 }
    310 
    311 /**
    312  * decode varuint to uint
    313  */
    314 internal u64 varintToUint(u8 **buf) {
    315   u64 r = 0;
    316 
    317   r = (**buf) & 0x7F;
    318   u8 c = 1;
    319   while (**buf & 0x80) {
    320     (*buf)++;
    321     // note: keep 0x7FUL to use 64 bit shift operation, without UL a 32 bit shift operation is use and then casted to 64 bit
    322     r |= (**buf & 0x7FUL) << (7*c);
    323     c++;
    324   }
    325   (*buf)++;
    326   ret r;
    327 }
    328 
    329 // -------------------------------------
    330 // Serializers
    331 
    332 // level 0
    333 // like smallJson with ints and length encoded as varints
    334 
    335 /**
    336  * serializer top function
    337  */
    338 internal sBytest* netSerialLevel0(smallt *o) {
    339   sBytest *r = NULL;
    340   sBytest *B = NULL;
    341 
    342   switch(o->type) {
    343     case UNDEFINED:
    344       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    345       break;
    346     case BOOL: {
    347       u8 c = NET_SERIAL_TYPES[(u8)o->type];
    348       // set bit 4 when true
    349       if (((sBoolt *)&(o->type))->value)
    350         c |= (1<<4);
    351       sBytesPush(&r, c);
    352       }
    353       break;
    354     case CONTAINER:
    355       // undefined
    356       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    357       break;
    358     case DICT:
    359       dictNetSerialLevel0(&r, (sDictt *)&(o->type));
    360       break;
    361     case DOUBLE:
    362       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    363       sBytesPushBuffer(&r, &((sDoublet *)&(o->type))->value, sizeof(double));
    364       break;
    365     case INT: {
    366       // encode int to varint
    367       // v is int64_t to convert to varint
    368       i64 v = ((sIntt *)&(o->type))->value;
    369       // encode v with arithmetic shifts
    370       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], (v << 1) ^ (v >> 63));
    371       }
    372       break;
    373     case STRING:
    374       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    375       sBytesPushBuffer(&r, &((sStringt *)&(o->type))->data, sizeof(sStringt) + strlen(&(((sStringt *)&(o->type))->data)) -1);
    376       break;
    377     case ARRAY:
    378       arrayNetSerialLevel0(&r, (sArrayt *)&(o->type));
    379       break;
    380     case BYTES:
    381       B = (sBytest *)&(o->type);
    382       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], B->count);
    383       sBytesPushBuffer(&r, &(B->data), B->count);
    384       break;
    385   }
    386   ret r;
    387 }
    388 
    389 /**
    390  * serialize dictionary
    391  *
    392  * the serialized dict is pushed to r.
    393  * All elements are serialized recursively
    394  *
    395  * the data in containers is not serialized
    396  *
    397  * \param
    398  *   r small bytes object
    399  *   dict dictionary to serialize
    400  */
    401 internal void dictNetSerialLevel0(sBytest **r, sDictt *dict) {
    402   sBytest *B = NULL;
    403 
    404   uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)dict->type], dict->count);
    405 
    406   forEachSDict(dict, e) {
    407     if (e->key) {
    408       sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
    409 
    410       switch(e->data->type) {
    411         case UNDEFINED:
    412           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->data->type]);
    413           break;
    414         case BOOL: {
    415             u8 c = NET_SERIAL_TYPES[(u8)e->data->type];
    416             // set bit 4 when true
    417             if (((sBoolt *)(e->data))->value)
    418               c |= (1<<4);
    419             sBytesPush(r, c);
    420            }
    421           break;
    422         case CONTAINER:
    423           // undefined
    424           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->data->type]);
    425           break;
    426         case DICT:
    427           dictNetSerialLevel0(r, (sDictt *)(e->data));
    428           break;
    429         case DOUBLE:
    430           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->data->type]);
    431           sBytesPushBuffer(r, &((sDoublet *)(e->data))->value, sizeof(double));
    432           break;
    433         case INT: {
    434             // encode int to varint
    435             // v is int64_t to convert to varint
    436             i64 v = ((sIntt *)(e->data))->value;
    437             // encode v with arithmetic shifts
    438             uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], (v << 1) ^ (v >> 63));
    439            }
    440           break;
    441         case STRING:
    442           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->data->type]);
    443           sBytesPushBuffer(r, &((sStringt *)(e->data))->data, sizeof(sStringt) + strlen(&(((sStringt *)(e->data))->data)) -1);
    444           break;
    445         case ARRAY:
    446           arrayNetSerialLevel0(r, (sArrayt *)(e->data));
    447           break;
    448         case BYTES:
    449           B = (sBytest *)(e->data);
    450           uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], B->count);
    451           sBytesPushBuffer(r, &(B->data), B->count);
    452           break;
    453        }
    454      }
    455   }
    456   ret;
    457 }
    458 
    459 /**
    460  * serialize array
    461  *
    462  * the serialized array is pushed to r.
    463  * All elements are serialized recursively
    464  *
    465  * the data in containers is not serialized
    466  *
    467  * \param
    468  *   r small bytes object
    469  *   array to serialize
    470  */
    471 internal void arrayNetSerialLevel0(sBytest **r, sArrayt *array) {
    472   sBytest *B = NULL;
    473 
    474   uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)array->type], array->count);
    475 
    476   forEachSArray(array, e) {
    477     if (!e) {
    478       // empty slots are represented as undefined elements
    479       sBytesPush(r, NET_SERIAL_TYPES[UNDEFINED]);
    480     }
    481     else {
    482       switch(e->type) {
    483         case UNDEFINED:
    484           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->type]);
    485           break;
    486         case BOOL: {
    487             u8 c = NET_SERIAL_TYPES[(u8)e->type];
    488             // set bit 4 when true
    489             if (((sBoolt *)&(e->type))->value)
    490               c |= (1<<4);
    491             sBytesPush(r, c);
    492           }
    493           break;
    494         case CONTAINER:
    495           // undefined
    496           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->type]);
    497           break;
    498         case DICT:
    499           dictNetSerialLevel0(r, (sDictt *)e);
    500           break;
    501         case DOUBLE:
    502           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->type]);
    503           sBytesPushBuffer(r, &((sDoublet *)e)->value, sizeof(double));
    504           break;
    505         case INT: {
    506             // encode int to varint
    507             // v is int64_t to convert to varint
    508             i64 v = ((sIntt *)&(e->type))->value;
    509             // encode v with arithmetic shifts
    510             uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], (v << 1) ^ (v >> 63));
    511           }
    512           break;
    513         case STRING:
    514           sBytesPush(r, NET_SERIAL_TYPES[(u8)e->type]);
    515           sBytesPushBuffer(r, &((sStringt *)e)->data, sizeof(sStringt) + strlen(&(((sStringt *)e)->data)) -1);
    516           break;
    517         case ARRAY:
    518           arrayNetSerialLevel0(r, (sArrayt *)e);
    519           break;
    520         case BYTES:
    521           B = (sBytest *)e;
    522           uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], B->count);
    523           sBytesPushBuffer(r, &(B->data), B->count);
    524           break;
    525       }
    526     }
    527   }
    528   ret;
    529 }
    530 
    531 internal smallBytest* serialNetSerialLevel0(smallJsont *self) {
    532 
    533   smallt *o = getsoG(self);
    534 
    535   if (o == NULL)
    536     ret NULL;
    537 
    538   sBytest *B = netSerialLevel0(o);
    539 
    540   if (!B) {
    541     ret NULL;
    542   }
    543 
    544   createAllocateSmallBytes(r);
    545   r->B = B;
    546   ret r;
    547 }
    548 
    549 // level 1
    550 // like level 0 with type encoded in nibbles and bools are packed
    551 
    552 /**
    553  * serializer top function
    554  */
    555 internal sBytest* netSerialLevel1(smallt *o) {
    556   sBytest *r = NULL;
    557   sBytest *B = NULL;
    558   contextt ctx = {.nibble=lowNbl, .nblOffset=0, .boolShift = 0, .boolOffset=0};
    559 
    560   switch(o->type) {
    561     case UNDEFINED:
    562     case CONTAINER:
    563       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    564       break;
    565     case BOOL: {
    566       u8 c = NET_SERIAL_TYPES[(u8)o->type];
    567       // set bit 4 when true
    568       if (((sBoolt *)&(o->type))->value)
    569         c |= (1<<4);
    570       sBytesPush(&r, c);
    571       }
    572       break;
    573     case DICT:
    574       dictNetSerialLevel1(&r, (sDictt *)&(o->type), &ctx);
    575       break;
    576     case DOUBLE:
    577       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    578       sBytesPushBuffer(&r, &((sDoublet *)&(o->type))->value, sizeof(double));
    579       break;
    580     case INT: {
    581       // encode int to varint
    582       // v is int64_t to convert to varint
    583       i64 v = ((sIntt *)&(o->type))->value;
    584       // encode v with arithmetic shifts
    585       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], (v << 1) ^ (v >> 63));
    586       }
    587       break;
    588     case STRING:
    589       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    590       sBytesPushBuffer(&r, &((sStringt *)&(o->type))->data, sizeof(sStringt) + strlen(&(((sStringt *)&(o->type))->data)) -1);
    591       break;
    592     case ARRAY:
    593       arrayNetSerialLevel1(&r, (sArrayt *)&(o->type), &ctx);
    594       break;
    595     case BYTES:
    596       B = (sBytest *)&(o->type);
    597       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], B->count);
    598       sBytesPushBuffer(&r, &(B->data), B->count);
    599       break;
    600   }
    601   ret r;
    602 }
    603 
    604 /**
    605  * serialize dictionary
    606  *
    607  * the serialized dict is pushed to r.
    608  * All elements are serialized recursively
    609  *
    610  * the data in containers is not serialized
    611  *
    612  * \param
    613  *   r small bytes object
    614  *   dict dictionary to serialize
    615  */
    616 internal void dictNetSerialLevel1(sBytest **r, sDictt *dict, contextt *ctx) {
    617   sBytest *B     = NULL;
    618   char    *data  = NULL;
    619 
    620   if (ctx->nibble == lowNbl) {
    621     uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)dict->type], dict->count);
    622   }
    623   else {
    624     // high nibble
    625     #define storeTypeInHighNbl(o)\
    626     ctx->nibble  = lowNbl;\
    627     data         = (char *)&((*r)->data) + ctx->nblOffset;\
    628     *data       |= NET_SERIAL_TYPES[(u8)o->type] << 4
    629     storeTypeInHighNbl(dict);
    630     uintToVarint(r, dict->count);
    631   }
    632 
    633   forEachSDict(dict, e) {
    634     if (e->key) {
    635       sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
    636 
    637       switch(e->data->type) {
    638         case UNDEFINED:
    639         case CONTAINER:
    640           if (ctx->nibble == lowNbl) {
    641             #define storeTypeOnly(o)\
    642             sBytesPush(r, NET_SERIAL_TYPES[(u8)o->type]);\
    643             ctx->nibble    = highNbl;\
    644             ctx->nblOffset = (*r)->count -1
    645             storeTypeOnly(e->data);
    646           }
    647           else {
    648             storeTypeInHighNbl(e->data);
    649           }
    650           break;
    651         case BOOL:
    652           if (!ctx->boolOffset) {
    653             // new packed bools
    654             if (ctx->nibble == lowNbl) {
    655               #define storeNew4bPackedBool(o)\
    656               u8 c = NET_SERIAL_TYPES[(u8)o->type];\
    657               /* set bit 4 when true */\
    658               if (((sBoolt *)(o))->value)\
    659                 c |= (1<<4);\
    660               sBytesPush(r, c);\
    661               ctx->boolShift  = 5;\
    662               ctx->boolOffset = (*r)->count -1
    663               storeNew4bPackedBool(e->data);
    664             }
    665             else {
    666               // high nibble, next byte is packed bools
    667               storeTypeInHighNbl(e->data);
    668               #define storeNew8bPackedBool(o)\
    669               u8 c         = 0;\
    670               if (((sBoolt *)(o))->value)\
    671                 c = 1;\
    672               sBytesPush(r, c);\
    673               ctx->boolShift  = 1;\
    674               ctx->boolOffset = (*r)->count -1
    675               storeNew8bPackedBool(e->data);
    676             }
    677           }
    678           else {
    679             // there was a bool before this one, fill bits in nibbles
    680             if (ctx->nibble == lowNbl) {
    681               if (ctx->boolShift == 8) {
    682                 // previous packed bool is full
    683                 // this byte is the new packed bools
    684                 storeNew4bPackedBool(e->data);
    685               }
    686               else {
    687                 storeTypeOnly(e->data);
    688                 #define storeBool(o)\
    689                 data           = (char *)&((*r)->data) + ctx->boolOffset;\
    690                 if (((sBoolt *)(o))->value)\
    691                   *data |= 1 << ctx->boolShift;\
    692                 ctx->boolShift++
    693                 storeBool(e->data);
    694               }
    695             }
    696             else {
    697               // high nibble
    698               storeTypeInHighNbl(e->data);
    699               if (ctx->boolShift == 8) {
    700                 // previous packed bool is full
    701                 // next byte is the new packed bools
    702                 storeNew8bPackedBool(e->data);
    703               }
    704               else {
    705                 storeBool(e->data);
    706               }
    707             }
    708           }
    709           break;
    710         case DICT:
    711           dictNetSerialLevel1(r, (sDictt *)(e->data), ctx);
    712           break;
    713         case DOUBLE:
    714           if (ctx->nibble == lowNbl) {
    715             storeTypeOnly(e->data);
    716           }
    717           else {
    718             // high nibble
    719             storeTypeInHighNbl(e->data);
    720           }
    721           sBytesPushBuffer(r, &((sDoublet *)(e->data))->value, sizeof(double));
    722           break;
    723         case INT: {
    724             // encode int to varint
    725             // v is int64_t to convert to varint
    726             i64 v = ((sIntt *)(e->data))->value;
    727             if (ctx->nibble == lowNbl) {
    728               // encode v with arithmetic shifts
    729               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], (v << 1) ^ (v >> 63));
    730             }
    731             else {
    732               // high nibble
    733               storeTypeInHighNbl(e->data);
    734               uintToVarint(r, (v << 1) ^ (v >> 63));
    735             }
    736           }
    737           break;
    738         case STRING:
    739           if (ctx->nibble == lowNbl) {
    740             storeTypeOnly(e->data);
    741           }
    742           else {
    743             // high nibble
    744             storeTypeInHighNbl(e->data);
    745           }
    746           sBytesPushBuffer(r, &((sStringt *)(e->data))->data, sizeof(sStringt) + strlen(&(((sStringt *)(e->data))->data)) -1);
    747           break;
    748         case ARRAY:
    749           arrayNetSerialLevel1(r, (sArrayt *)(e->data), ctx);
    750           break;
    751         case BYTES:
    752           B = (sBytest *)(e->data);
    753           if (ctx->nibble == lowNbl) {
    754             uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], B->count);
    755           }
    756           else {
    757             // high nibble
    758             storeTypeInHighNbl(e->data);
    759             uintToVarint(r, B->count);
    760           }
    761           sBytesPushBuffer(r, &(B->data), B->count);
    762           break;
    763       }
    764     }
    765   }
    766   ret;
    767 }
    768 
    769 /**
    770  * serialize array
    771  *
    772  * the serialized array is pushed to r.
    773  * All elements are serialized recursively
    774  *
    775  * the data in containers is not serialized
    776  *
    777  * \param
    778  *   r small bytes object
    779  *   array to serialize
    780  */
    781 internal void arrayNetSerialLevel1(sBytest **r, sArrayt *array, contextt *ctx) {
    782   sBytest *B     = NULL;
    783   char    *data  = NULL;
    784 
    785   if (ctx->nibble == lowNbl) {
    786     uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)array->type], array->count);
    787   }
    788   else {
    789     // high nibble
    790     storeTypeInHighNbl(array);
    791     uintToVarint(r, array->count);
    792   }
    793 
    794   forEachSArray(array, e) {
    795     if (!e) {
    796       // empty slots are represented as undefined elements
    797       if (ctx->nibble == lowNbl) {
    798         sBytesPush(r, NET_SERIAL_TYPES[UNDEFINED]);
    799         ctx->nibble    = highNbl;
    800         ctx->nblOffset = (*r)->count -1;
    801       }
    802       else {
    803         // high nibble
    804         ctx->nibble  = lowNbl;
    805         data         = (char *)&((*r)->data) + ctx->nblOffset;
    806         *data       |= NET_SERIAL_TYPES[UNDEFINED] << 4;
    807       }
    808     }
    809     else {
    810       switch(e->type) {
    811         case UNDEFINED:
    812         case CONTAINER:
    813           if (ctx->nibble == lowNbl) {
    814             storeTypeOnly(e);
    815           }
    816           else {
    817             // high nibble
    818             storeTypeInHighNbl(e);
    819           }
    820           break;
    821         case BOOL:
    822           if (!ctx->boolOffset) {
    823             // new packed bools
    824             if (ctx->nibble == lowNbl) {
    825               storeNew4bPackedBool(e);
    826             }
    827             else {
    828               // high nibble, next byte is packed bools
    829               storeTypeInHighNbl(e);
    830               storeNew8bPackedBool(e);
    831             }
    832           }
    833           else {
    834             // there was a bool before this one, fill bits in nibbles
    835             if (ctx->nibble == lowNbl) {
    836               if (ctx->boolShift == 8) {
    837                 // previous packed bool is full
    838                 // this byte is the new packed bools
    839                 storeNew4bPackedBool(e);
    840               }
    841               else {
    842                 storeTypeOnly(e);
    843                 storeBool(e);
    844               }
    845             }
    846             else {
    847               // high nibble
    848               storeTypeInHighNbl(e);
    849               if (ctx->boolShift == 8) {
    850                 // previous packed bool is full
    851                 // next byte is the new packed bools
    852                 storeNew8bPackedBool(e);
    853               }
    854               else {
    855                 storeBool(e);
    856               }
    857             }
    858           }
    859           break;
    860         case DICT:
    861           dictNetSerialLevel1(r, (sDictt *)e, ctx);
    862           break;
    863         case DOUBLE:
    864           if (ctx->nibble == lowNbl) {
    865             storeTypeOnly(e);
    866           }
    867           else {
    868             // high nibble
    869             storeTypeInHighNbl(e);
    870           }
    871           sBytesPushBuffer(r, &((sDoublet *)e)->value, sizeof(double));
    872           break;
    873         case INT: {
    874             // encode int to varint
    875             // v is int64_t to convert to varint
    876             i64 v = ((sIntt *)&(e->type))->value;
    877             if (ctx->nibble == lowNbl) {
    878               // encode v with arithmetic shifts
    879               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], (v << 1) ^ (v >> 63));
    880             }
    881             else {
    882               // high nibble
    883               storeTypeInHighNbl(e);
    884               uintToVarint(r, (v << 1) ^ (v >> 63));
    885             }
    886           }
    887           break;
    888         case STRING:
    889           if (ctx->nibble == lowNbl) {
    890             storeTypeOnly(e);
    891           }
    892           else {
    893             // high nibble
    894             storeTypeInHighNbl(e);
    895           }
    896           sBytesPushBuffer(r, &((sStringt *)e)->data, sizeof(sStringt) + strlen(&(((sStringt *)e)->data)) -1);
    897           break;
    898         case ARRAY:
    899           arrayNetSerialLevel1(r, (sArrayt *)e, ctx);
    900           break;
    901         case BYTES:
    902           B = (sBytest *)e;
    903           if (ctx->nibble == lowNbl) {
    904             uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], B->count);
    905           }
    906           else {
    907             // high nibble
    908             storeTypeInHighNbl(e);
    909             uintToVarint(r, B->count);
    910           }
    911           sBytesPushBuffer(r, &(B->data), B->count);
    912           break;
    913       }
    914     }
    915   }
    916   ret;
    917 }
    918 
    919 internal smallBytest* serialNetSerialLevel1(smallJsont *self) {
    920 
    921   smallt *o = getsoG(self);
    922 
    923   if (o == NULL)
    924     ret NULL;
    925 
    926   sBytest *B = netSerialLevel1(o);
    927 
    928   if (!B) {
    929     ret NULL;
    930   }
    931 
    932   createAllocateSmallBytes(r);
    933   r->B = B;
    934   ret r;
    935 }
    936 
    937 // level 2
    938 // like level 1, arrays are set to uniform when all elements are same type
    939 
    940 /**
    941  * serializer top function
    942  */
    943 internal sBytest* netSerialLevel2(smallt *o) {
    944   sBytest *r = NULL;
    945   sBytest *B = NULL;
    946   contextt ctx = {.nibble=lowNbl, .nblOffset=0, .boolShift = 0, .boolOffset=0};
    947 
    948   switch(o->type) {
    949     case UNDEFINED:
    950     case CONTAINER:
    951       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    952       break;
    953     case BOOL: {
    954       u8 c = NET_SERIAL_TYPES[(u8)o->type];
    955       // set bit 4 when true
    956       if (((sBoolt *)&(o->type))->value)
    957         c |= (1<<4);
    958       sBytesPush(&r, c);
    959       }
    960       break;
    961     case DICT:
    962       dictNetSerialLevel2(&r, (sDictt *)&(o->type), &ctx, /*packed=*/false);
    963       break;
    964     case DOUBLE:
    965       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    966       sBytesPushBuffer(&r, &((sDoublet *)&(o->type))->value, sizeof(double));
    967       break;
    968     case INT: {
    969       // encode int to varint
    970       // v is int64_t to convert to varint
    971       i64 v = ((sIntt *)&(o->type))->value;
    972       // encode v with arithmetic shifts
    973       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], (v << 1) ^ (v >> 63));
    974       }
    975       break;
    976     case STRING:
    977       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
    978       sBytesPushBuffer(&r, &((sStringt *)&(o->type))->data, sizeof(sStringt) + strlen(&(((sStringt *)&(o->type))->data)) -1);
    979       break;
    980     case ARRAY:
    981       arrayNetSerialLevel2(&r, (sArrayt *)&(o->type), &ctx, /*packed=*/false);
    982       break;
    983     case BYTES:
    984       B = (sBytest *)&(o->type);
    985       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], B->count);
    986       sBytesPushBuffer(&r, &(B->data), B->count);
    987       break;
    988   }
    989   ret r;
    990 }
    991 
    992 internal u8 isDictUniform(sDictt *dict) {
    993   bool allElementsHaveSameType = true;
    994   bool foundFirstType          = false;
    995   u8 type                      = 0;
    996   forEachSDict(dict, e) {
    997     if (e->key) {
    998       if (foundFirstType) {
    999         u8 nextType;
   1000         switch(e->data->type) {
   1001           case DICT:
   1002             nextType = isDictUniform((sDictt*)e->data);
   1003             break;
   1004           case ARRAY:
   1005             nextType = isArrayUniform((sArrayt*)e->data);
   1006             break;
   1007           default:
   1008             nextType = NET_SERIAL_TYPES[(u8)e->data->type];
   1009         }
   1010         if (nextType != type) {
   1011           allElementsHaveSameType = false;
   1012           break;
   1013         }
   1014       }
   1015       else {
   1016         switch(e->data->type) {
   1017           case DICT:
   1018             type = isDictUniform((sDictt*)e->data);
   1019             break;
   1020           case ARRAY:
   1021             type = isArrayUniform((sArrayt*)e->data);
   1022             break;
   1023           default:
   1024             type = NET_SERIAL_TYPES[(u8)e->data->type];
   1025         }
   1026         foundFirstType = true;
   1027       }
   1028     }
   1029   }
   1030   if (allElementsHaveSameType)
   1031     type = UNIFORM_DICT;
   1032   else
   1033     type = S_DICT;
   1034   ret type;
   1035 }
   1036 
   1037 internal u8 isArrayUniform(sArrayt *array) {
   1038   bool allElementsHaveSameType = true;
   1039   bool foundFirstType          = false;
   1040   char type                    = 0;
   1041   forEachSArray(array, e) {
   1042     if (!e) {
   1043       if (foundFirstType) {
   1044         if (type != S_UNDEFINED) {
   1045           allElementsHaveSameType = false;
   1046           break;
   1047         }
   1048       }
   1049       else {
   1050         type = S_UNDEFINED;
   1051         foundFirstType = true;
   1052       }
   1053     }
   1054     else {
   1055       if (foundFirstType) {
   1056         u8 nextType;
   1057         switch(e->type) {
   1058           case DICT:
   1059             nextType = isDictUniform((sDictt*)e);
   1060             break;
   1061           case ARRAY:
   1062             nextType = isArrayUniform((sArrayt*)e);
   1063             break;
   1064           default:
   1065             nextType = NET_SERIAL_TYPES[(u8)e->type];
   1066         }
   1067         if (nextType != type) {
   1068           allElementsHaveSameType = false;
   1069           break;
   1070         }
   1071       }
   1072       else {
   1073         switch(e->type) {
   1074           case DICT:
   1075             type = isDictUniform((sDictt*)e);
   1076             break;
   1077           case ARRAY:
   1078             type = isArrayUniform((sArrayt*)e);
   1079             break;
   1080           default:
   1081             type = NET_SERIAL_TYPES[(u8)e->type];
   1082         }
   1083         foundFirstType = true;
   1084       }
   1085     }
   1086   }
   1087   if (allElementsHaveSameType)
   1088     type = UNIFORM_ARRAY;
   1089   else
   1090     type = S_ARRAY;
   1091   ret type;
   1092 }
   1093 
   1094 /**
   1095  * serialize dictionary
   1096  *
   1097  * the serialized dict is pushed to r.
   1098  * All elements are serialized recursively
   1099  *
   1100  * the data in containers is not serialized
   1101  *
   1102  * \param
   1103  *   r small bytes object
   1104  *   dict dictionary to serialize
   1105  */
   1106 internal void dictNetSerialLevel2(sBytest **r, sDictt *dict, contextt *ctx, bool packed) {
   1107   sBytest *B     = NULL;
   1108   char    *data  = NULL;
   1109 
   1110   // check if all elements have same type
   1111   bool allElementsHaveSameType = true;
   1112   bool foundFirstType          = false;
   1113   char type                    = 0;
   1114   {forEachSDict(dict, e) {
   1115     if (e->key) {
   1116       if (foundFirstType) {
   1117         u8 nextType;
   1118         switch(e->data->type) {
   1119           case DICT:
   1120             nextType = isDictUniform((sDictt*)e->data);
   1121             break;
   1122           case ARRAY:
   1123             nextType = isArrayUniform((sArrayt*)e->data);
   1124             break;
   1125           default:
   1126             nextType = NET_SERIAL_TYPES[(u8)e->data->type];
   1127         }
   1128         if (nextType != type) {
   1129           allElementsHaveSameType = false;
   1130           break;
   1131         }
   1132       }
   1133       else {
   1134         switch(e->data->type) {
   1135           case DICT:
   1136             type = isDictUniform((sDictt*)e->data);
   1137             break;
   1138           case ARRAY:
   1139             type = isArrayUniform((sArrayt*)e->data);
   1140             break;
   1141           default:
   1142             type = NET_SERIAL_TYPES[(u8)e->data->type];
   1143         }
   1144         foundFirstType = true;
   1145       }
   1146     }
   1147   }}
   1148 
   1149   if (allElementsHaveSameType) {
   1150     // pack dictionary
   1151     if (packed) {
   1152       uintToNetTypeVarint(r, type, dict->count);
   1153     }
   1154     else {
   1155       if (ctx->nibble == lowNbl) {
   1156         sBytesPush(r, (type << 4) + UNIFORM_DICT);
   1157         uintToVarint(r, dict->count);
   1158       }
   1159       else {
   1160         // high nibble
   1161         ctx->nibble  = lowNbl;
   1162         data         = (char *)&((*r)->data) + ctx->nblOffset;
   1163         *data       |= UNIFORM_DICT << 4;
   1164         uintToNetTypeVarint(r, type, dict->count);
   1165       }
   1166     }
   1167 
   1168     switch(type) {
   1169       case S_UNDEFINED:
   1170         {forEachSDict(dict, e) {
   1171           if (e->key) {
   1172             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1173           }
   1174         }}
   1175 
   1176         break;
   1177       case S_BOOL:
   1178         {forEachSDict(dict, e) {
   1179           if (e->key) {
   1180             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1181 
   1182             if (!ctx->boolOffset) {
   1183               // new packed bools
   1184               storeNew8bPackedBool(e->data);
   1185             }
   1186             else {
   1187               // there was a bool before this one, fill bits in nibbles
   1188               if (ctx->boolShift == 8) {
   1189                 // previous packed bool is full
   1190                 // next byte is the new packed bools
   1191                 storeNew8bPackedBool(e->data);
   1192               }
   1193               else {
   1194                 storeBool(e->data);
   1195               }
   1196             }
   1197           }
   1198         }}
   1199         break;
   1200       case S_DICT:
   1201       case UNIFORM_DICT:
   1202         {forEachSDict(dict, e) {
   1203           if (e->key) {
   1204             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1205             dictNetSerialLevel2(r, (sDictt *)(e->data), ctx, /*packed=*/true);
   1206           }
   1207         }}
   1208         break;
   1209       case S_DOUBLE:
   1210         {forEachSDict(dict, e) {
   1211           if (e->key) {
   1212             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1213             sBytesPushBuffer(r, &((sDoublet *)(e->data))->value, sizeof(double));
   1214           }
   1215         }}
   1216         break;
   1217       case S_INT:
   1218         {forEachSDict(dict, e) {
   1219           if (e->key) {
   1220             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1221             i64 v = ((sIntt *)(e->data))->value;
   1222             uintToVarint(r, (v << 1) ^ (v >> 63));
   1223           }
   1224         }}
   1225         break;
   1226       case S_STRING:
   1227         {forEachSDict(dict, e) {
   1228           if (e->key) {
   1229             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1230             sBytesPushBuffer(r, &((sStringt *)(e->data))->data, sizeof(sStringt) + strlen(&(((sStringt *)(e->data))->data)) -1);
   1231           }
   1232         }}
   1233         break;
   1234       case S_ARRAY:
   1235       case UNIFORM_ARRAY:
   1236         {forEachSDict(dict, e) {
   1237           if (e->key) {
   1238             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1239             arrayNetSerialLevel2(r, (sArrayt *)(e->data), ctx, /*packed=*/true);
   1240           }
   1241         }}
   1242         break;
   1243       case S_BYTES:
   1244         {forEachSDict(dict, e) {
   1245           if (e->key) {
   1246             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1247             B = (sBytest *)(e->data);
   1248             uintToVarint(r, B->count);
   1249             sBytesPushBuffer(r, &(B->data), B->count);
   1250           }
   1251         }}
   1252         break;
   1253     }
   1254     ret;
   1255   }
   1256 
   1257   if (packed) {
   1258     uintToVarint(r, dict->count);
   1259   }
   1260   else {
   1261     if (ctx->nibble == lowNbl) {
   1262       uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)dict->type], dict->count);
   1263     }
   1264     else {
   1265       // high nibble
   1266 #define storeTypeInHighNbl(o)\
   1267       ctx->nibble  = lowNbl;\
   1268       data         = (char *)&((*r)->data) + ctx->nblOffset;\
   1269       *data       |= NET_SERIAL_TYPES[(u8)o->type] << 4
   1270       storeTypeInHighNbl(dict);
   1271       uintToVarint(r, dict->count);
   1272     }
   1273   }
   1274 
   1275   forEachSDict(dict, e) {
   1276     if (e->key) {
   1277       sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1278 
   1279       switch(e->data->type) {
   1280         case UNDEFINED:
   1281         case CONTAINER:
   1282           if (ctx->nibble == lowNbl) {
   1283             #define storeTypeOnly(o)\
   1284             sBytesPush(r, NET_SERIAL_TYPES[(u8)o->type]);\
   1285             ctx->nibble    = highNbl;\
   1286             ctx->nblOffset = (*r)->count -1
   1287             storeTypeOnly(e->data);
   1288           }
   1289           else {
   1290             storeTypeInHighNbl(e->data);
   1291           }
   1292           break;
   1293         case BOOL:
   1294           if (!ctx->boolOffset) {
   1295             // new packed bools
   1296             if (ctx->nibble == lowNbl) {
   1297               #define storeNew4bPackedBool(o)\
   1298               u8 c = NET_SERIAL_TYPES[(u8)o->type];\
   1299               /* set bit 4 when true */\
   1300               if (((sBoolt *)(o))->value)\
   1301                 c |= (1<<4);\
   1302               sBytesPush(r, c);\
   1303               ctx->boolShift  = 5;\
   1304               ctx->boolOffset = (*r)->count -1
   1305               storeNew4bPackedBool(e->data);
   1306             }
   1307             else {
   1308               // high nibble, next byte is packed bools
   1309               storeTypeInHighNbl(e->data);
   1310               #define storeNew8bPackedBool(o)\
   1311               u8 c         = 0;\
   1312               if (((sBoolt *)(o))->value)\
   1313                 c = 1;\
   1314               sBytesPush(r, c);\
   1315               ctx->boolShift  = 1;\
   1316               ctx->boolOffset = (*r)->count -1
   1317               storeNew8bPackedBool(e->data);
   1318             }
   1319           }
   1320           else {
   1321             // there was a bool before this one, fill bits in nibbles
   1322             if (ctx->nibble == lowNbl) {
   1323               if (ctx->boolShift == 8) {
   1324                 // previous packed bool is full
   1325                 // this byte is the new packed bools
   1326                 storeNew4bPackedBool(e->data);
   1327               }
   1328               else {
   1329                 storeTypeOnly(e->data);
   1330                 #define storeBool(o)\
   1331                 data           = (char *)&((*r)->data) + ctx->boolOffset;\
   1332                 if (((sBoolt *)(o))->value)\
   1333                   *data |= 1 << ctx->boolShift;\
   1334                 ctx->boolShift++
   1335                 storeBool(e->data);
   1336               }
   1337             }
   1338             else {
   1339               // high nibble
   1340               storeTypeInHighNbl(e->data);
   1341               if (ctx->boolShift == 8) {
   1342                 // previous packed bool is full
   1343                 // next byte is the new packed bools
   1344                 storeNew8bPackedBool(e->data);
   1345               }
   1346               else {
   1347                 storeBool(e->data);
   1348               }
   1349             }
   1350           }
   1351           break;
   1352         case DICT:
   1353           dictNetSerialLevel2(r, (sDictt *)(e->data), ctx, /*packed=*/false);
   1354           break;
   1355         case DOUBLE:
   1356           if (ctx->nibble == lowNbl) {
   1357             storeTypeOnly(e->data);
   1358           }
   1359           else {
   1360             // high nibble
   1361             storeTypeInHighNbl(e->data);
   1362           }
   1363           sBytesPushBuffer(r, &((sDoublet *)(e->data))->value, sizeof(double));
   1364           break;
   1365         case INT: {
   1366             // encode int to varint
   1367             // v is int64_t to convert to varint
   1368             i64 v = ((sIntt *)(e->data))->value;
   1369             if (ctx->nibble == lowNbl) {
   1370               // encode v with arithmetic shifts
   1371               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], (v << 1) ^ (v >> 63));
   1372             }
   1373             else {
   1374               // high nibble
   1375               storeTypeInHighNbl(e->data);
   1376               uintToVarint(r, (v << 1) ^ (v >> 63));
   1377             }
   1378           }
   1379           break;
   1380         case STRING:
   1381           if (ctx->nibble == lowNbl) {
   1382             storeTypeOnly(e->data);
   1383           }
   1384           else {
   1385             // high nibble
   1386             storeTypeInHighNbl(e->data);
   1387           }
   1388           sBytesPushBuffer(r, &((sStringt *)(e->data))->data, sizeof(sStringt) + strlen(&(((sStringt *)(e->data))->data)) -1);
   1389           break;
   1390         case ARRAY:
   1391           arrayNetSerialLevel2(r, (sArrayt *)(e->data), ctx, /*packed=*/false);
   1392           break;
   1393         case BYTES:
   1394           B = (sBytest *)(e->data);
   1395           if (ctx->nibble == lowNbl) {
   1396             uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], B->count);
   1397           }
   1398           else {
   1399             // high nibble
   1400             storeTypeInHighNbl(e->data);
   1401             uintToVarint(r, B->count);
   1402           }
   1403           sBytesPushBuffer(r, &(B->data), B->count);
   1404           break;
   1405       }
   1406     }
   1407   }
   1408   ret;
   1409 }
   1410 
   1411 /**
   1412  * serialize array
   1413  *
   1414  * the serialized array is pushed to r.
   1415  * All elements are serialized recursively
   1416  *
   1417  * the data in containers is not serialized
   1418  *
   1419  * \param
   1420  *   r small bytes object
   1421  *   array to serialize
   1422  */
   1423 internal void arrayNetSerialLevel2(sBytest **r, sArrayt *array, contextt *ctx, bool packed) {
   1424   sBytest *B     = NULL;
   1425   char    *data  = NULL;
   1426 
   1427   // check if all elements have same type
   1428   bool allElementsHaveSameType = true;
   1429   bool foundFirstType          = false;
   1430   char type                    = 0;
   1431 
   1432   {forEachSArray(array, e) {
   1433     if (!e) {
   1434       if (foundFirstType) {
   1435         if (type != S_UNDEFINED) {
   1436           allElementsHaveSameType = false;
   1437           break;
   1438         }
   1439       }
   1440       else {
   1441         type = S_UNDEFINED;
   1442         foundFirstType = true;
   1443       }
   1444     }
   1445     else {
   1446       if (foundFirstType) {
   1447         u8 nextType;
   1448         switch(e->type) {
   1449           case DICT:
   1450             nextType = isDictUniform((sDictt*)e);
   1451             break;
   1452           case ARRAY:
   1453             nextType = isArrayUniform((sArrayt*)e);
   1454             break;
   1455           default:
   1456             nextType = NET_SERIAL_TYPES[(u8)e->type];
   1457         }
   1458         if (nextType != type) {
   1459           allElementsHaveSameType = false;
   1460           break;
   1461         }
   1462       }
   1463       else {
   1464         switch(e->type) {
   1465           case DICT:
   1466             type = isDictUniform((sDictt*)e);
   1467             break;
   1468           case ARRAY:
   1469             type = isArrayUniform((sArrayt*)e);
   1470             break;
   1471           default:
   1472             type = NET_SERIAL_TYPES[(u8)e->type];
   1473         }
   1474         foundFirstType = true;
   1475       }
   1476     }
   1477   }}
   1478 
   1479   if (allElementsHaveSameType) {
   1480     // pack array
   1481     if (packed) {
   1482       uintToNetTypeVarint(r, type, array->count);
   1483     }
   1484     else {
   1485       if (ctx->nibble == lowNbl) {
   1486         sBytesPush(r, (type << 4) + UNIFORM_ARRAY);
   1487         uintToVarint(r, array->count);
   1488       }
   1489       else {
   1490         // high nibble
   1491         ctx->nibble  = lowNbl;
   1492         data         = (char *)&((*r)->data) + ctx->nblOffset;
   1493         *data       |= UNIFORM_ARRAY << 4;
   1494         uintToNetTypeVarint(r, type, array->count);
   1495       }
   1496     }
   1497 
   1498     switch(type) {
   1499       case S_BOOL:
   1500         {forEachSArray(array, e) {
   1501           if (!ctx->boolOffset) {
   1502             // new packed bools
   1503             storeNew8bPackedBool(e);
   1504           }
   1505           else {
   1506             // there was a bool before this one, fill bits in nibbles
   1507             if (ctx->boolShift == 8) {
   1508               // previous packed bool is full
   1509               // next byte is the new packed bools
   1510               storeNew8bPackedBool(e);
   1511             }
   1512             else {
   1513               storeBool(e);
   1514             }
   1515           }
   1516         }}
   1517         break;
   1518       case S_DICT:
   1519       case UNIFORM_DICT:
   1520         {forEachSArray(array, e) {
   1521           dictNetSerialLevel2(r, (sDictt *)e, ctx, /*packed=*/true);
   1522         }}
   1523         break;
   1524       case S_DOUBLE:
   1525         {forEachSArray(array, e) {
   1526           sBytesPushBuffer(r, &((sDoublet *)e)->value, sizeof(double));
   1527         }}
   1528         break;
   1529       case S_INT:
   1530         {forEachSArray(array, e) {
   1531           i64 v = ((sIntt *)e)->value;
   1532           uintToVarint(r, (v << 1) ^ (v >> 63));
   1533         }}
   1534         break;
   1535       case S_STRING:
   1536         {forEachSArray(array, e) {
   1537           sBytesPushBuffer(r, &((sStringt *)e)->data, sizeof(sStringt) + strlen(&(((sStringt *)e)->data)) -1);
   1538         }}
   1539         break;
   1540       case S_ARRAY:
   1541       case UNIFORM_ARRAY:
   1542         {forEachSArray(array, e) {
   1543           arrayNetSerialLevel2(r, (sArrayt *)e, ctx, /*packed=*/true);
   1544         }}
   1545         break;
   1546       case S_BYTES:
   1547         {forEachSArray(array, e) {
   1548           B = (sBytest *)e;
   1549           uintToVarint(r, B->count);
   1550           sBytesPushBuffer(r, &(B->data), B->count);
   1551         }}
   1552         break;
   1553     }
   1554     ret;
   1555   }
   1556 
   1557   if (packed) {
   1558     uintToVarint(r, array->count);
   1559   }
   1560   else {
   1561     if (ctx->nibble == lowNbl) {
   1562       uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)array->type], array->count);
   1563     }
   1564     else {
   1565       // high nibble
   1566       storeTypeInHighNbl(array);
   1567       uintToVarint(r, array->count);
   1568     }
   1569   }
   1570 
   1571   forEachSArray(array, e) {
   1572     if (!e) {
   1573       // empty slots are represented as undefined elements
   1574       if (ctx->nibble == lowNbl) {
   1575         sBytesPush(r, NET_SERIAL_TYPES[UNDEFINED]);
   1576         ctx->nibble    = highNbl;
   1577         ctx->nblOffset = (*r)->count -1;
   1578       }
   1579       else {
   1580         // high nibble
   1581         ctx->nibble  = lowNbl;
   1582         data         = (char *)&((*r)->data) + ctx->nblOffset;
   1583         *data       |= NET_SERIAL_TYPES[UNDEFINED] << 4;
   1584       }
   1585     }
   1586     else {
   1587       switch(e->type) {
   1588         case UNDEFINED:
   1589         case CONTAINER:
   1590           if (ctx->nibble == lowNbl) {
   1591             storeTypeOnly(e);
   1592           }
   1593           else {
   1594             // high nibble
   1595             storeTypeInHighNbl(e);
   1596           }
   1597           break;
   1598         case BOOL:
   1599           if (!ctx->boolOffset) {
   1600             // new packed bools
   1601             if (ctx->nibble == lowNbl) {
   1602               storeNew4bPackedBool(e);
   1603             }
   1604             else {
   1605               // high nibble, next byte is packed bools
   1606               storeTypeInHighNbl(e);
   1607               storeNew8bPackedBool(e);
   1608             }
   1609           }
   1610           else {
   1611             // there was a bool before this one, fill bits in nibbles
   1612             if (ctx->nibble == lowNbl) {
   1613               if (ctx->boolShift == 8) {
   1614                 // previous packed bool is full
   1615                 // this byte is the new packed bools
   1616                 storeNew4bPackedBool(e);
   1617               }
   1618               else {
   1619                 storeTypeOnly(e);
   1620                 storeBool(e);
   1621               }
   1622             }
   1623             else {
   1624               // high nibble
   1625               storeTypeInHighNbl(e);
   1626               if (ctx->boolShift == 8) {
   1627                 // previous packed bool is full
   1628                 // next byte is the new packed bools
   1629                 storeNew8bPackedBool(e);
   1630               }
   1631               else {
   1632                 storeBool(e);
   1633               }
   1634             }
   1635           }
   1636           break;
   1637         case DICT:
   1638           dictNetSerialLevel2(r, (sDictt *)e, ctx, /*packed=*/false);
   1639           break;
   1640         case DOUBLE:
   1641           if (ctx->nibble == lowNbl) {
   1642             storeTypeOnly(e);
   1643           }
   1644           else {
   1645             // high nibble
   1646             storeTypeInHighNbl(e);
   1647           }
   1648           sBytesPushBuffer(r, &((sDoublet *)e)->value, sizeof(double));
   1649           break;
   1650         case INT: {
   1651             // encode int to varint
   1652             // v is int64_t to convert to varint
   1653             i64 v = ((sIntt *)&(e->type))->value;
   1654             if (ctx->nibble == lowNbl) {
   1655               // encode v with arithmetic shifts
   1656               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], (v << 1) ^ (v >> 63));
   1657             }
   1658             else {
   1659               // high nibble
   1660               storeTypeInHighNbl(e);
   1661               uintToVarint(r, (v << 1) ^ (v >> 63));
   1662             }
   1663           }
   1664           break;
   1665         case STRING:
   1666           if (ctx->nibble == lowNbl) {
   1667             storeTypeOnly(e);
   1668           }
   1669           else {
   1670             // high nibble
   1671             storeTypeInHighNbl(e);
   1672           }
   1673           sBytesPushBuffer(r, &((sStringt *)e)->data, sizeof(sStringt) + strlen(&(((sStringt *)e)->data)) -1);
   1674           break;
   1675         case ARRAY:
   1676           arrayNetSerialLevel2(r, (sArrayt *)e, ctx, /*packed=*/false);
   1677           break;
   1678         case BYTES:
   1679           B = (sBytest *)e;
   1680           if (ctx->nibble == lowNbl) {
   1681             uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], B->count);
   1682           }
   1683           else {
   1684             // high nibble
   1685             storeTypeInHighNbl(e);
   1686             uintToVarint(r, B->count);
   1687           }
   1688           sBytesPushBuffer(r, &(B->data), B->count);
   1689           break;
   1690       }
   1691     }
   1692   }
   1693   ret;
   1694 }
   1695 
   1696 internal smallBytest* serialNetSerialLevel2(smallJsont *self) {
   1697 
   1698   smallt *o = getsoG(self);
   1699 
   1700   if (o == NULL)
   1701     ret NULL;
   1702 
   1703   sBytest *B = netSerialLevel2(o);
   1704 
   1705   if (!B) {
   1706     ret NULL;
   1707   }
   1708 
   1709   createAllocateSmallBytes(r);
   1710   r->B = B;
   1711   ret r;
   1712 }
   1713 
   1714 // level 3
   1715 // like level 2, elements of identical type in a row are packed
   1716 
   1717 /**
   1718  * serializer top function
   1719  */
   1720 internal sBytest* netSerial(smallt *o) {
   1721   sBytest *r = NULL;
   1722   sBytest *B = NULL;
   1723   contextt ctx = {.nibble=lowNbl, .nblOffset=0, .boolShift = 0, .boolOffset=0};
   1724 
   1725   switch(o->type) {
   1726     case UNDEFINED:
   1727     case CONTAINER:
   1728       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
   1729       break;
   1730     case BOOL: {
   1731       u8 c = NET_SERIAL_TYPES[(u8)o->type];
   1732       // set bit 4 when true
   1733       if (((sBoolt *)&(o->type))->value)
   1734         c |= (1<<4);
   1735       sBytesPush(&r, c);
   1736       }
   1737       break;
   1738     case DICT:
   1739       dictNetSerial(&r, (sDictt *)&(o->type), &ctx, /*packing=*/NOPACKING);
   1740       break;
   1741     case DOUBLE:
   1742       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
   1743       sBytesPushBuffer(&r, &((sDoublet *)&(o->type))->value, sizeof(double));
   1744       break;
   1745     case INT: {
   1746       // encode int to varint
   1747       // v is int64_t to convert to varint
   1748       i64 v = ((sIntt *)&(o->type))->value;
   1749       // encode v with arithmetic shifts
   1750       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], (v << 1) ^ (v >> 63));
   1751       }
   1752       break;
   1753     case STRING:
   1754       sBytesPush(&r, NET_SERIAL_TYPES[(u8)o->type]);
   1755       sBytesPushBuffer(&r, &((sStringt *)&(o->type))->data, sizeof(sStringt) + strlen(&(((sStringt *)&(o->type))->data)) -1);
   1756       break;
   1757     case ARRAY:
   1758       arrayNetSerial(&r, (sArrayt *)&(o->type), &ctx, /*packing=*/NOPACKING);
   1759       break;
   1760     case BYTES:
   1761       B = (sBytest *)&(o->type);
   1762       uintToNetTypeVarint(&r, NET_SERIAL_TYPES[(u8)o->type], B->count);
   1763       sBytesPushBuffer(&r, &(B->data), B->count);
   1764       break;
   1765   }
   1766   ret r;
   1767 }
   1768 
   1769 /**
   1770  * serialize dictionary
   1771  *
   1772  * the serialized dict is pushed to r.
   1773  * All elements are serialized recursively
   1774  *
   1775  * the data in containers is not serialized
   1776  *
   1777  * \param
   1778  *   r small bytes object
   1779  *   dict dictionary to serialize
   1780  */
   1781 internal void dictNetSerial(sBytest **r, sDictt *dict, contextt *ctx, packingT packing) {
   1782   sBytest *B     = NULL;
   1783   char    *data  = NULL;
   1784 
   1785   // compute allocated element count and use that instead of dict->count
   1786   // when elements are deleted, some positions are NULL
   1787   // and should not be counted
   1788   u32 cnt =0;
   1789   {forEachSDict(dict, e) {
   1790     if (e->key) {
   1791       inc cnt;
   1792     }
   1793   }}
   1794 
   1795   // check if all elements have same type
   1796   // then set dict type normal or uniform
   1797   // array and dict have to be checked recursively to know if they are normal or uniform
   1798   bool allElementsHaveSameType = true;
   1799   bool foundFirstType          = false;
   1800   char type                    = 0;
   1801   // get first element type
   1802   // compare to other element type
   1803   {forEachSDict(dict, e) {
   1804     if (e->key) {
   1805       if (foundFirstType) {
   1806         u8 nextType;
   1807         switch(e->data->type) {
   1808           case DICT:
   1809             nextType = isDictUniform((sDictt*)e->data);
   1810             break;
   1811           case ARRAY:
   1812             nextType = isArrayUniform((sArrayt*)e->data);
   1813             break;
   1814           default:
   1815             nextType = NET_SERIAL_TYPES[(u8)e->data->type];
   1816         }
   1817         if (nextType != type) {
   1818           allElementsHaveSameType = false;
   1819           break;
   1820         }
   1821       }
   1822       else {
   1823         switch(e->data->type) {
   1824           case DICT:
   1825             type = isDictUniform((sDictt*)e->data);
   1826             break;
   1827           case ARRAY:
   1828             type = isArrayUniform((sArrayt*)e->data);
   1829             break;
   1830           default:
   1831             type = NET_SERIAL_TYPES[(u8)e->data->type];
   1832         }
   1833         foundFirstType = true;
   1834       }
   1835     }
   1836   }}
   1837 
   1838   if (allElementsHaveSameType) {
   1839     // uniform dict
   1840     // encode type and element count
   1841 
   1842     // in pack dictionary
   1843     if (packing == PACK) {
   1844       // uniform dict can't be packed
   1845       // because there is only one type of packed arrays
   1846       goto normalDict;
   1847     }
   1848     elif (packing == UNIFORM) {
   1849       // when the packing is uniform, there is no need to encode UNIFORM_DICT since all elements have this type
   1850       uintToNetTypeVarint(r, type, cnt);
   1851     }
   1852     else {
   1853       if (ctx->nibble == lowNbl) {
   1854         sBytesPush(r, (type << 4) + UNIFORM_DICT);
   1855         uintToVarint(r, cnt);
   1856       }
   1857       else {
   1858         // high nibble
   1859         ctx->nibble  = lowNbl;
   1860         data         = (char *)&((*r)->data) + ctx->nblOffset;
   1861         *data       |= UNIFORM_DICT << 4;
   1862         uintToNetTypeVarint(r, type, cnt);
   1863       }
   1864     }
   1865 
   1866     // encode all element keys and values
   1867     switch(type) {
   1868       case S_UNDEFINED:
   1869         {forEachSDict(dict, e) {
   1870           if (e->key) {
   1871             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1872           }
   1873         }}
   1874 
   1875         break;
   1876       case S_BOOL:
   1877         {forEachSDict(dict, e) {
   1878           if (e->key) {
   1879             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1880 
   1881             if (!ctx->boolOffset) {
   1882               // new packed bools
   1883               storeNew8bPackedBool(e->data);
   1884             }
   1885             else {
   1886               // there was a bool before this one, fill bits in nibbles
   1887               if (ctx->boolShift == 8) {
   1888                 // previous packed bool is full
   1889                 // next byte is the new packed bools
   1890                 storeNew8bPackedBool(e->data);
   1891               }
   1892               else {
   1893                 storeBool(e->data);
   1894               }
   1895             }
   1896           }
   1897         }}
   1898         break;
   1899       case S_DICT:
   1900       case UNIFORM_DICT:
   1901         {forEachSDict(dict, e) {
   1902           if (e->key) {
   1903             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1904             dictNetSerial(r, (sDictt *)(e->data), ctx, /*packing=*/UNIFORM);
   1905           }
   1906         }}
   1907         break;
   1908       case S_DOUBLE:
   1909         {forEachSDict(dict, e) {
   1910           if (e->key) {
   1911             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1912             sBytesPushBuffer(r, &((sDoublet *)(e->data))->value, sizeof(double));
   1913           }
   1914         }}
   1915         break;
   1916       case S_INT:
   1917         {forEachSDict(dict, e) {
   1918           if (e->key) {
   1919             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1920             i64 v = ((sIntt *)(e->data))->value;
   1921             uintToVarint(r, (v << 1) ^ (v >> 63));
   1922           }
   1923         }}
   1924         break;
   1925       case S_STRING:
   1926         {forEachSDict(dict, e) {
   1927           if (e->key) {
   1928             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1929             sBytesPushBuffer(r, &((sStringt *)(e->data))->data, sizeof(sStringt) + strlen(&(((sStringt *)(e->data))->data)) -1);
   1930           }
   1931         }}
   1932         break;
   1933       case S_ARRAY:
   1934       case UNIFORM_ARRAY:
   1935         {forEachSDict(dict, e) {
   1936           if (e->key) {
   1937             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1938             arrayNetSerial(r, (sArrayt *)(e->data), ctx, /*packing=*/UNIFORM);
   1939           }
   1940         }}
   1941         break;
   1942       case S_BYTES:
   1943         {forEachSDict(dict, e) {
   1944           if (e->key) {
   1945             sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   1946             B = (sBytest *)(e->data);
   1947             uintToVarint(r, B->count);
   1948             sBytesPushBuffer(r, &(B->data), B->count);
   1949           }
   1950         }}
   1951         break;
   1952     }
   1953     ret;
   1954   }
   1955 
   1956   normalDict:
   1957   // encode type and element count
   1958   if (packing == PACK or packing == UNIFORM) {
   1959     // when the packing is packed or uniform, there is no need to encode DICT type since all elements have this type
   1960     uintToVarint(r, cnt);
   1961   }
   1962   else {
   1963     if (ctx->nibble == lowNbl) {
   1964       uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)dict->type], cnt);
   1965     }
   1966     else {
   1967       // high nibble
   1968       #define storeTypeInHighNbl(o)\
   1969       ctx->nibble  = lowNbl;\
   1970       data         = (char *)&((*r)->data) + ctx->nblOffset;\
   1971       *data       |= NET_SERIAL_TYPES[(u8)o->type] << 4
   1972       storeTypeInHighNbl(dict);
   1973       uintToVarint(r, cnt);
   1974     }
   1975   }
   1976 
   1977   bool   pack = false;
   1978   size_t packCount;
   1979   enumerateSDict(dict, e, eIdx) {
   1980     if (e->key) {
   1981       if (!pack) {
   1982         // scan dict for packing
   1983         if ((cnt - eIdx) > 3) {
   1984           // at least 4 elements, less than that is not worth it
   1985           if (   e->data->type == DICT
   1986               or e->data->type == DOUBLE
   1987               or e->data->type == INT
   1988               or e->data->type == STRING
   1989               or e->data->type == ARRAY
   1990               or e->data->type == BYTES) {
   1991             type      = e->data->type;
   1992             packCount = 1;
   1993             sDictElemt *element = &((dict)->elements) + eIdx +1;
   1994             for (size_t i = eIdx+1; i < (dict)->count ; i++, element = &((dict)->elements) + i) {
   1995               if (element->key) {
   1996                 if (element->data->type != type) {
   1997                   break;
   1998                 }
   1999                 packCount++;
   2000               } // element->key
   2001             } // for
   2002             if (packCount > 3) {
   2003               type = PACKED_NET_SERIAL_TYPES[(u8)type];
   2004               pack = true;
   2005             }
   2006           } // test current element type
   2007         } // is dict big enough
   2008       } // not already packing
   2009 
   2010       // encode key
   2011       sBytesPushBuffer(r, e->key, strlen(e->key) + 1);
   2012 
   2013       // encode value
   2014       switch(e->data->type) {
   2015         case UNDEFINED:
   2016         case CONTAINER:
   2017           if (ctx->nibble == lowNbl) {
   2018             #define storeTypeOnly(o)\
   2019             sBytesPush(r, NET_SERIAL_TYPES[(u8)o->type]);\
   2020             ctx->nibble    = highNbl;\
   2021             ctx->nblOffset = (*r)->count -1
   2022             storeTypeOnly(e->data);
   2023           }
   2024           else {
   2025             storeTypeInHighNbl(e->data);
   2026           }
   2027           break;
   2028         case BOOL:
   2029           if (!ctx->boolOffset) {
   2030             // new packed bools
   2031             if (ctx->nibble == lowNbl) {
   2032               #define storeNew4bPackedBool(o)\
   2033               u8 c = NET_SERIAL_TYPES[(u8)o->type];\
   2034               /* set bit 4 when true */\
   2035               if (((sBoolt *)(o))->value)\
   2036                 c |= (1<<4);\
   2037               sBytesPush(r, c);\
   2038               ctx->boolShift  = 5;\
   2039               ctx->boolOffset = (*r)->count -1
   2040               storeNew4bPackedBool(e->data);
   2041             }
   2042             else {
   2043               // high nibble, next byte is packed bools
   2044               storeTypeInHighNbl(e->data);
   2045               #define storeNew8bPackedBool(o)\
   2046               u8 c         = 0;\
   2047               if (((sBoolt *)(o))->value)\
   2048                 c = 1;\
   2049               sBytesPush(r, c);\
   2050               ctx->boolShift  = 1;\
   2051               ctx->boolOffset = (*r)->count -1
   2052               storeNew8bPackedBool(e->data);
   2053             }
   2054           }
   2055           else {
   2056             // there was a bool before this one, fill bits in nibbles
   2057             if (ctx->nibble == lowNbl) {
   2058               if (ctx->boolShift == 8) {
   2059                 // previous packed bool is full
   2060                 // this byte is the new packed bools
   2061                 storeNew4bPackedBool(e->data);
   2062               }
   2063               else {
   2064                 storeTypeOnly(e->data);
   2065                 #define storeBool(o)\
   2066                 data           = (char *)&((*r)->data) + ctx->boolOffset;\
   2067                 if (((sBoolt *)(o))->value)\
   2068                   *data |= 1 << ctx->boolShift;\
   2069                 ctx->boolShift++
   2070                 storeBool(e->data);
   2071               }
   2072             }
   2073             else {
   2074               // high nibble
   2075               storeTypeInHighNbl(e->data);
   2076               if (ctx->boolShift == 8) {
   2077                 // previous packed bool is full
   2078                 // next byte is the new packed bools
   2079                 storeNew8bPackedBool(e->data);
   2080               }
   2081               else {
   2082                 storeBool(e->data);
   2083               }
   2084             }
   2085           }
   2086           break;
   2087         case DICT:
   2088           if (pack) {
   2089             if (type) {
   2090               // this is the first packed element
   2091               if (ctx->nibble == lowNbl) {
   2092                 uintToNetTypeVarint(r, type, packCount);
   2093               }
   2094               else {
   2095                 // high nibble
   2096                 // store type in high nibble
   2097                 ctx->nibble  = lowNbl;
   2098                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2099                 *data       |= type << 4;
   2100                 uintToVarint(r, packCount);
   2101               }
   2102               type = 0;
   2103             } // if type
   2104 
   2105             dictNetSerial(r, (sDictt *)(e->data), ctx, /*packing=*/PACK);
   2106             // stop packing when packCount == 0
   2107             packCount--;
   2108             if (!packCount) pack = false;
   2109           } // if pack
   2110           else
   2111             dictNetSerial(r, (sDictt *)(e->data), ctx, /*packing=*/NOPACKING);
   2112           break;
   2113         case DOUBLE:
   2114           if (pack) {
   2115             if (type) {
   2116               // this is the first packed element
   2117               if (ctx->nibble == lowNbl) {
   2118                 uintToNetTypeVarint(r, type, packCount);
   2119               }
   2120               else {
   2121                 // high nibble
   2122                 // store type in high nibble
   2123                 ctx->nibble  = lowNbl;
   2124                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2125                 *data       |= type << 4;
   2126                 uintToVarint(r, packCount);
   2127               }
   2128               type = 0;
   2129             } // if type
   2130 
   2131             sBytesPushBuffer(r, &((sDoublet *)(e->data))->value, sizeof(double));
   2132             // stop packing when packCount == 0
   2133             packCount--;
   2134             if (!packCount) pack = false;
   2135           } // if pack
   2136           else {
   2137             if (ctx->nibble == lowNbl) {
   2138               storeTypeOnly(e->data);
   2139             }
   2140             else {
   2141               // high nibble
   2142               storeTypeInHighNbl(e->data);
   2143             }
   2144             sBytesPushBuffer(r, &((sDoublet *)(e->data))->value, sizeof(double));
   2145           }
   2146           break;
   2147         case INT:
   2148           if (pack) {
   2149             if (type) {
   2150               // this is the first packed element
   2151               if (ctx->nibble == lowNbl) {
   2152                 uintToNetTypeVarint(r, type, packCount);
   2153               }
   2154               else {
   2155                 // high nibble
   2156                 // store type in high nibble
   2157                 ctx->nibble  = lowNbl;
   2158                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2159                 *data       |= type << 4;
   2160                 uintToVarint(r, packCount);
   2161               }
   2162               type = 0;
   2163             } // if type
   2164 
   2165             i64 v = ((sIntt *)(e->data))->value;
   2166             uintToVarint(r, (v << 1) ^ (v >> 63));
   2167             // stop packing when packCount == 0
   2168             packCount--;
   2169             if (!packCount) pack = false;
   2170           } // if pack
   2171           else {
   2172             // encode int to varint
   2173             // v is int64_t to convert to varint
   2174             i64 v = ((sIntt *)(e->data))->value;
   2175             if (ctx->nibble == lowNbl) {
   2176               // encode v with arithmetic shifts
   2177               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], (v << 1) ^ (v >> 63));
   2178             }
   2179             else {
   2180               // high nibble
   2181               storeTypeInHighNbl(e->data);
   2182               uintToVarint(r, (v << 1) ^ (v >> 63));
   2183             }
   2184           }
   2185           break;
   2186         case STRING:
   2187           if (pack) {
   2188             if (type) {
   2189               // this is the first packed element
   2190               if (ctx->nibble == lowNbl) {
   2191                 uintToNetTypeVarint(r, type, packCount);
   2192               }
   2193               else {
   2194                 // high nibble
   2195                 // store type in high nibble
   2196                 ctx->nibble  = lowNbl;
   2197                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2198                 *data       |= type << 4;
   2199                 uintToVarint(r, packCount);
   2200               }
   2201               type = 0;
   2202             } // if type
   2203 
   2204             sBytesPushBuffer(r, &((sStringt *)(e->data))->data, sizeof(sStringt) + strlen(&(((sStringt *)(e->data))->data)) -1);
   2205             // stop packing when packCount == 0
   2206             packCount--;
   2207             if (!packCount) pack = false;
   2208           } // if pack
   2209           else {
   2210             if (ctx->nibble == lowNbl) {
   2211               storeTypeOnly(e->data);
   2212             }
   2213             else {
   2214               // high nibble
   2215               storeTypeInHighNbl(e->data);
   2216             }
   2217             sBytesPushBuffer(r, &((sStringt *)(e->data))->data, sizeof(sStringt) + strlen(&(((sStringt *)(e->data))->data)) -1);
   2218           }
   2219           break;
   2220         case ARRAY:
   2221           if (pack) {
   2222             if (type) {
   2223               // this is the first packed element
   2224               if (ctx->nibble == lowNbl) {
   2225                 uintToNetTypeVarint(r, type, packCount);
   2226               }
   2227               else {
   2228                 // high nibble
   2229                 // store type in high nibble
   2230                 ctx->nibble  = lowNbl;
   2231                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2232                 *data       |= type << 4;
   2233                 uintToVarint(r, packCount);
   2234               }
   2235               type = 0;
   2236             } // if type
   2237 
   2238             arrayNetSerial(r, (sArrayt *)(e->data), ctx, /*packing=*/PACK);
   2239             // stop packing when packCount == 0
   2240             packCount--;
   2241             if (!packCount) pack = false;
   2242           } // if pack
   2243           else
   2244             arrayNetSerial(r, (sArrayt *)(e->data), ctx, /*packing=*/NOPACKING);
   2245           break;
   2246         case BYTES:
   2247           if (pack) {
   2248             if (type) {
   2249               // this is the first packed element
   2250               if (ctx->nibble == lowNbl) {
   2251                 uintToNetTypeVarint(r, type, packCount);
   2252               }
   2253               else {
   2254                 // high nibble
   2255                 // store type in high nibble
   2256                 ctx->nibble  = lowNbl;
   2257                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2258                 *data       |= type << 4;
   2259                 uintToVarint(r, packCount);
   2260               }
   2261               type = 0;
   2262             } // if type
   2263 
   2264             B = (sBytest *)(e->data);
   2265             uintToVarint(r, B->count);
   2266             sBytesPushBuffer(r, &(B->data), B->count);
   2267             // stop packing when packCount == 0
   2268             packCount--;
   2269             if (!packCount) pack = false;
   2270           } // if pack
   2271           else {
   2272             B = (sBytest *)(e->data);
   2273             if (ctx->nibble == lowNbl) {
   2274               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->data->type], B->count);
   2275             }
   2276             else {
   2277               // high nibble
   2278               storeTypeInHighNbl(e->data);
   2279               uintToVarint(r, B->count);
   2280             }
   2281             sBytesPushBuffer(r, &(B->data), B->count);
   2282           }
   2283           break;
   2284       }
   2285     }
   2286   }
   2287   ret;
   2288 }
   2289 
   2290 /**
   2291  * serialize array
   2292  *
   2293  * the serialized array is pushed to r.
   2294  * All elements are serialized recursively
   2295  *
   2296  * the data in containers is not serialized
   2297  *
   2298  * \param
   2299  *   r small bytes object
   2300  *   array to serialize
   2301  */
   2302 internal void arrayNetSerial(sBytest **r, sArrayt *array, contextt *ctx, packingT packing) {
   2303   sBytest *B     = NULL;
   2304   char    *data  = NULL;
   2305 
   2306   // check if all elements have same type
   2307   // then set array type normal or uniform
   2308   // array and dict have to be checked recursively to know if they are normal or uniform
   2309   bool allElementsHaveSameType = true;
   2310   bool foundFirstType          = false;
   2311   char type                    = 0;
   2312 
   2313   // get first element type
   2314   // compare to other element type
   2315   // null element are interpreted as undefined
   2316   {forEachSArray(array, e) {
   2317     if (!e) {
   2318       if (foundFirstType) {
   2319         if (type != S_UNDEFINED) {
   2320           allElementsHaveSameType = false;
   2321           break;
   2322         }
   2323       }
   2324       else {
   2325         type = S_UNDEFINED;
   2326         foundFirstType = true;
   2327       }
   2328     }
   2329     else {
   2330       if (foundFirstType) {
   2331         u8 nextType;
   2332         switch(e->type) {
   2333           case DICT:
   2334             nextType = isDictUniform((sDictt*)e);
   2335             break;
   2336           case ARRAY:
   2337             nextType = isArrayUniform((sArrayt*)e);
   2338             break;
   2339           default:
   2340             nextType = NET_SERIAL_TYPES[(u8)e->type];
   2341         }
   2342         if (nextType != type) {
   2343           allElementsHaveSameType = false;
   2344           break;
   2345         }
   2346       }
   2347       else {
   2348         switch(e->type) {
   2349           case DICT:
   2350             type = isDictUniform((sDictt*)e);
   2351             break;
   2352           case ARRAY:
   2353             type = isArrayUniform((sArrayt*)e);
   2354             break;
   2355           default:
   2356             type = NET_SERIAL_TYPES[(u8)e->type];
   2357         }
   2358         foundFirstType = true;
   2359       }
   2360     }
   2361   }}
   2362 
   2363   if (allElementsHaveSameType) {
   2364     // uniform array
   2365     // encode type and element count
   2366 
   2367     // in pack array
   2368     if (packing == PACK) {
   2369       // uniform array can't be packed
   2370       // because there is only one type of packed arrays
   2371       goto normalArray;
   2372     }
   2373     elif (packing == UNIFORM) {
   2374       // when the packing is uniform, there is no need to encode UNIFORM_ARRAY since all elements have this type
   2375       uintToNetTypeVarint(r, type, array->count);
   2376     }
   2377     else {
   2378       if (ctx->nibble == lowNbl) {
   2379         sBytesPush(r, (type << 4) + UNIFORM_ARRAY);
   2380         uintToVarint(r, array->count);
   2381       }
   2382       else {
   2383         // high nibble
   2384         ctx->nibble  = lowNbl;
   2385         data         = (char *)&((*r)->data) + ctx->nblOffset;
   2386         *data       |= UNIFORM_ARRAY << 4;
   2387         uintToNetTypeVarint(r, type, array->count);
   2388       }
   2389     }
   2390 
   2391     // encode all element values
   2392     switch(type) {
   2393       case S_BOOL:
   2394         {forEachSArray(array, e) {
   2395           if (!ctx->boolOffset) {
   2396             // new packed bools
   2397             storeNew8bPackedBool(e);
   2398           }
   2399           else {
   2400             // there was a bool before this one, fill bits in nibbles
   2401             if (ctx->boolShift == 8) {
   2402               // previous packed bool is full
   2403               // next byte is the new packed bools
   2404               storeNew8bPackedBool(e);
   2405             }
   2406             else {
   2407               storeBool(e);
   2408             }
   2409           }
   2410         }}
   2411         break;
   2412       case S_DICT:
   2413       case UNIFORM_DICT:
   2414         {forEachSArray(array, e) {
   2415           dictNetSerial(r, (sDictt *)e, ctx, /*packing=*/UNIFORM);
   2416         }}
   2417         break;
   2418       case S_DOUBLE:
   2419         {forEachSArray(array, e) {
   2420           sBytesPushBuffer(r, &((sDoublet *)e)->value, sizeof(double));
   2421         }}
   2422         break;
   2423       case S_INT:
   2424         {forEachSArray(array, e) {
   2425           i64 v = ((sIntt *)e)->value;
   2426           uintToVarint(r, (v << 1) ^ (v >> 63));
   2427         }}
   2428         break;
   2429       case S_STRING:
   2430         {forEachSArray(array, e) {
   2431           sBytesPushBuffer(r, &((sStringt *)e)->data, sizeof(sStringt) + strlen(&(((sStringt *)e)->data)) -1);
   2432         }}
   2433         break;
   2434       case S_ARRAY:
   2435       case UNIFORM_ARRAY:
   2436         {forEachSArray(array, e) {
   2437           arrayNetSerial(r, (sArrayt *)e, ctx, /*packing=*/UNIFORM);
   2438         }}
   2439         break;
   2440       case S_BYTES:
   2441         {forEachSArray(array, e) {
   2442           B = (sBytest *)e;
   2443           uintToVarint(r, B->count);
   2444           sBytesPushBuffer(r, &(B->data), B->count);
   2445         }}
   2446         break;
   2447     }
   2448     ret;
   2449   }
   2450 
   2451   normalArray:
   2452   // encode type and element count
   2453   if (packing == PACK or packing == UNIFORM) {
   2454     // when the packing is packed or uniform, there is no need to encode ARRAY type since all elements have this type
   2455     uintToVarint(r, array->count);
   2456   }
   2457   else {
   2458     if (ctx->nibble == lowNbl) {
   2459       uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)array->type], array->count);
   2460     }
   2461     else {
   2462       // high nibble
   2463       storeTypeInHighNbl(array);
   2464       uintToVarint(r, array->count);
   2465     }
   2466   }
   2467 
   2468   bool   pack = false;
   2469   size_t packCount;
   2470   enumerateSArray(array, e, eIdx) {
   2471     if (!e) {
   2472       // empty slots are represented as undefined elements
   2473       if (ctx->nibble == lowNbl) {
   2474         sBytesPush(r, NET_SERIAL_TYPES[UNDEFINED]);
   2475         ctx->nibble    = highNbl;
   2476         ctx->nblOffset = (*r)->count -1;
   2477       }
   2478       else {
   2479         // high nibble
   2480         ctx->nibble  = lowNbl;
   2481         data         = (char *)&((*r)->data) + ctx->nblOffset;
   2482         *data       |= NET_SERIAL_TYPES[UNDEFINED] << 4;
   2483       }
   2484     }
   2485     else {
   2486       if (!pack) {
   2487         // scan array for packing
   2488         if ((array->count - eIdx) > 3) {
   2489           // at least 4 elements, less than that is not worth it
   2490           if (   e->type == DICT
   2491               or e->type == DOUBLE
   2492               or e->type == INT
   2493               or e->type == STRING
   2494               or e->type == ARRAY
   2495               or e->type == BYTES) {
   2496             type      = e->type;
   2497             packCount = 1;
   2498             smallt *element = ((smallt **) &((array)->data))[eIdx+1];
   2499             for (size_t i = eIdx+1; i < (array)->count ; i++, element = ((smallt **) &((array)->data))[i]) {
   2500               if (!element) {
   2501                 // null element are undefined
   2502                 break;
   2503               }
   2504               else {
   2505                 if (element->type != type) {
   2506                   break;
   2507                 }
   2508                 packCount++;
   2509               } // if element
   2510             } // for
   2511             if (packCount > 3) {
   2512               type = PACKED_NET_SERIAL_TYPES[(u8)type];
   2513               pack = true;
   2514             }
   2515           } // test current element type
   2516         } // is array big enough
   2517       } // not already packing
   2518 
   2519       // encode element value
   2520       switch(e->type) {
   2521         case UNDEFINED:
   2522         case CONTAINER:
   2523           if (ctx->nibble == lowNbl) {
   2524             storeTypeOnly(e);
   2525           }
   2526           else {
   2527             // high nibble
   2528             storeTypeInHighNbl(e);
   2529           }
   2530           break;
   2531         case BOOL:
   2532           if (!ctx->boolOffset) {
   2533             // new packed bools
   2534             if (ctx->nibble == lowNbl) {
   2535               storeNew4bPackedBool(e);
   2536             }
   2537             else {
   2538               // high nibble, next byte is packed bools
   2539               storeTypeInHighNbl(e);
   2540               storeNew8bPackedBool(e);
   2541             }
   2542           }
   2543           else {
   2544             // there was a bool before this one, fill bits in nibbles
   2545             if (ctx->nibble == lowNbl) {
   2546               if (ctx->boolShift == 8) {
   2547                 // previous packed bool is full
   2548                 // this byte is the new packed bools
   2549                 storeNew4bPackedBool(e);
   2550               }
   2551               else {
   2552                 storeTypeOnly(e);
   2553                 storeBool(e);
   2554               }
   2555             }
   2556             else {
   2557               // high nibble
   2558               storeTypeInHighNbl(e);
   2559               if (ctx->boolShift == 8) {
   2560                 // previous packed bool is full
   2561                 // next byte is the new packed bools
   2562                 storeNew8bPackedBool(e);
   2563               }
   2564               else {
   2565                 storeBool(e);
   2566               }
   2567             }
   2568           }
   2569           break;
   2570         case DICT:
   2571           if (pack) {
   2572             if (type) {
   2573               // this is the first packed element
   2574               if (ctx->nibble == lowNbl) {
   2575                 uintToNetTypeVarint(r, type, packCount);
   2576               }
   2577               else {
   2578                 // high nibble
   2579                 // store type in high nibble
   2580                 ctx->nibble  = lowNbl;
   2581                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2582                 *data       |= type << 4;
   2583                 uintToVarint(r, packCount);
   2584               }
   2585               type = 0;
   2586             } // if type
   2587 
   2588             dictNetSerial(r, (sDictt *)e, ctx, /*packing=*/PACK);
   2589             // stop packing when packCount == 0
   2590             packCount--;
   2591             if (!packCount) pack = false;
   2592           } // if pack
   2593           else
   2594             dictNetSerial(r, (sDictt *)e, ctx, /*packing=*/NOPACKING);
   2595           break;
   2596         case DOUBLE:
   2597           if (pack) {
   2598             if (type) {
   2599               // this is the first packed element
   2600               if (ctx->nibble == lowNbl) {
   2601                 uintToNetTypeVarint(r, type, packCount);
   2602               }
   2603               else {
   2604                 // high nibble
   2605                 // store type in high nibble
   2606                 ctx->nibble  = lowNbl;
   2607                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2608                 *data       |= type << 4;
   2609                 uintToVarint(r, packCount);
   2610               }
   2611               type = 0;
   2612             } // if type
   2613 
   2614             sBytesPushBuffer(r, &((sDoublet *)e)->value, sizeof(double));
   2615             // stop packing when packCount == 0
   2616             packCount--;
   2617             if (!packCount) pack = false;
   2618           } // if pack
   2619           else {
   2620             if (ctx->nibble == lowNbl) {
   2621               storeTypeOnly(e);
   2622             }
   2623             else {
   2624               // high nibble
   2625               storeTypeInHighNbl(e);
   2626             }
   2627             sBytesPushBuffer(r, &((sDoublet *)e)->value, sizeof(double));
   2628           }
   2629           break;
   2630         case INT:
   2631           if (pack) {
   2632             if (type) {
   2633               // this is the first packed element
   2634               if (ctx->nibble == lowNbl) {
   2635                 uintToNetTypeVarint(r, type, packCount);
   2636               }
   2637               else {
   2638                 // high nibble
   2639                 // store type in high nibble
   2640                 ctx->nibble  = lowNbl;
   2641                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2642                 *data       |= type << 4;
   2643                 uintToVarint(r, packCount);
   2644               }
   2645               type = 0;
   2646             } // if type
   2647 
   2648             i64 v = ((sIntt *)&(e->type))->value;
   2649             uintToVarint(r, (v << 1) ^ (v >> 63));
   2650             // stop packing when packCount == 0
   2651             packCount--;
   2652             if (!packCount) pack = false;
   2653           } // if pack
   2654           else {
   2655             // encode int to varint
   2656             // v is int64_t to convert to varint
   2657             i64 v = ((sIntt *)&(e->type))->value;
   2658             if (ctx->nibble == lowNbl) {
   2659               // encode v with arithmetic shifts
   2660               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], (v << 1) ^ (v >> 63));
   2661             }
   2662             else {
   2663               // high nibble
   2664               storeTypeInHighNbl(e);
   2665               uintToVarint(r, (v << 1) ^ (v >> 63));
   2666             }
   2667           }
   2668           break;
   2669         case STRING:
   2670           if (pack) {
   2671             if (type) {
   2672               // this is the first packed element
   2673               if (ctx->nibble == lowNbl) {
   2674                 uintToNetTypeVarint(r, type, packCount);
   2675               }
   2676               else {
   2677                 // high nibble
   2678                 // store type in high nibble
   2679                 ctx->nibble  = lowNbl;
   2680                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2681                 *data       |= type << 4;
   2682                 uintToVarint(r, packCount);
   2683               }
   2684               type = 0;
   2685             } // if type
   2686 
   2687             sBytesPushBuffer(r, &((sStringt *)e)->data, sizeof(sStringt) + strlen(&(((sStringt *)e)->data)) -1);
   2688             // stop packing when packCount == 0
   2689             packCount--;
   2690             if (!packCount) pack = false;
   2691           } // if pack
   2692           else {
   2693             if (ctx->nibble == lowNbl) {
   2694               storeTypeOnly(e);
   2695             }
   2696             else {
   2697               // high nibble
   2698               storeTypeInHighNbl(e);
   2699             }
   2700             sBytesPushBuffer(r, &((sStringt *)e)->data, sizeof(sStringt) + strlen(&(((sStringt *)e)->data)) -1);
   2701           }
   2702           break;
   2703         case ARRAY:
   2704           if (pack) {
   2705             if (type) {
   2706               // this is the first packed element
   2707               if (ctx->nibble == lowNbl) {
   2708                 uintToNetTypeVarint(r, type, packCount);
   2709               }
   2710               else {
   2711                 // high nibble
   2712                 // store type in high nibble
   2713                 ctx->nibble  = lowNbl;
   2714                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2715                 *data       |= type << 4;
   2716                 uintToVarint(r, packCount);
   2717               }
   2718               type = 0;
   2719             } // if type
   2720 
   2721             arrayNetSerial(r, (sArrayt *)e, ctx, /*packing=*/PACK);
   2722             // stop packing when packCount == 0
   2723             packCount--;
   2724             if (!packCount) pack = false;
   2725           } // if pack
   2726           else
   2727             arrayNetSerial(r, (sArrayt *)e, ctx, /*packing=*/NOPACKING);
   2728           break;
   2729         case BYTES:
   2730           if (pack) {
   2731             if (type) {
   2732               // this is the first packed element
   2733               if (ctx->nibble == lowNbl) {
   2734                 uintToNetTypeVarint(r, type, packCount);
   2735               }
   2736               else {
   2737                 // high nibble
   2738                 // store type in high nibble
   2739                 ctx->nibble  = lowNbl;
   2740                 data         = (char *)&((*r)->data) + ctx->nblOffset;
   2741                 *data       |= type << 4;
   2742                 uintToVarint(r, packCount);
   2743               }
   2744               type = 0;
   2745             } // if type
   2746 
   2747             B = (sBytest *)e;
   2748             uintToVarint(r, B->count);
   2749             sBytesPushBuffer(r, &(B->data), B->count);
   2750             // stop packing when packCount == 0
   2751             packCount--;
   2752             if (!packCount) pack = false;
   2753           } // if pack
   2754           else {
   2755             B = (sBytest *)e;
   2756             if (ctx->nibble == lowNbl) {
   2757               uintToNetTypeVarint(r, NET_SERIAL_TYPES[(u8)e->type], B->count);
   2758             }
   2759             else {
   2760               // high nibble
   2761               storeTypeInHighNbl(e);
   2762               uintToVarint(r, B->count);
   2763             }
   2764             sBytesPushBuffer(r, &(B->data), B->count);
   2765           }
   2766           break;
   2767       }
   2768     }
   2769   }
   2770   ret;
   2771 }
   2772 
   2773 internal smallBytest* serialNetSerial(smallJsont *self) {
   2774 
   2775   smallt *o = getsoG(self);
   2776 
   2777   if (o == NULL)
   2778     ret NULL;
   2779 
   2780   sBytest *B = netSerial(o);
   2781 
   2782   if (!B) {
   2783     ret NULL;
   2784   }
   2785 
   2786   createAllocateSmallBytes(r);
   2787   r->B = B;
   2788   ret r;
   2789 }
   2790 
   2791 // -------------------------------------
   2792 // Deserializers
   2793 
   2794 // level 0
   2795 // like smallJson with ints and length encoded as varints
   2796 
   2797 /**
   2798  * deserializer top function
   2799  */
   2800 internal smallt* netDeserialLevel0(sBytest *obj) {
   2801   smallt   *r = NULL;
   2802   double   *D = NULL;
   2803   char     *s = NULL;
   2804   sBytest  *B = NULL;
   2805   uint32_t count;
   2806   char     *data  = NULL;
   2807 
   2808   switch(obj->data & 0xF) {
   2809     case S_UNDEFINED:
   2810       r = (smallt *) allocSUndefined();
   2811       break;
   2812     case S_BOOL:
   2813       r = (smallt *) allocSBool(obj->data & 0x10);
   2814       break;
   2815     case S_DICT:
   2816       data = (char *)&(obj->data);
   2817       dictNetDeserialLevel0((sDictt **)&r, &data);
   2818       break;
   2819     case S_DOUBLE:
   2820       data = &(obj->data)+1;
   2821       D = (double *)data;
   2822       r = (smallt *) allocSDouble(*D);
   2823       break;
   2824     case S_INT:
   2825       data  = &(obj->data);
   2826       u64 v = netTypeVarintToUint((u8**)&data);
   2827       v = (v >> 1) ^ (~(v & 1) + 1);
   2828       r = (smallt *) allocSInt(v);
   2829       break;
   2830     case S_STRING:
   2831       s = (char *)&(obj->data)+1;
   2832       r = (smallt *) allocSStringTiny(s);
   2833       break;
   2834     case S_ARRAY:
   2835       data = (char *)&(obj->data);
   2836       arrayNetDeserialLevel0((sArrayt **)&r, &data);
   2837       break;
   2838     case S_BYTES:
   2839       B     = allocSBytes();
   2840       data  = &(obj->data);
   2841       count = netTypeVarintToUint((u8**)&data);
   2842       sBytesPushBuffer(&B, data, count);
   2843       r = (smallt *)B;
   2844       break;
   2845   }
   2846 
   2847   ret r;
   2848 }
   2849 
   2850 /**
   2851  * deserialize dictionary from data
   2852  *
   2853  * a new dictionary is allocated
   2854  *
   2855  * \param
   2856  *    dict dictionary holding the elements
   2857  *    data serialized dictionary
   2858  */
   2859 internal void dictNetDeserialLevel0(sDictt **dict, char **data) {
   2860   sUndefinedt *u = NULL;
   2861   sBoolt *bo = NULL;
   2862   double *D = NULL;
   2863   sDoublet *Do = NULL;
   2864   sDictt *d = NULL;
   2865   sIntt *io = NULL;
   2866   char *s = NULL;
   2867   sStringt *so = NULL;
   2868   sArrayt *a = NULL;
   2869   sBytest *B = NULL;
   2870   uint32_t count;
   2871   uint32_t dictCount;
   2872 
   2873   dictCount = netTypeVarintToUint((u8**)data);
   2874 
   2875   if (!dictCount) {
   2876     *dict = allocSDict();
   2877     ret;
   2878   }
   2879 
   2880   loop(dictCount) {
   2881     char type;
   2882     char *key;
   2883     key    = *data;
   2884     *data += strlen(key)+1;
   2885     type   = **data;
   2886 
   2887     switch(type & 0xF) {
   2888       case S_UNDEFINED:
   2889         (*data)++;
   2890         u    = allocSUndefined();
   2891         sDictPushTiny(dict, key, (smallt *) u);
   2892         break;
   2893       case S_BOOL:
   2894         (*data)++;
   2895         bo = allocSBool(type & 0x10);
   2896         sDictPushTiny(dict, key, (smallt *) bo);
   2897         break;
   2898       case S_DICT:
   2899         d = NULL;
   2900         dictNetDeserialLevel0(&d, data);
   2901         sDictPushTiny(dict, key, (smallt *) d);
   2902         break;
   2903       case S_DOUBLE:
   2904         (*data)++;
   2905         D      = (double *)(*data);
   2906         *data += sizeof(double);
   2907         Do     = allocSDouble(*D);
   2908         sDictPushTiny(dict, key, (smallt *) Do);
   2909         break;
   2910       case S_INT: {
   2911           u64 v = netTypeVarintToUint((u8**)data);
   2912           v = (v >> 1) ^ (~(v & 1) + 1);
   2913           io     = allocSInt(v);
   2914           sDictPushTiny(dict, key, (smallt *) io);
   2915         }
   2916         break;
   2917       case S_STRING:
   2918         (*data)++;
   2919         s      = (char *)(*data);
   2920         *data += strlen(s)+1;
   2921         so     = allocSStringTiny(s);
   2922         sDictPushTiny(dict, key, (smallt *) so);
   2923         break;
   2924       case S_ARRAY:
   2925         a = NULL;
   2926         arrayNetDeserialLevel0(&a, data);
   2927         sDictPushTiny(dict, key, (smallt *) a);
   2928         break;
   2929       case S_BYTES:
   2930         B      = allocSBytes();
   2931         count  = netTypeVarintToUint((u8**)data);
   2932         sBytesPushBuffer(&B, *data, count);
   2933         *data += count;
   2934         sDictPushTiny(dict, key, (smallt *) B);
   2935         break;
   2936     }
   2937   }
   2938 }
   2939 
   2940 /**
   2941  * deserialize array from data
   2942  *
   2943  * a new array is allocated
   2944  *
   2945  * \param
   2946  *    array holding the elements
   2947  *    data serialized dictionary
   2948  */
   2949 internal void arrayNetDeserialLevel0(sArrayt **array, char **data) {
   2950   sUndefinedt *u = NULL;
   2951   sBoolt *bo = NULL;
   2952   double *D = NULL;
   2953   sDoublet *Do = NULL;
   2954   sDictt *d = NULL;
   2955   sIntt *io = NULL;
   2956   char *s = NULL;
   2957   sStringt *so = NULL;
   2958   sArrayt *a = NULL;
   2959   sBytest *B = NULL;
   2960   uint32_t count;
   2961   uint32_t arrayCount;
   2962 
   2963   arrayCount = netTypeVarintToUint((u8**)data);
   2964 
   2965   if (!arrayCount) {
   2966     *array = allocSArray();;
   2967     ret;
   2968   }
   2969 
   2970   loop(arrayCount) {
   2971     char type;
   2972     type  = **data;
   2973 
   2974     switch(type & 0xF) {
   2975       case S_UNDEFINED:
   2976         (*data)++;
   2977         u = allocSUndefined();
   2978         sArrayPushTiny(array, (smallt *) u);
   2979         break;
   2980       case S_BOOL:
   2981         (*data)++;
   2982         bo     = allocSBool(type & 0x10);
   2983         sArrayPushTiny(array, (smallt *) bo);
   2984         break;
   2985       case S_DICT:
   2986         d = NULL;
   2987         dictNetDeserialLevel0(&d, data);
   2988         sArrayPushTiny(array, (smallt *) d);
   2989         break;
   2990       case S_DOUBLE:
   2991         (*data)++;
   2992         D      = (double *)(*data);
   2993         *data += sizeof(double);
   2994         Do     = allocSDouble(*D);
   2995         sArrayPushTiny(array, (smallt *) Do);
   2996         break;
   2997       case S_INT: {
   2998           u64 v = netTypeVarintToUint((u8**)data);
   2999           v = (v >> 1) ^ (~(v & 1) + 1);
   3000           io     = allocSInt(v);
   3001           sArrayPushTiny(array, (smallt *) io);
   3002         }
   3003         break;
   3004       case S_STRING:
   3005         (*data)++;
   3006         s      = (char *)(*data);
   3007         *data += strlen(s)+1;
   3008         so     = allocSStringTiny(s);
   3009         sArrayPushTiny(array, (smallt *) so);
   3010         break;
   3011       case S_ARRAY:
   3012         a = NULL;
   3013         arrayNetDeserialLevel0(&a, data);
   3014         sArrayPushTiny(array, (smallt *) a);
   3015         break;
   3016       case S_BYTES:
   3017         B      = allocSBytes();
   3018         count  = netTypeVarintToUint((u8**)data);
   3019         sBytesPushBuffer(&B, *data, count);
   3020         *data += count;
   3021         sArrayPushTiny(array, (smallt *) B);
   3022         break;
   3023     }
   3024   }
   3025 }
   3026 
   3027 internal smallJsont* deserialNetSerialLevel0(smallJsont *self, smallBytest *data) {
   3028 
   3029   if (!data or !data->B or !data->B->count) {
   3030     ret self;
   3031   }
   3032 
   3033   smallt *o = netDeserialLevel0(data->B);
   3034 
   3035   if (!o) {
   3036     ret self;
   3037   }
   3038 
   3039   freeG(self);
   3040 
   3041   setsoG(self, o);
   3042 
   3043   ret self;
   3044 }
   3045 
   3046 // level 1
   3047 // like level 0 with type encoded in nibbles and bools are packed
   3048 
   3049 /**
   3050  * deserializer top function
   3051  */
   3052 internal smallt* netDeserialLevel1(sBytest *obj) {
   3053   smallt   *r = NULL;
   3054   double   *D = NULL;
   3055   char     *s = NULL;
   3056   sBytest  *B = NULL;
   3057   uint32_t count;
   3058   char     *data  = NULL;
   3059   contextt ctx = {.nibble=lowNbl, .nblAddr=NULL, .boolShift = 0, .boolAddr=NULL};
   3060 
   3061   switch(obj->data & 0xF) {
   3062     case S_UNDEFINED:
   3063       r = (smallt *) allocSUndefined();
   3064       break;
   3065     case S_BOOL:
   3066       r = (smallt *) allocSBool(obj->data & 0x10);
   3067       break;
   3068     case S_DICT:
   3069       data     = (char *)&(obj->data);
   3070       //debug - ctx.dbuf = (u8*) data;
   3071       dictNetDeserialLevel1((sDictt **)&r, (u8**)&data, &ctx);
   3072       break;
   3073     case S_DOUBLE:
   3074       data = &(obj->data)+1;
   3075       D = (double *)data;
   3076       r = (smallt *) allocSDouble(*D);
   3077       break;
   3078     case S_INT:
   3079       data  = &(obj->data);
   3080       u64 v = netTypeVarintToUint((u8**)&data);
   3081       v = (v >> 1) ^ (~(v & 1) + 1);
   3082       r = (smallt *) allocSInt(v);
   3083       break;
   3084     case S_STRING:
   3085       s = (char *)&(obj->data)+1;
   3086       r = (smallt *) allocSStringTiny(s);
   3087       break;
   3088     case S_ARRAY:
   3089       data = (char *)&(obj->data);
   3090       //debug - ctx.dbuf = (u8*) data;
   3091       arrayNetDeserialLevel1((sArrayt **)&r, (u8**)&data, &ctx);
   3092       break;
   3093     case S_BYTES:
   3094       B     = allocSBytes();
   3095       data  = &(obj->data);
   3096       count = netTypeVarintToUint((u8**)&data);
   3097       sBytesPushBuffer(&B, data, count);
   3098       r = (smallt *)B;
   3099       break;
   3100   }
   3101 
   3102   ret r;
   3103 }
   3104 
   3105 /**
   3106  * deserialize dictionary from data
   3107  *
   3108  * a new dictionary is allocated
   3109  *
   3110  * \param
   3111  *    dict dictionary holding the elements
   3112  *    data serialized dictionary
   3113  */
   3114 internal void dictNetDeserialLevel1(sDictt **dict, u8 **data, contextt *ctx) {
   3115   sUndefinedt *u = NULL;
   3116   sBoolt *bo = NULL;
   3117   double *D = NULL;
   3118   sDoublet *Do = NULL;
   3119   sDictt *d = NULL;
   3120   sIntt *io = NULL;
   3121   char *s = NULL;
   3122   sStringt *so = NULL;
   3123   sArrayt *a = NULL;
   3124   sBytest *B = NULL;
   3125   uint32_t count;
   3126   uint32_t dictCount;
   3127 
   3128   if (ctx->nibble == lowNbl) {
   3129     dictCount = netTypeVarintToUint(data);
   3130   }
   3131   else {
   3132     // high nibble
   3133     // type = *(ctx->dbuf + ctx->nblOffset) >> 4;
   3134     #define readTypeInHighNbl\
   3135     ctx->nibble  = lowNbl;\
   3136     if (ctx->nblAddr == *data)\
   3137       /* data points to the type, next byte is count */\
   3138       (*data)++
   3139     readTypeInHighNbl;
   3140     dictCount = varintToUint(data);
   3141   }
   3142 
   3143   if (!dictCount) {
   3144     *dict = allocSDict();
   3145     ret;
   3146   }
   3147 
   3148   loop(dictCount) {
   3149     char *key  = (char*)*data;
   3150     *data     += strlen(key)+1;
   3151     char type;
   3152     if (ctx->nibble == lowNbl) {
   3153       type = (**data) & 0xF;
   3154     }
   3155     else {
   3156       // high nibble
   3157       type = (*ctx->nblAddr) >> 4;
   3158     }
   3159 
   3160     switch(type) {
   3161       case S_UNDEFINED:
   3162         if (ctx->nibble == lowNbl) {
   3163           #define readTypeOnly\
   3164           ctx->nibble  = highNbl;\
   3165           ctx->nblAddr = *data;\
   3166           (*data)++
   3167           readTypeOnly;
   3168         }
   3169         else {
   3170           // high nibble
   3171           readTypeInHighNbl;
   3172         }
   3173         u = allocSUndefined();
   3174         sDictPushTiny(dict, key, (smallt *) u);
   3175         break;
   3176       case S_BOOL:
   3177         if (!ctx->boolAddr) {
   3178           // new packed bools
   3179           if (ctx->nibble == lowNbl) {
   3180             #define read4bPackedBool\
   3181             ctx->boolShift  = 5;\
   3182             ctx->boolAddr   = *data;\
   3183             (*data)++
   3184             read4bPackedBool;
   3185             bo = allocSBool((*ctx->boolAddr) & 0x10);
   3186           }
   3187           else {
   3188             // high nibble
   3189             readTypeInHighNbl;
   3190             #define read8bPackedBool\
   3191             ctx->boolShift  = 1;\
   3192             ctx->boolAddr   = *data;\
   3193             (*data)++
   3194             read8bPackedBool;
   3195             bo = allocSBool((*ctx->boolAddr) & 0x1);
   3196           }
   3197         }
   3198         else {
   3199           // there was a bool before this one, read bits in nibbles
   3200           if (ctx->nibble == lowNbl) {
   3201             if (ctx->boolShift == 8) {
   3202               read4bPackedBool;
   3203               bo = allocSBool((*ctx->boolAddr) & 0x10);
   3204             }
   3205             else {
   3206               readTypeOnly;
   3207               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3208             }
   3209           }
   3210           else {
   3211             // high nibble
   3212             readTypeInHighNbl;
   3213             if (ctx->boolShift == 8) {
   3214               read8bPackedBool;
   3215               bo = allocSBool((*ctx->boolAddr) & 0x1);
   3216             }
   3217             else {
   3218               // high nibble
   3219               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3220             }
   3221           }
   3222         }
   3223         sDictPushTiny(dict, key, (smallt *) bo);
   3224         break;
   3225       case S_DICT:
   3226         d = NULL;
   3227         dictNetDeserialLevel1(&d, data, ctx);
   3228         sDictPushTiny(dict, key, (smallt *) d);
   3229         break;
   3230       case S_DOUBLE:
   3231         if (ctx->nibble == lowNbl) {
   3232           readTypeOnly;
   3233         }
   3234         else {
   3235           // high nibble
   3236           readTypeInHighNbl;
   3237         }
   3238         D      = (double *)(*data);
   3239         *data += sizeof(double);
   3240         Do     = allocSDouble(*D);
   3241         sDictPushTiny(dict, key, (smallt *) Do);
   3242         break;
   3243       case S_INT: {
   3244           u64 v;
   3245           if (ctx->nibble == lowNbl) {
   3246             v = netTypeVarintToUint((u8**)data);
   3247           }
   3248           else {
   3249             // high nibble
   3250             readTypeInHighNbl;
   3251             v = varintToUint(data);
   3252           }
   3253           v  = (v >> 1) ^ (~(v & 1) + 1);
   3254           io = allocSInt(v);
   3255           sDictPushTiny(dict, key, (smallt *) io);
   3256         }
   3257         break;
   3258       case S_STRING:
   3259         if (ctx->nibble == lowNbl) {
   3260           readTypeOnly;
   3261         }
   3262         else {
   3263           // high nibble
   3264           readTypeInHighNbl;
   3265         }
   3266         s      = (char *)(*data);
   3267         *data += strlen(s)+1;
   3268         so     = allocSStringTiny(s);
   3269         sDictPushTiny(dict, key, (smallt *) so);
   3270         break;
   3271       case S_ARRAY:
   3272         a = NULL;
   3273         arrayNetDeserialLevel1(&a, data, ctx);
   3274         sDictPushTiny(dict, key, (smallt *) a);
   3275         break;
   3276       case S_BYTES:
   3277         B      = allocSBytes();
   3278         if (ctx->nibble == lowNbl) {
   3279           count  = netTypeVarintToUint((u8**)data);
   3280         }
   3281         else {
   3282           // high nibble
   3283           readTypeInHighNbl;
   3284           count  = varintToUint((u8**)data);
   3285         }
   3286         sBytesPushBuffer(&B, *data, count);
   3287         *data += count;
   3288         sDictPushTiny(dict, key, (smallt *) B);
   3289         break;
   3290     }
   3291   }
   3292 }
   3293 
   3294 /**
   3295  * deserialize array from data
   3296  *
   3297  * a new array is allocated
   3298  *
   3299  * \param
   3300  *    array holding the elements
   3301  *    data serialized dictionary
   3302  */
   3303 internal void arrayNetDeserialLevel1(sArrayt **array, u8 **data, contextt *ctx) {
   3304   sUndefinedt *u = NULL;
   3305   sBoolt *bo = NULL;
   3306   double *D = NULL;
   3307   sDoublet *Do = NULL;
   3308   sDictt *d = NULL;
   3309   sIntt *io = NULL;
   3310   char *s = NULL;
   3311   sStringt *so = NULL;
   3312   sArrayt *a = NULL;
   3313   sBytest *B = NULL;
   3314   uint32_t count;
   3315   uint32_t arrayCount;
   3316 
   3317   if (ctx->nibble == lowNbl) {
   3318     arrayCount = netTypeVarintToUint(data);
   3319   }
   3320   else {
   3321     // high nibble
   3322     readTypeInHighNbl;
   3323     arrayCount = varintToUint(data);
   3324   }
   3325 
   3326   if (!arrayCount) {
   3327     *array = allocSArray();;
   3328     ret;
   3329   }
   3330 
   3331   loop(arrayCount) {
   3332     char type;
   3333     if (ctx->nibble == lowNbl) {
   3334       type = (**data) & 0xF;
   3335     }
   3336     else {
   3337       // high nibble
   3338       type = (*ctx->nblAddr) >> 4;
   3339     }
   3340 
   3341     switch(type) {
   3342       case S_UNDEFINED:
   3343         if (ctx->nibble == lowNbl) {
   3344           readTypeOnly;
   3345         }
   3346         else {
   3347           // high nibble
   3348           readTypeInHighNbl;
   3349         }
   3350         u = allocSUndefined();
   3351         sArrayPushTiny(array, (smallt *) u);
   3352         break;
   3353       case S_BOOL:
   3354         if (!ctx->boolAddr) {
   3355           // new packed bools
   3356           if (ctx->nibble == lowNbl) {
   3357             read4bPackedBool;
   3358             bo = allocSBool((*ctx->boolAddr) & 0x10);
   3359           }
   3360           else {
   3361             // high nibble
   3362             readTypeInHighNbl;
   3363             read8bPackedBool;
   3364             bo = allocSBool((*ctx->boolAddr) & 0x1);
   3365           }
   3366         }
   3367         else {
   3368           // there was a bool before this one, read bits in nibbles
   3369           if (ctx->nibble == lowNbl) {
   3370             if (ctx->boolShift == 8) {
   3371               read4bPackedBool;
   3372               bo = allocSBool((*ctx->boolAddr) & 0x10);
   3373             }
   3374             else {
   3375               readTypeOnly;
   3376               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3377             }
   3378           }
   3379           else {
   3380             // high nibble
   3381             readTypeInHighNbl;
   3382             if (ctx->boolShift == 8) {
   3383               read8bPackedBool;
   3384               bo = allocSBool((*ctx->boolAddr) & 0x1);
   3385             }
   3386             else {
   3387               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3388             }
   3389           }
   3390         }
   3391         sArrayPushTiny(array, (smallt *) bo);
   3392         break;
   3393       case S_DICT:
   3394         d = NULL;
   3395         dictNetDeserialLevel1(&d, data, ctx);
   3396         sArrayPushTiny(array, (smallt *) d);
   3397         break;
   3398       case S_DOUBLE:
   3399         if (ctx->nibble == lowNbl) {
   3400           readTypeOnly;
   3401         }
   3402         else {
   3403           // high nibble
   3404           readTypeInHighNbl;
   3405         }
   3406         D      = (double *)(*data);
   3407         *data += sizeof(double);
   3408         Do     = allocSDouble(*D);
   3409         sArrayPushTiny(array, (smallt *) Do);
   3410         break;
   3411       case S_INT: {
   3412           u64 v;
   3413           if (ctx->nibble == lowNbl) {
   3414             v = netTypeVarintToUint((u8**)data);
   3415           }
   3416           else {
   3417             // high nibble
   3418             readTypeInHighNbl;
   3419             v = varintToUint(data);
   3420           }
   3421           v  = (v >> 1) ^ (~(v & 1) + 1);
   3422           io = allocSInt(v);
   3423           sArrayPushTiny(array, (smallt *) io);
   3424         }
   3425         break;
   3426       case S_STRING:
   3427         if (ctx->nibble == lowNbl) {
   3428           readTypeOnly;
   3429         }
   3430         else {
   3431           // high nibble
   3432           readTypeInHighNbl;
   3433         }
   3434         s      = (char *)(*data);
   3435         *data += strlen(s)+1;
   3436         so     = allocSStringTiny(s);
   3437         sArrayPushTiny(array, (smallt *) so);
   3438         break;
   3439       case S_ARRAY:
   3440         a = NULL;
   3441         arrayNetDeserialLevel1(&a, data, ctx);
   3442         sArrayPushTiny(array, (smallt *) a);
   3443         break;
   3444       case S_BYTES:
   3445         B      = allocSBytes();
   3446         if (ctx->nibble == lowNbl) {
   3447           count  = netTypeVarintToUint((u8**)data);
   3448         }
   3449         else {
   3450           // high nibble
   3451           readTypeInHighNbl;
   3452           count  = varintToUint((u8**)data);
   3453         }
   3454         sBytesPushBuffer(&B, *data, count);
   3455         *data += count;
   3456         sArrayPushTiny(array, (smallt *) B);
   3457         break;
   3458     }
   3459   }
   3460 }
   3461 
   3462 internal smallJsont* deserialNetSerialLevel1(smallJsont *self, smallBytest *data) {
   3463 
   3464   if (!data or !data->B or !data->B->count) {
   3465     ret self;
   3466   }
   3467 
   3468   smallt *o = netDeserialLevel1(data->B);
   3469 
   3470   if (!o) {
   3471     ret self;
   3472   }
   3473 
   3474   freeG(self);
   3475 
   3476   setsoG(self, o);
   3477 
   3478   ret self;
   3479 }
   3480 
   3481 // level 2
   3482 // like level 1, arrays are set to uniform when all elements are same type
   3483 
   3484 /**
   3485  * deserializer top function
   3486  */
   3487 internal smallt* netDeserialLevel2(sBytest *obj) {
   3488   smallt   *r = NULL;
   3489   double   *D = NULL;
   3490   char     *s = NULL;
   3491   sBytest  *B = NULL;
   3492   uint32_t count;
   3493   char     *data  = NULL;
   3494   contextt ctx = {.nibble=lowNbl, .nblAddr=NULL, .boolShift = 0, .boolAddr=NULL};
   3495 
   3496   switch(obj->data & 0xF) {
   3497     case S_UNDEFINED:
   3498       r = (smallt *) allocSUndefined();
   3499       break;
   3500     case S_BOOL:
   3501       r = (smallt *) allocSBool(obj->data & 0x10);
   3502       break;
   3503     case S_DICT:
   3504       data     = (char *)&(obj->data);
   3505       //debug - ctx.dbuf = (u8*) data;
   3506       dictNetDeserialLevel2((sDictt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   3507       break;
   3508     case S_DOUBLE:
   3509       data = &(obj->data)+1;
   3510       D = (double *)data;
   3511       r = (smallt *) allocSDouble(*D);
   3512       break;
   3513     case S_INT:
   3514       data  = &(obj->data);
   3515       u64 v = netTypeVarintToUint((u8**)&data);
   3516       v     = (v >> 1) ^ (~(v & 1) + 1);
   3517       r = (smallt *) allocSInt(v);
   3518       break;
   3519     case S_STRING:
   3520       s = (char *)&(obj->data)+1;
   3521       r = (smallt *) allocSStringTiny(s);
   3522       break;
   3523     case S_ARRAY:
   3524       data = (char *)&(obj->data);
   3525       //debug - ctx.dbuf = (u8*) data;
   3526       arrayNetDeserialLevel2((sArrayt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   3527       break;
   3528     case S_BYTES:
   3529       B     = allocSBytes();
   3530       data  = &(obj->data);
   3531       count = netTypeVarintToUint((u8**)&data);
   3532       sBytesPushBuffer(&B, data, count);
   3533       r = (smallt *)B;
   3534       break;
   3535     case UNIFORM_DICT:
   3536       data     = (char *)&(obj->data);
   3537       //debug - ctx.dbuf = (u8*) data;
   3538       uniformDictNetDeserialLevel2((sDictt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   3539       break;
   3540     case UNIFORM_ARRAY:
   3541       data     = (char *)&(obj->data);
   3542       //debug - ctx.dbuf = (u8*) data;
   3543       uniformArrayNetDeserialLevel2((sArrayt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   3544       break;
   3545   }
   3546 
   3547   ret r;
   3548 }
   3549 
   3550 /**
   3551  * deserialize dictionary from data
   3552  *
   3553  * a new dictionary is allocated
   3554  *
   3555  * \param
   3556  *    dict dictionary holding the elements
   3557  *    data serialized dictionary
   3558  */
   3559 internal void dictNetDeserialLevel2(sDictt **dict, u8 **data, contextt *ctx, bool packed) {
   3560   sUndefinedt *u = NULL;
   3561   sBoolt *bo = NULL;
   3562   double *D = NULL;
   3563   sDoublet *Do = NULL;
   3564   sDictt *d = NULL;
   3565   sIntt *io = NULL;
   3566   char *s = NULL;
   3567   sStringt *so = NULL;
   3568   sArrayt *a = NULL;
   3569   sBytest *B = NULL;
   3570   uint32_t count;
   3571   uint32_t dictCount;
   3572 
   3573   if (packed) {
   3574     dictCount = varintToUint(data);
   3575   }
   3576   else {
   3577     if (ctx->nibble == lowNbl) {
   3578       dictCount = netTypeVarintToUint(data);
   3579     }
   3580     else {
   3581       // high nibble
   3582       // type = *(ctx->dbuf + ctx->nblOffset) >> 4;
   3583       #define readTypeInHighNbl\
   3584       ctx->nibble  = lowNbl;\
   3585       if (ctx->nblAddr == *data)\
   3586       /* data points to the type, next byte is count */\
   3587       (*data)++
   3588       readTypeInHighNbl;
   3589       dictCount = varintToUint(data);
   3590     }
   3591   }
   3592 
   3593   if (!dictCount) {
   3594     *dict = allocSDict();
   3595     ret;
   3596   }
   3597 
   3598   loop(dictCount) {
   3599     char *key  = (char*)*data;
   3600     *data     += strlen(key)+1;
   3601     char type;
   3602     if (ctx->nibble == lowNbl) {
   3603       type = (**data) & 0xF;
   3604     }
   3605     else {
   3606       // high nibble
   3607       type = (*ctx->nblAddr) >> 4;
   3608     }
   3609 
   3610     switch(type) {
   3611       case S_UNDEFINED:
   3612         if (ctx->nibble == lowNbl) {
   3613           #define readTypeOnly\
   3614           ctx->nibble  = highNbl;\
   3615           ctx->nblAddr = *data;\
   3616           (*data)++
   3617           readTypeOnly;
   3618         }
   3619         else {
   3620           // high nibble
   3621           readTypeInHighNbl;
   3622         }
   3623         u = allocSUndefined();
   3624         sDictPushTiny(dict, key, (smallt *) u);
   3625         break;
   3626       case S_BOOL:
   3627         if (!ctx->boolAddr) {
   3628           // new packed bools
   3629           if (ctx->nibble == lowNbl) {
   3630             #define read4bPackedBool\
   3631             ctx->boolShift  = 5;\
   3632             ctx->boolAddr   = *data;\
   3633             (*data)++
   3634             read4bPackedBool;
   3635             bo = allocSBool((*ctx->boolAddr) & 0x10);
   3636           }
   3637           else {
   3638             // high nibble
   3639             readTypeInHighNbl;
   3640             #define read8bPackedBool\
   3641             ctx->boolShift  = 1;\
   3642             ctx->boolAddr   = *data;\
   3643             (*data)++
   3644             read8bPackedBool;
   3645             bo = allocSBool((*ctx->boolAddr) & 0x1);
   3646           }
   3647         }
   3648         else {
   3649           // there was a bool before this one, read bits in nibbles
   3650           if (ctx->nibble == lowNbl) {
   3651             if (ctx->boolShift == 8) {
   3652               read4bPackedBool;
   3653               bo = allocSBool((*ctx->boolAddr) & 0x10);
   3654             }
   3655             else {
   3656               readTypeOnly;
   3657               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3658             }
   3659           }
   3660           else {
   3661             // high nibble
   3662             readTypeInHighNbl;
   3663             if (ctx->boolShift == 8) {
   3664               read8bPackedBool;
   3665               bo = allocSBool((*ctx->boolAddr) & 0x1);
   3666             }
   3667             else {
   3668               // high nibble
   3669               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3670             }
   3671           }
   3672         }
   3673         sDictPushTiny(dict, key, (smallt *) bo);
   3674         break;
   3675       case S_DICT:
   3676         d = NULL;
   3677         dictNetDeserialLevel2(&d, data, ctx, /*packed=*/false);
   3678         sDictPushTiny(dict, key, (smallt *) d);
   3679         break;
   3680       case S_DOUBLE:
   3681         if (ctx->nibble == lowNbl) {
   3682           readTypeOnly;
   3683         }
   3684         else {
   3685           // high nibble
   3686           readTypeInHighNbl;
   3687         }
   3688         D      = (double *)(*data);
   3689         *data += sizeof(double);
   3690         Do     = allocSDouble(*D);
   3691         sDictPushTiny(dict, key, (smallt *) Do);
   3692         break;
   3693       case S_INT: {
   3694           u64 v;
   3695           if (ctx->nibble == lowNbl) {
   3696             v = netTypeVarintToUint((u8**)data);
   3697           }
   3698           else {
   3699             // high nibble
   3700             readTypeInHighNbl;
   3701             v = varintToUint(data);
   3702           }
   3703           v  = (v >> 1) ^ (~(v & 1) + 1);
   3704           io = allocSInt(v);
   3705           sDictPushTiny(dict, key, (smallt *) io);
   3706         }
   3707         break;
   3708       case S_STRING:
   3709         if (ctx->nibble == lowNbl) {
   3710           readTypeOnly;
   3711         }
   3712         else {
   3713           // high nibble
   3714           readTypeInHighNbl;
   3715         }
   3716         s      = (char *)(*data);
   3717         *data += strlen(s)+1;
   3718         so     = allocSStringTiny(s);
   3719         sDictPushTiny(dict, key, (smallt *) so);
   3720         break;
   3721       case S_ARRAY:
   3722         a = NULL;
   3723         arrayNetDeserialLevel2(&a, data, ctx, /*packed=*/false);
   3724         sDictPushTiny(dict, key, (smallt *) a);
   3725         break;
   3726       case S_BYTES:
   3727         B      = allocSBytes();
   3728         if (ctx->nibble == lowNbl) {
   3729           count  = netTypeVarintToUint((u8**)data);
   3730         }
   3731         else {
   3732           // high nibble
   3733           readTypeInHighNbl;
   3734           count  = varintToUint((u8**)data);
   3735         }
   3736         sBytesPushBuffer(&B, *data, count);
   3737         *data += count;
   3738         sDictPushTiny(dict, key, (smallt *) B);
   3739         break;
   3740       case UNIFORM_DICT:
   3741         d = NULL;
   3742         uniformDictNetDeserialLevel2(&d, data, ctx, /*packed=*/false);
   3743         sDictPushTiny(dict, key, (smallt *) d);
   3744         break;
   3745       case UNIFORM_ARRAY:
   3746         a = NULL;
   3747         uniformArrayNetDeserialLevel2(&a, data, ctx, /*packed=*/false);
   3748         sDictPushTiny(dict, key, (smallt *) a);
   3749         break;
   3750     }
   3751   }
   3752 }
   3753 
   3754 /**
   3755  * deserialize dictionary from data
   3756  *
   3757  * a new dictionary is allocated
   3758  *
   3759  * \param
   3760  *    dict dictionary holding the elements
   3761  *    data serialized dictionary
   3762  */
   3763 internal void uniformDictNetDeserialLevel2(sDictt **dict, u8 **data, contextt *ctx, bool packed) {
   3764   sUndefinedt *u = NULL;
   3765   sBoolt *bo = NULL;
   3766   double *D = NULL;
   3767   sDoublet *Do = NULL;
   3768   sDictt *d = NULL;
   3769   sIntt *io = NULL;
   3770   char *s = NULL;
   3771   sStringt *so = NULL;
   3772   sArrayt *a = NULL;
   3773   sBytest *B = NULL;
   3774   uint32_t count;
   3775   uint32_t dictCount;
   3776   u8 type;
   3777 
   3778   if (packed) {
   3779     type = (**data) & 0xF;
   3780     dictCount = netTypeVarintToUint(data);
   3781   }
   3782   else {
   3783     if (ctx->nibble == lowNbl) {
   3784       type = (**data) >> 4;
   3785       (*data)++;
   3786       dictCount = varintToUint(data);
   3787     }
   3788     else {
   3789       readTypeInHighNbl;
   3790       type = (**data) & 0xF;
   3791       dictCount = netTypeVarintToUint(data);
   3792     }
   3793   }
   3794 
   3795   if (!dictCount) {
   3796     *dict = allocSDict();
   3797     ret;
   3798   }
   3799 
   3800 
   3801   switch(type) {
   3802     case S_UNDEFINED:
   3803       loop(dictCount) {
   3804         char *key  = (char*)*data;
   3805         *data     += strlen(key)+1;
   3806         u = allocSUndefined();
   3807         sDictPushTiny(dict, key, (smallt *) u);
   3808       }
   3809       break;
   3810     case S_BOOL:
   3811       loop(dictCount) {
   3812         char *key  = (char*)*data;
   3813         *data     += strlen(key)+1;
   3814         if (!ctx->boolAddr) {
   3815           read8bPackedBool;
   3816           ctx->boolShift = 0;
   3817         }
   3818         if (ctx->boolShift == 8) {
   3819           readTypeInHighNbl;
   3820           read8bPackedBool;
   3821           bo = allocSBool((*ctx->boolAddr) & 0x1);
   3822         }
   3823         else {
   3824           // high nibble
   3825           readTypeInHighNbl;
   3826           bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3827         }
   3828         sDictPushTiny(dict, key, (smallt *) bo);
   3829       }
   3830       break;
   3831     case S_DICT:
   3832       loop(dictCount) {
   3833         char *key  = (char*)*data;
   3834         *data     += strlen(key)+1;
   3835         d          = NULL;
   3836         dictNetDeserialLevel2(&d, data, ctx, /*packed=*/true);
   3837         sDictPushTiny(dict, key, (smallt *) d);
   3838       }
   3839       break;
   3840     case S_DOUBLE:
   3841       loop(dictCount) {
   3842         char *key  = (char*)*data;
   3843         *data     += strlen(key)+1;
   3844         D          = (double *)(*data);
   3845         *data     += sizeof(double);
   3846         Do         = allocSDouble(*D);
   3847         sDictPushTiny(dict, key, (smallt *) Do);
   3848       }
   3849       break;
   3850     case S_INT:
   3851       loop(dictCount) {
   3852         char *key  = (char*)*data;
   3853         *data     += strlen(key)+1;
   3854         u64 v      = varintToUint(data);
   3855         v          = (v >> 1) ^ (~(v & 1) + 1);
   3856         io         = allocSInt(v);
   3857         sDictPushTiny(dict, key, (smallt *) io);
   3858       }
   3859       break;
   3860     case S_STRING:
   3861       loop(dictCount) {
   3862         char *key  = (char*)*data;
   3863         *data     += strlen(key)+1;
   3864         s          = (char *)(*data);
   3865         *data     += strlen(s)+1;
   3866         so         = allocSStringTiny(s);
   3867         sDictPushTiny(dict, key, (smallt *) so);
   3868       }
   3869       break;
   3870     case S_ARRAY:
   3871       loop(dictCount) {
   3872         char *key  = (char*)*data;
   3873         *data     += strlen(key)+1;
   3874         a          = NULL;
   3875         arrayNetDeserialLevel2(&a, data, ctx, /*packed=*/true);
   3876         sDictPushTiny(dict, key, (smallt *) a);
   3877       }
   3878       break;
   3879     case S_BYTES:
   3880       loop(dictCount) {
   3881         char *key  = (char*)*data;
   3882         *data     += strlen(key)+1;
   3883         B          = allocSBytes();
   3884         count      = varintToUint((u8**)data);
   3885         sBytesPushBuffer(&B, *data, count);
   3886         *data     += count;
   3887         sDictPushTiny(dict, key, (smallt *) B);
   3888       }
   3889       break;
   3890     case UNIFORM_DICT:
   3891       loop(dictCount) {
   3892         char *key  = (char*)*data;
   3893         *data     += strlen(key)+1;
   3894         d          = NULL;
   3895         uniformDictNetDeserialLevel2(&d, data, ctx, /*packed=*/true);
   3896         sDictPushTiny(dict, key, (smallt *) d);
   3897       }
   3898       break;
   3899     case UNIFORM_ARRAY:
   3900       loop(dictCount) {
   3901         char *key  = (char*)*data;
   3902         *data     += strlen(key)+1;
   3903         a          = NULL;
   3904         uniformArrayNetDeserialLevel2(&a, data, ctx, /*packed=*/true);
   3905         sDictPushTiny(dict, key, (smallt *) a);
   3906       }
   3907       break;
   3908   }
   3909 }
   3910 
   3911 /**
   3912  * deserialize array from data
   3913  *
   3914  * a new array is allocated
   3915  *
   3916  * \param
   3917  *    array holding the elements
   3918  *    data serialized dictionary
   3919  */
   3920 internal void arrayNetDeserialLevel2(sArrayt **array, u8 **data, contextt *ctx, bool packed) {
   3921   sUndefinedt *u = NULL;
   3922   sBoolt *bo = NULL;
   3923   double *D = NULL;
   3924   sDoublet *Do = NULL;
   3925   sDictt *d = NULL;
   3926   sIntt *io = NULL;
   3927   char *s = NULL;
   3928   sStringt *so = NULL;
   3929   sArrayt *a = NULL;
   3930   sBytest *B = NULL;
   3931   uint32_t count;
   3932   uint32_t arrayCount;
   3933 
   3934   if (packed) {
   3935     arrayCount = varintToUint(data);
   3936   }
   3937   else {
   3938     if (ctx->nibble == lowNbl) {
   3939       arrayCount = netTypeVarintToUint(data);
   3940     }
   3941     else {
   3942       // high nibble
   3943       readTypeInHighNbl;
   3944       arrayCount = varintToUint(data);
   3945     }
   3946   }
   3947 
   3948   if (!arrayCount) {
   3949     *array = allocSArray();;
   3950     ret;
   3951   }
   3952 
   3953   loop(arrayCount) {
   3954     char type;
   3955     if (ctx->nibble == lowNbl) {
   3956       type = (**data) & 0xF;
   3957     }
   3958     else {
   3959       // high nibble
   3960       type = (*ctx->nblAddr) >> 4;
   3961     }
   3962 
   3963     switch(type) {
   3964       case S_UNDEFINED:
   3965         if (ctx->nibble == lowNbl) {
   3966           readTypeOnly;
   3967         }
   3968         else {
   3969           // high nibble
   3970           readTypeInHighNbl;
   3971         }
   3972         u = allocSUndefined();
   3973         sArrayPushTiny(array, (smallt *) u);
   3974         break;
   3975       case S_BOOL:
   3976         if (!ctx->boolAddr) {
   3977           // new packed bools
   3978           if (ctx->nibble == lowNbl) {
   3979             read4bPackedBool;
   3980             bo = allocSBool((*ctx->boolAddr) & 0x10);
   3981           }
   3982           else {
   3983             // high nibble
   3984             readTypeInHighNbl;
   3985             read8bPackedBool;
   3986             bo = allocSBool((*ctx->boolAddr) & 0x1);
   3987           }
   3988         }
   3989         else {
   3990           // there was a bool before this one, read bits in nibbles
   3991           if (ctx->nibble == lowNbl) {
   3992             if (ctx->boolShift == 8) {
   3993               read4bPackedBool;
   3994               bo = allocSBool((*ctx->boolAddr) & 0x10);
   3995             }
   3996             else {
   3997               readTypeOnly;
   3998               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   3999             }
   4000           }
   4001           else {
   4002             // high nibble
   4003             readTypeInHighNbl;
   4004             if (ctx->boolShift == 8) {
   4005               read8bPackedBool;
   4006               bo = allocSBool((*ctx->boolAddr) & 0x1);
   4007             }
   4008             else {
   4009               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   4010             }
   4011           }
   4012         }
   4013         sArrayPushTiny(array, (smallt *) bo);
   4014         break;
   4015       case S_DICT:
   4016         d = NULL;
   4017         dictNetDeserialLevel2(&d, data, ctx, /*packed=*/false);
   4018         sArrayPushTiny(array, (smallt *) d);
   4019         break;
   4020       case S_DOUBLE:
   4021         if (ctx->nibble == lowNbl) {
   4022           readTypeOnly;
   4023         }
   4024         else {
   4025           // high nibble
   4026           readTypeInHighNbl;
   4027         }
   4028         D      = (double *)(*data);
   4029         *data += sizeof(double);
   4030         Do     = allocSDouble(*D);
   4031         sArrayPushTiny(array, (smallt *) Do);
   4032         break;
   4033       case S_INT: {
   4034           u64 v;
   4035           if (ctx->nibble == lowNbl) {
   4036             v = netTypeVarintToUint((u8**)data);
   4037           }
   4038           else {
   4039             // high nibble
   4040             readTypeInHighNbl;
   4041             v = varintToUint(data);
   4042           }
   4043           v  = (v >> 1) ^ (~(v & 1) + 1);
   4044           io = allocSInt(v);
   4045           sArrayPushTiny(array, (smallt *) io);
   4046         }
   4047         break;
   4048       case S_STRING:
   4049         if (ctx->nibble == lowNbl) {
   4050           readTypeOnly;
   4051         }
   4052         else {
   4053           // high nibble
   4054           readTypeInHighNbl;
   4055         }
   4056         s      = (char *)(*data);
   4057         *data += strlen(s)+1;
   4058         so     = allocSStringTiny(s);
   4059         sArrayPushTiny(array, (smallt *) so);
   4060         break;
   4061       case S_ARRAY:
   4062         a = NULL;
   4063         arrayNetDeserialLevel2(&a, data, ctx, /*packed=*/false);
   4064         sArrayPushTiny(array, (smallt *) a);
   4065         break;
   4066       case S_BYTES:
   4067         B      = allocSBytes();
   4068         if (ctx->nibble == lowNbl) {
   4069           count  = netTypeVarintToUint((u8**)data);
   4070         }
   4071         else {
   4072           // high nibble
   4073           readTypeInHighNbl;
   4074           count  = varintToUint((u8**)data);
   4075         }
   4076         sBytesPushBuffer(&B, *data, count);
   4077         *data += count;
   4078         sArrayPushTiny(array, (smallt *) B);
   4079         break;
   4080       case UNIFORM_DICT:
   4081         d = NULL;
   4082         uniformDictNetDeserialLevel2(&d, data, ctx, /*packed=*/false);
   4083         sArrayPushTiny(array, (smallt *) d);
   4084         break;
   4085       case UNIFORM_ARRAY:
   4086         a = NULL;
   4087         uniformArrayNetDeserialLevel2(&a, data, ctx, /*packed=*/false);
   4088         sArrayPushTiny(array, (smallt *) a);
   4089         break;
   4090     }
   4091   }
   4092 }
   4093 
   4094 internal void uniformArrayNetDeserialLevel2(sArrayt **array, u8 **data, contextt *ctx, bool packed) {
   4095   sUndefinedt *u = NULL;
   4096   sBoolt *bo = NULL;
   4097   double *D = NULL;
   4098   sDoublet *Do = NULL;
   4099   sDictt *d = NULL;
   4100   sIntt *io = NULL;
   4101   char *s = NULL;
   4102   sStringt *so = NULL;
   4103   sArrayt *a = NULL;
   4104   sBytest *B = NULL;
   4105   uint32_t count;
   4106   uint32_t arrayCount;
   4107   u8 type;
   4108 
   4109   if (packed) {
   4110     type = (**data) & 0xF;
   4111     arrayCount = netTypeVarintToUint(data);
   4112   }
   4113   else {
   4114     if (ctx->nibble == lowNbl) {
   4115       type = (**data) >> 4;
   4116       (*data)++;
   4117       arrayCount = varintToUint(data);
   4118     }
   4119     else {
   4120       readTypeInHighNbl;
   4121       type = (**data) & 0xF;
   4122       arrayCount = netTypeVarintToUint(data);
   4123     }
   4124   }
   4125 
   4126   if (!arrayCount) {
   4127     *array = allocSArray();;
   4128     ret;
   4129   }
   4130 
   4131   switch(type) {
   4132     case S_UNDEFINED:
   4133       loop(arrayCount) {
   4134         u = allocSUndefined();
   4135         sArrayPushTiny(array, (smallt *) u);
   4136       }
   4137       break;
   4138     case S_BOOL:
   4139       loop(arrayCount) {
   4140         if (!ctx->boolAddr) {
   4141           read8bPackedBool;
   4142           ctx->boolShift = 0;
   4143         }
   4144         if (ctx->boolShift == 8) {
   4145           read8bPackedBool;
   4146           bo = allocSBool((*ctx->boolAddr) & 0x1);
   4147         }
   4148         else {
   4149           bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   4150         }
   4151         sArrayPushTiny(array, (smallt *) bo);
   4152       }
   4153       break;
   4154     case S_DICT:
   4155       loop(arrayCount) {
   4156         d = NULL;
   4157         dictNetDeserialLevel2(&d, data, ctx, /*packed=*/true);
   4158         sArrayPushTiny(array, (smallt *) d);
   4159       }
   4160       break;
   4161     case S_DOUBLE:
   4162       loop(arrayCount) {
   4163         D      = (double *)(*data);
   4164         *data += sizeof(double);
   4165         Do     = allocSDouble(*D);
   4166         sArrayPushTiny(array, (smallt *) Do);
   4167       }
   4168       break;
   4169     case S_INT:
   4170       loop(arrayCount) {
   4171         u64 v = varintToUint(data);
   4172         v     = (v >> 1) ^ (~(v & 1) + 1);
   4173         io    = allocSInt(v);
   4174         sArrayPushTiny(array, (smallt *) io);
   4175       }
   4176       break;
   4177     case S_STRING:
   4178       loop(arrayCount) {
   4179         s      = (char *)(*data);
   4180         *data += strlen(s)+1;
   4181         so     = allocSStringTiny(s);
   4182         sArrayPushTiny(array, (smallt *) so);
   4183       }
   4184       break;
   4185     case S_ARRAY:
   4186       loop(arrayCount) {
   4187         a = NULL;
   4188         arrayNetDeserialLevel2(&a, data, ctx, /*packed=*/true);
   4189         sArrayPushTiny(array, (smallt *) a);
   4190       }
   4191       break;
   4192     case S_BYTES:
   4193       loop(arrayCount) {
   4194         B      = allocSBytes();
   4195         count  = varintToUint((u8**)data);
   4196         sBytesPushBuffer(&B, *data, count);
   4197         *data += count;
   4198         sArrayPushTiny(array, (smallt *) B);
   4199       }
   4200       break;
   4201     case UNIFORM_DICT:
   4202       loop(arrayCount) {
   4203         d = NULL;
   4204         uniformDictNetDeserialLevel2(&d, data, ctx, /*packed=*/true);
   4205         sArrayPushTiny(array, (smallt *) d);
   4206       }
   4207       break;
   4208     case UNIFORM_ARRAY:
   4209       loop(arrayCount) {
   4210         a = NULL;
   4211         uniformArrayNetDeserialLevel2(&a, data, ctx, /*packed=*/true);
   4212         sArrayPushTiny(array, (smallt *) a);
   4213       }
   4214       break;
   4215   }
   4216 }
   4217 
   4218 internal smallJsont* deserialNetSerialLevel2(smallJsont *self, smallBytest *data) {
   4219 
   4220   if (!data or !data->B or !data->B->count) {
   4221     ret self;
   4222   }
   4223 
   4224   smallt *o = netDeserialLevel2(data->B);
   4225 
   4226   if (!o) {
   4227     ret self;
   4228   }
   4229 
   4230   freeG(self);
   4231 
   4232   setsoG(self, o);
   4233 
   4234   ret self;
   4235 }
   4236 
   4237 // level 3
   4238 // like level 2, elements of identical type in a row are packed
   4239 
   4240 /**
   4241  * deserializer top function
   4242  */
   4243 internal smallt* netDeserial(sBytest *obj) {
   4244   smallt   *r = NULL;
   4245   double   *D = NULL;
   4246   char     *s = NULL;
   4247   sBytest  *B = NULL;
   4248   uint32_t count;
   4249   char     *data  = NULL;
   4250   contextt ctx = {.nibble=lowNbl, .nblAddr=NULL, .boolShift = 0, .boolAddr=NULL};
   4251 
   4252   switch(obj->data & 0xF) {
   4253     case S_UNDEFINED:
   4254       r = (smallt *) allocSUndefined();
   4255       break;
   4256     case S_BOOL:
   4257       r = (smallt *) allocSBool(obj->data & 0x10);
   4258       break;
   4259     case S_DICT:
   4260       data     = (char *)&(obj->data);
   4261       //debug - ctx.dbuf = (u8*) data;
   4262       dictNetDeserial((sDictt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   4263       break;
   4264     case S_DOUBLE:
   4265       data = &(obj->data)+1;
   4266       D = (double *)data;
   4267       r = (smallt *) allocSDouble(*D);
   4268       break;
   4269     case S_INT:
   4270       data  = &(obj->data);
   4271       u64 v = netTypeVarintToUint((u8**)&data);
   4272       v = (v >> 1) ^ (~(v & 1) + 1);
   4273       r = (smallt *) allocSInt(v);
   4274       break;
   4275     case S_STRING:
   4276       s = (char *)&(obj->data)+1;
   4277       r = (smallt *) allocSStringTiny(s);
   4278       break;
   4279     case S_ARRAY:
   4280       data = (char *)&(obj->data);
   4281       //debug - ctx.dbuf = (u8*) data;
   4282       arrayNetDeserial((sArrayt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   4283       break;
   4284     case S_BYTES:
   4285       B     = allocSBytes();
   4286       data  = &(obj->data);
   4287       count = netTypeVarintToUint((u8**)&data);
   4288       sBytesPushBuffer(&B, data, count);
   4289       r = (smallt *)B;
   4290       break;
   4291     case UNIFORM_DICT:
   4292       data     = (char *)&(obj->data);
   4293       //debug - ctx.dbuf = (u8*) data;
   4294       uniformDictNetDeserial((sDictt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   4295       break;
   4296     case UNIFORM_ARRAY:
   4297       data     = (char *)&(obj->data);
   4298       //debug - ctx.dbuf = (u8*) data;
   4299       uniformArrayNetDeserial((sArrayt **)&r, (u8**)&data, &ctx, /*packed=*/false);
   4300       break;
   4301   }
   4302 
   4303   ret r;
   4304 }
   4305 
   4306 /**
   4307  * deserialize dictionary from data
   4308  *
   4309  * a new dictionary is allocated
   4310  *
   4311  * \param
   4312  *    dict dictionary holding the elements
   4313  *    data serialized dictionary
   4314  */
   4315 internal void dictNetDeserial(sDictt **dict, u8 **data, contextt *ctx, bool packed) {
   4316   sUndefinedt *u = NULL;
   4317   sBoolt *bo = NULL;
   4318   double *D = NULL;
   4319   sDoublet *Do = NULL;
   4320   sDictt *d = NULL;
   4321   sIntt *io = NULL;
   4322   char *s = NULL;
   4323   sStringt *so = NULL;
   4324   sArrayt *a = NULL;
   4325   sBytest *B = NULL;
   4326   uint32_t count;
   4327   uint32_t dictCount;
   4328 
   4329   if (packed) {
   4330     dictCount = varintToUint(data);
   4331   }
   4332   else {
   4333     if (ctx->nibble == lowNbl) {
   4334       dictCount = netTypeVarintToUint(data);
   4335     }
   4336     else {
   4337       // high nibble
   4338       // type = *(ctx->dbuf + ctx->nblOffset) >> 4;
   4339       #define readTypeInHighNbl\
   4340       ctx->nibble  = lowNbl;\
   4341       if (ctx->nblAddr == *data)\
   4342       /* data points to the type, next byte is count */\
   4343       (*data)++
   4344       readTypeInHighNbl;
   4345       dictCount = varintToUint(data);
   4346     }
   4347   }
   4348 
   4349   if (!dictCount) {
   4350     *dict = allocSDict();
   4351     ret;
   4352   }
   4353 
   4354   bool inPack = false;
   4355   u8 packedType;
   4356   size_t packCount;
   4357   loop(dictCount) {
   4358     char *key  = (char*)*data;
   4359     *data     += strlen(key)+1;
   4360     char type;
   4361     if (inPack) {
   4362       type = packedType;
   4363     }
   4364     else {
   4365       if (ctx->nibble == lowNbl) {
   4366         type = (**data) & 0xF;
   4367       }
   4368       else {
   4369         // high nibble
   4370         type = (*ctx->nblAddr) >> 4;
   4371       }
   4372     }
   4373 
   4374     switch(type) {
   4375       case S_UNDEFINED:
   4376         if (ctx->nibble == lowNbl) {
   4377           #define readTypeOnly\
   4378           ctx->nibble  = highNbl;\
   4379           ctx->nblAddr = *data;\
   4380           (*data)++
   4381           readTypeOnly;
   4382         }
   4383         else {
   4384           // high nibble
   4385           readTypeInHighNbl;
   4386         }
   4387         u = allocSUndefined();
   4388         sDictPushTiny(dict, key, (smallt *) u);
   4389         break;
   4390       case S_BOOL:
   4391         if (!ctx->boolAddr) {
   4392           // new packed bools
   4393           if (ctx->nibble == lowNbl) {
   4394             #define read4bPackedBool\
   4395             ctx->boolShift  = 5;\
   4396             ctx->boolAddr   = *data;\
   4397             (*data)++
   4398             read4bPackedBool;
   4399             bo = allocSBool((*ctx->boolAddr) & 0x10);
   4400           }
   4401           else {
   4402             // high nibble
   4403             readTypeInHighNbl;
   4404             #define read8bPackedBool\
   4405             ctx->boolShift  = 1;\
   4406             ctx->boolAddr   = *data;\
   4407             (*data)++
   4408             read8bPackedBool;
   4409             bo = allocSBool((*ctx->boolAddr) & 0x1);
   4410           }
   4411         }
   4412         else {
   4413           // there was a bool before this one, read bits in nibbles
   4414           if (ctx->nibble == lowNbl) {
   4415             if (ctx->boolShift == 8) {
   4416               read4bPackedBool;
   4417               bo = allocSBool((*ctx->boolAddr) & 0x10);
   4418             }
   4419             else {
   4420               readTypeOnly;
   4421               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   4422             }
   4423           }
   4424           else {
   4425             // high nibble
   4426             readTypeInHighNbl;
   4427             if (ctx->boolShift == 8) {
   4428               read8bPackedBool;
   4429               bo = allocSBool((*ctx->boolAddr) & 0x1);
   4430             }
   4431             else {
   4432               // high nibble
   4433               readTypeInHighNbl;
   4434               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   4435             }
   4436           }
   4437         }
   4438         sDictPushTiny(dict, key, (smallt *) bo);
   4439         break;
   4440       case S_DICT:
   4441         d = NULL;
   4442         dictNetDeserial(&d, data, ctx, /*packed=*/false);
   4443         sDictPushTiny(dict, key, (smallt *) d);
   4444         break;
   4445       case S_DOUBLE:
   4446         if (ctx->nibble == lowNbl) {
   4447           readTypeOnly;
   4448         }
   4449         else {
   4450           // high nibble
   4451           readTypeInHighNbl;
   4452         }
   4453         D      = (double *)(*data);
   4454         *data += sizeof(double);
   4455         Do     = allocSDouble(*D);
   4456         sDictPushTiny(dict, key, (smallt *) Do);
   4457         break;
   4458       case S_INT: {
   4459           u64 v;
   4460           if (ctx->nibble == lowNbl) {
   4461             v = netTypeVarintToUint((u8**)data);
   4462           }
   4463           else {
   4464             // high nibble
   4465             readTypeInHighNbl;
   4466             v = varintToUint(data);
   4467           }
   4468           v  = (v >> 1) ^ (~(v & 1) + 1);
   4469           io = allocSInt(v);
   4470           sDictPushTiny(dict, key, (smallt *) io);
   4471         }
   4472         break;
   4473       case S_STRING:
   4474         if (ctx->nibble == lowNbl) {
   4475           readTypeOnly;
   4476         }
   4477         else {
   4478           // high nibble
   4479           readTypeInHighNbl;
   4480         }
   4481         s      = (char *)(*data);
   4482         *data += strlen(s)+1;
   4483         so     = allocSStringTiny(s);
   4484         sDictPushTiny(dict, key, (smallt *) so);
   4485         break;
   4486       case S_ARRAY:
   4487         a = NULL;
   4488         arrayNetDeserial(&a, data, ctx, /*packed=*/false);
   4489         sDictPushTiny(dict, key, (smallt *) a);
   4490         break;
   4491       case S_BYTES:
   4492         B      = allocSBytes();
   4493         if (ctx->nibble == lowNbl) {
   4494           count  = netTypeVarintToUint((u8**)data);
   4495         }
   4496         else {
   4497           // high nibble
   4498           readTypeInHighNbl;
   4499           count  = varintToUint((u8**)data);
   4500         }
   4501         sBytesPushBuffer(&B, *data, count);
   4502         *data += count;
   4503         sDictPushTiny(dict, key, (smallt *) B);
   4504         break;
   4505       case PK_DICT:
   4506         if (!inPack) {
   4507           inPack = true;
   4508           if (ctx->nibble == lowNbl) {
   4509             packCount  = netTypeVarintToUint((u8**)data);
   4510           }
   4511           else {
   4512             // high nibble
   4513             readTypeInHighNbl;
   4514             packCount  = varintToUint((u8**)data);
   4515           }
   4516           packedType = PK_DICT;
   4517         }
   4518 
   4519         d = NULL;
   4520         dictNetDeserial(&d, data, ctx, /*packed=*/true);
   4521         sDictPushTiny(dict, key, (smallt *) d);
   4522         // stop unpacking when packCount == 0
   4523         packCount--;
   4524         if (!packCount) inPack = false;
   4525         break;
   4526       case PK_DOUBLE:
   4527         if (!inPack) {
   4528           inPack = true;
   4529           if (ctx->nibble == lowNbl) {
   4530             packCount  = netTypeVarintToUint((u8**)data);
   4531           }
   4532           else {
   4533             // high nibble
   4534             readTypeInHighNbl;
   4535             packCount  = varintToUint((u8**)data);
   4536           }
   4537           packedType = PK_DOUBLE;
   4538         }
   4539 
   4540         D      = (double *)(*data);
   4541         *data += sizeof(double);
   4542         Do     = allocSDouble(*D);
   4543         sDictPushTiny(dict, key, (smallt *) Do);
   4544         // stop unpacking when packCount == 0
   4545         packCount--;
   4546         if (!packCount) inPack = false;
   4547         break;
   4548       case PK_INT:
   4549         if (!inPack) {
   4550           inPack = true;
   4551           if (ctx->nibble == lowNbl) {
   4552             packCount  = netTypeVarintToUint((u8**)data);
   4553           }
   4554           else {
   4555             // high nibble
   4556             readTypeInHighNbl;
   4557             packCount  = varintToUint((u8**)data);
   4558           }
   4559           packedType = PK_INT;
   4560         }
   4561 
   4562         u64 v = varintToUint(data);
   4563         v     = (v >> 1) ^ (~(v & 1) + 1);
   4564         io    = allocSInt(v);
   4565         sDictPushTiny(dict, key, (smallt *) io);
   4566         // stop unpacking when packCount == 0
   4567         packCount--;
   4568         if (!packCount) inPack = false;
   4569         break;
   4570       case PK_STRING:
   4571         if (!inPack) {
   4572           inPack = true;
   4573           if (ctx->nibble == lowNbl) {
   4574             packCount  = netTypeVarintToUint((u8**)data);
   4575           }
   4576           else {
   4577             // high nibble
   4578             readTypeInHighNbl;
   4579             packCount  = varintToUint((u8**)data);
   4580           }
   4581           packedType = PK_STRING;
   4582         }
   4583 
   4584         s          = (char *)(*data);
   4585         *data     += strlen(s)+1;
   4586         so         = allocSStringTiny(s);
   4587         sDictPushTiny(dict, key, (smallt *) so);
   4588         // stop unpacking when packCount == 0
   4589         packCount--;
   4590         if (!packCount) inPack = false;
   4591         break;
   4592       case PK_ARRAY:
   4593         if (!inPack) {
   4594           inPack = true;
   4595           if (ctx->nibble == lowNbl) {
   4596             packCount  = netTypeVarintToUint((u8**)data);
   4597           }
   4598           else {
   4599             // high nibble
   4600             readTypeInHighNbl;
   4601             packCount  = varintToUint((u8**)data);
   4602           }
   4603           packedType = PK_ARRAY;
   4604         }
   4605 
   4606         a          = NULL;
   4607         arrayNetDeserial(&a, data, ctx, /*packed=*/true);
   4608         sDictPushTiny(dict, key, (smallt *) a);
   4609         // stop unpacking when packCount == 0
   4610         packCount--;
   4611         if (!packCount) inPack = false;
   4612         break;
   4613       case PK_BYTES:
   4614         if (!inPack) {
   4615           inPack = true;
   4616           if (ctx->nibble == lowNbl) {
   4617             packCount  = netTypeVarintToUint((u8**)data);
   4618           }
   4619           else {
   4620             // high nibble
   4621             readTypeInHighNbl;
   4622             packCount  = varintToUint((u8**)data);
   4623           }
   4624           packedType = PK_BYTES;
   4625         }
   4626 
   4627         B          = allocSBytes();
   4628         count      = varintToUint((u8**)data);
   4629         sBytesPushBuffer(&B, *data, count);
   4630         *data     += count;
   4631         sDictPushTiny(dict, key, (smallt *) B);
   4632         // stop unpacking when packCount == 0
   4633         packCount--;
   4634         if (!packCount) inPack = false;
   4635         break;
   4636       case UNIFORM_DICT:
   4637         d = NULL;
   4638         uniformDictNetDeserial(&d, data, ctx, /*packed=*/false);
   4639         sDictPushTiny(dict, key, (smallt *) d);
   4640         break;
   4641       case UNIFORM_ARRAY:
   4642         a = NULL;
   4643         uniformArrayNetDeserial(&a, data, ctx, /*packed=*/false);
   4644         sDictPushTiny(dict, key, (smallt *) a);
   4645         break;
   4646     }
   4647   }
   4648 }
   4649 
   4650 /**
   4651  * deserialize dictionary from data
   4652  *
   4653  * a new dictionary is allocated
   4654  *
   4655  * \param
   4656  *    dict dictionary holding the elements
   4657  *    data serialized dictionary
   4658  */
   4659 internal void uniformDictNetDeserial(sDictt **dict, u8 **data, contextt *ctx, bool packed) {
   4660   sUndefinedt *u = NULL;
   4661   sBoolt *bo = NULL;
   4662   double *D = NULL;
   4663   sDoublet *Do = NULL;
   4664   sDictt *d = NULL;
   4665   sIntt *io = NULL;
   4666   char *s = NULL;
   4667   sStringt *so = NULL;
   4668   sArrayt *a = NULL;
   4669   sBytest *B = NULL;
   4670   uint32_t count;
   4671   uint32_t dictCount;
   4672   u8 type;
   4673 
   4674   if (packed) {
   4675     type = (**data) & 0xF;
   4676     dictCount = netTypeVarintToUint(data);
   4677   }
   4678   else {
   4679     if (ctx->nibble == lowNbl) {
   4680       type = (**data) >> 4;
   4681       (*data)++;
   4682       dictCount = varintToUint(data);
   4683     }
   4684     else {
   4685       readTypeInHighNbl;
   4686       type = (**data) & 0xF;
   4687       dictCount = netTypeVarintToUint(data);
   4688     }
   4689   }
   4690 
   4691   if (!dictCount) {
   4692     *dict = allocSDict();
   4693     ret;
   4694   }
   4695 
   4696 
   4697   switch(type) {
   4698     case S_UNDEFINED:
   4699       loop(dictCount) {
   4700         char *key  = (char*)*data;
   4701         *data     += strlen(key)+1;
   4702         u = allocSUndefined();
   4703         sDictPushTiny(dict, key, (smallt *) u);
   4704       }
   4705       break;
   4706     case S_BOOL:
   4707       loop(dictCount) {
   4708         char *key  = (char*)*data;
   4709         *data     += strlen(key)+1;
   4710         if (!ctx->boolAddr) {
   4711           read8bPackedBool;
   4712           ctx->boolShift = 0;
   4713         }
   4714         if (ctx->boolShift == 8) {
   4715           readTypeInHighNbl;
   4716           read8bPackedBool;
   4717           bo = allocSBool((*ctx->boolAddr) & 0x1);
   4718         }
   4719         else {
   4720           // high nibble
   4721           readTypeInHighNbl;
   4722           bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   4723         }
   4724         sDictPushTiny(dict, key, (smallt *) bo);
   4725       }
   4726       break;
   4727     case S_DICT:
   4728       loop(dictCount) {
   4729         char *key  = (char*)*data;
   4730         *data     += strlen(key)+1;
   4731         d          = NULL;
   4732         dictNetDeserial(&d, data, ctx, /*packed=*/true);
   4733         sDictPushTiny(dict, key, (smallt *) d);
   4734       }
   4735       break;
   4736     case S_DOUBLE:
   4737       loop(dictCount) {
   4738         char *key  = (char*)*data;
   4739         *data     += strlen(key)+1;
   4740         D          = (double *)(*data);
   4741         *data     += sizeof(double);
   4742         Do         = allocSDouble(*D);
   4743         sDictPushTiny(dict, key, (smallt *) Do);
   4744       }
   4745       break;
   4746     case S_INT:
   4747       loop(dictCount) {
   4748         char *key  = (char*)*data;
   4749         *data     += strlen(key)+1;
   4750         u64 v      = varintToUint(data);
   4751         v          = (v >> 1) ^ (~(v & 1) + 1);
   4752         io         = allocSInt(v);
   4753         sDictPushTiny(dict, key, (smallt *) io);
   4754       }
   4755       break;
   4756     case S_STRING:
   4757       loop(dictCount) {
   4758         char *key  = (char*)*data;
   4759         *data     += strlen(key)+1;
   4760         s          = (char *)(*data);
   4761         *data     += strlen(s)+1;
   4762         so         = allocSStringTiny(s);
   4763         sDictPushTiny(dict, key, (smallt *) so);
   4764       }
   4765       break;
   4766     case S_ARRAY:
   4767       loop(dictCount) {
   4768         char *key  = (char*)*data;
   4769         *data     += strlen(key)+1;
   4770         a          = NULL;
   4771         arrayNetDeserial(&a, data, ctx, /*packed=*/true);
   4772         sDictPushTiny(dict, key, (smallt *) a);
   4773       }
   4774       break;
   4775     case S_BYTES:
   4776       loop(dictCount) {
   4777         char *key  = (char*)*data;
   4778         *data     += strlen(key)+1;
   4779         B          = allocSBytes();
   4780         count      = varintToUint((u8**)data);
   4781         sBytesPushBuffer(&B, *data, count);
   4782         *data     += count;
   4783         sDictPushTiny(dict, key, (smallt *) B);
   4784       }
   4785       break;
   4786     case UNIFORM_DICT:
   4787       loop(dictCount) {
   4788         char *key  = (char*)*data;
   4789         *data     += strlen(key)+1;
   4790         d          = NULL;
   4791         uniformDictNetDeserial(&d, data, ctx, /*packed=*/true);
   4792         sDictPushTiny(dict, key, (smallt *) d);
   4793       }
   4794       break;
   4795     case UNIFORM_ARRAY:
   4796       loop(dictCount) {
   4797         char *key  = (char*)*data;
   4798         *data     += strlen(key)+1;
   4799         a          = NULL;
   4800         uniformArrayNetDeserial(&a, data, ctx, /*packed=*/true);
   4801         sDictPushTiny(dict, key, (smallt *) a);
   4802       }
   4803       break;
   4804   }
   4805 }
   4806 
   4807 /**
   4808  * deserialize array from data
   4809  *
   4810  * a new array is allocated
   4811  *
   4812  * \param
   4813  *    array holding the elements
   4814  *    data serialized dictionary
   4815  */
   4816 internal void arrayNetDeserial(sArrayt **array, u8 **data, contextt *ctx, bool packed) {
   4817   sUndefinedt *u = NULL;
   4818   sBoolt *bo = NULL;
   4819   double *D = NULL;
   4820   sDoublet *Do = NULL;
   4821   sDictt *d = NULL;
   4822   sIntt *io = NULL;
   4823   char *s = NULL;
   4824   sStringt *so = NULL;
   4825   sArrayt *a = NULL;
   4826   sBytest *B = NULL;
   4827   uint32_t count;
   4828   uint32_t arrayCount;
   4829 
   4830   if (packed) {
   4831     arrayCount = varintToUint(data);
   4832   }
   4833   else {
   4834     if (ctx->nibble == lowNbl) {
   4835       arrayCount = netTypeVarintToUint(data);
   4836     }
   4837     else {
   4838       // high nibble
   4839       readTypeInHighNbl;
   4840       arrayCount = varintToUint(data);
   4841     }
   4842   }
   4843 
   4844   if (!arrayCount) {
   4845     *array = allocSArray();;
   4846     ret;
   4847   }
   4848 
   4849   bool inPack = false;
   4850   u8 packedType;
   4851   size_t packCount;
   4852   loop(arrayCount) {
   4853     char type;
   4854     if (inPack) {
   4855       type = packedType;
   4856     }
   4857     else {
   4858       if (ctx->nibble == lowNbl) {
   4859         type = (**data) & 0xF;
   4860       }
   4861       else {
   4862         // high nibble
   4863         type = (*ctx->nblAddr) >> 4;
   4864       }
   4865     }
   4866 
   4867     switch(type) {
   4868       case S_UNDEFINED:
   4869         if (ctx->nibble == lowNbl) {
   4870           readTypeOnly;
   4871         }
   4872         else {
   4873           // high nibble
   4874           readTypeInHighNbl;
   4875         }
   4876         u = allocSUndefined();
   4877         sArrayPushTiny(array, (smallt *) u);
   4878         break;
   4879       case S_BOOL:
   4880         if (!ctx->boolAddr) {
   4881           // new packed bools
   4882           if (ctx->nibble == lowNbl) {
   4883             read4bPackedBool;
   4884             bo = allocSBool((*ctx->boolAddr) & 0x10);
   4885           }
   4886           else {
   4887             // high nibble
   4888             readTypeInHighNbl;
   4889             read8bPackedBool;
   4890             bo = allocSBool((*ctx->boolAddr) & 0x1);
   4891           }
   4892         }
   4893         else {
   4894           // there was a bool before this one, read bits in nibbles
   4895           if (ctx->nibble == lowNbl) {
   4896             if (ctx->boolShift == 8) {
   4897               read4bPackedBool;
   4898               bo = allocSBool((*ctx->boolAddr) & 0x10);
   4899             }
   4900             else {
   4901               readTypeOnly;
   4902               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   4903             }
   4904           }
   4905           else {
   4906             // high nibble
   4907             readTypeInHighNbl;
   4908             if (ctx->boolShift == 8) {
   4909               read8bPackedBool;
   4910               bo = allocSBool((*ctx->boolAddr) & 0x1);
   4911             }
   4912             else {
   4913               bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   4914             }
   4915           }
   4916         }
   4917         sArrayPushTiny(array, (smallt *) bo);
   4918         break;
   4919       case S_DICT:
   4920         d = NULL;
   4921         dictNetDeserial(&d, data, ctx, /*packed=*/false);
   4922         sArrayPushTiny(array, (smallt *) d);
   4923         break;
   4924       case S_DOUBLE:
   4925         if (ctx->nibble == lowNbl) {
   4926           readTypeOnly;
   4927         }
   4928         else {
   4929           // high nibble
   4930           readTypeInHighNbl;
   4931         }
   4932         D      = (double *)(*data);
   4933         *data += sizeof(double);
   4934         Do     = allocSDouble(*D);
   4935         sArrayPushTiny(array, (smallt *) Do);
   4936         break;
   4937       case S_INT: {
   4938           u64 v;
   4939           if (ctx->nibble == lowNbl) {
   4940             v = netTypeVarintToUint((u8**)data);
   4941           }
   4942           else {
   4943             // high nibble
   4944             readTypeInHighNbl;
   4945             v = varintToUint(data);
   4946           }
   4947           v  = (v >> 1) ^ (~(v & 1) + 1);
   4948           io = allocSInt(v);
   4949           sArrayPushTiny(array, (smallt *) io);
   4950         }
   4951         break;
   4952       case S_STRING:
   4953         if (ctx->nibble == lowNbl) {
   4954           readTypeOnly;
   4955         }
   4956         else {
   4957           // high nibble
   4958           readTypeInHighNbl;
   4959         }
   4960         s      = (char *)(*data);
   4961         *data += strlen(s)+1;
   4962         so     = allocSStringTiny(s);
   4963         sArrayPushTiny(array, (smallt *) so);
   4964         break;
   4965       case S_ARRAY:
   4966         a = NULL;
   4967         arrayNetDeserial(&a, data, ctx, /*packed=*/false);
   4968         sArrayPushTiny(array, (smallt *) a);
   4969         break;
   4970       case S_BYTES:
   4971         B      = allocSBytes();
   4972         if (ctx->nibble == lowNbl) {
   4973           count  = netTypeVarintToUint((u8**)data);
   4974         }
   4975         else {
   4976           // high nibble
   4977           readTypeInHighNbl;
   4978           count  = varintToUint((u8**)data);
   4979         }
   4980         sBytesPushBuffer(&B, *data, count);
   4981         *data += count;
   4982         sArrayPushTiny(array, (smallt *) B);
   4983         break;
   4984       case PK_DICT:
   4985         if (!inPack) {
   4986           inPack = true;
   4987           if (ctx->nibble == lowNbl) {
   4988             packCount  = netTypeVarintToUint((u8**)data);
   4989           }
   4990           else {
   4991             // high nibble
   4992             readTypeInHighNbl;
   4993             packCount  = varintToUint((u8**)data);
   4994           }
   4995           packedType = PK_DICT;
   4996         }
   4997 
   4998         d = NULL;
   4999         dictNetDeserial(&d, data, ctx, /*packed=*/true);
   5000         sArrayPushTiny(array, (smallt *) d);
   5001         // stop unpacking when packCount == 0
   5002         packCount--;
   5003         if (!packCount) inPack = false;
   5004         break;
   5005       case PK_DOUBLE:
   5006         if (!inPack) {
   5007           inPack = true;
   5008           if (ctx->nibble == lowNbl) {
   5009             packCount  = netTypeVarintToUint((u8**)data);
   5010           }
   5011           else {
   5012             // high nibble
   5013             readTypeInHighNbl;
   5014             packCount  = varintToUint((u8**)data);
   5015           }
   5016           packedType = PK_DOUBLE;
   5017         }
   5018 
   5019         D      = (double *)(*data);
   5020         *data += sizeof(double);
   5021         Do     = allocSDouble(*D);
   5022         sArrayPushTiny(array, (smallt *) Do);
   5023         // stop unpacking when packCount == 0
   5024         packCount--;
   5025         if (!packCount) inPack = false;
   5026         break;
   5027       case PK_INT:
   5028         if (!inPack) {
   5029           inPack = true;
   5030           if (ctx->nibble == lowNbl) {
   5031             packCount  = netTypeVarintToUint((u8**)data);
   5032           }
   5033           else {
   5034             // high nibble
   5035             readTypeInHighNbl;
   5036             packCount  = varintToUint((u8**)data);
   5037           }
   5038           packedType = PK_INT;
   5039         }
   5040 
   5041         u64 v = varintToUint(data);
   5042         v     = (v >> 1) ^ (~(v & 1) + 1);
   5043         io     = allocSInt(v);
   5044         sArrayPushTiny(array, (smallt *) io);
   5045         // stop unpacking when packCount == 0
   5046         packCount--;
   5047         if (!packCount) inPack = false;
   5048         break;
   5049       case PK_STRING:
   5050         if (!inPack) {
   5051           inPack = true;
   5052           if (ctx->nibble == lowNbl) {
   5053             packCount  = netTypeVarintToUint((u8**)data);
   5054           }
   5055           else {
   5056             // high nibble
   5057             readTypeInHighNbl;
   5058             packCount  = varintToUint((u8**)data);
   5059           }
   5060           packedType = PK_STRING;
   5061         }
   5062 
   5063         s      = (char *)(*data);
   5064         *data += strlen(s)+1;
   5065         so     = allocSStringTiny(s);
   5066         sArrayPushTiny(array, (smallt *) so);
   5067         // stop unpacking when packCount == 0
   5068         packCount--;
   5069         if (!packCount) inPack = false;
   5070         break;
   5071       case PK_ARRAY:
   5072         if (!inPack) {
   5073           inPack = true;
   5074           if (ctx->nibble == lowNbl) {
   5075             packCount  = netTypeVarintToUint((u8**)data);
   5076           }
   5077           else {
   5078             // high nibble
   5079             readTypeInHighNbl;
   5080             packCount  = varintToUint((u8**)data);
   5081           }
   5082           packedType = PK_ARRAY;
   5083         }
   5084 
   5085         a = NULL;
   5086         arrayNetDeserial(&a, data, ctx, /*packed=*/true);
   5087         sArrayPushTiny(array, (smallt *) a);
   5088         // stop unpacking when packCount == 0
   5089         packCount--;
   5090         if (!packCount) inPack = false;
   5091         break;
   5092       case PK_BYTES:
   5093         if (!inPack) {
   5094           inPack = true;
   5095           if (ctx->nibble == lowNbl) {
   5096             packCount  = netTypeVarintToUint((u8**)data);
   5097           }
   5098           else {
   5099             // high nibble
   5100             readTypeInHighNbl;
   5101             packCount  = varintToUint((u8**)data);
   5102           }
   5103           packedType = PK_BYTES;
   5104         }
   5105 
   5106         B      = allocSBytes();
   5107         count  = varintToUint((u8**)data);
   5108         sBytesPushBuffer(&B, *data, count);
   5109         *data += count;
   5110         sArrayPushTiny(array, (smallt *) B);
   5111         // stop unpacking when packCount == 0
   5112         packCount--;
   5113         if (!packCount) inPack = false;
   5114         break;
   5115       case UNIFORM_DICT:
   5116         d = NULL;
   5117         uniformDictNetDeserial(&d, data, ctx, /*packed=*/false);
   5118         sArrayPushTiny(array, (smallt *) d);
   5119         break;
   5120       case UNIFORM_ARRAY:
   5121         a = NULL;
   5122         uniformArrayNetDeserial(&a, data, ctx, /*packed=*/false);
   5123         sArrayPushTiny(array, (smallt *) a);
   5124         break;
   5125     }
   5126   }
   5127 }
   5128 
   5129 internal void uniformArrayNetDeserial(sArrayt **array, u8 **data, contextt *ctx, bool packed) {
   5130   sUndefinedt *u = NULL;
   5131   sBoolt *bo = NULL;
   5132   double *D = NULL;
   5133   sDoublet *Do = NULL;
   5134   sDictt *d = NULL;
   5135   sIntt *io = NULL;
   5136   char *s = NULL;
   5137   sStringt *so = NULL;
   5138   sArrayt *a = NULL;
   5139   sBytest *B = NULL;
   5140   uint32_t count;
   5141   uint32_t arrayCount;
   5142   u8 type;
   5143 
   5144   if (packed) {
   5145     type = (**data) & 0xF;
   5146     arrayCount = netTypeVarintToUint(data);
   5147   }
   5148   else {
   5149     if (ctx->nibble == lowNbl) {
   5150       type = (**data) >> 4;
   5151       (*data)++;
   5152       arrayCount = varintToUint(data);
   5153     }
   5154     else {
   5155       readTypeInHighNbl;
   5156       type = (**data) & 0xF;
   5157       arrayCount = netTypeVarintToUint(data);
   5158     }
   5159   }
   5160 
   5161   if (!arrayCount) {
   5162     *array = allocSArray();;
   5163     ret;
   5164   }
   5165 
   5166   switch(type) {
   5167     case S_UNDEFINED:
   5168       loop(arrayCount) {
   5169         u = allocSUndefined();
   5170         sArrayPushTiny(array, (smallt *) u);
   5171       }
   5172       break;
   5173     case S_BOOL:
   5174       loop(arrayCount) {
   5175         if (!ctx->boolAddr) {
   5176           read8bPackedBool;
   5177           ctx->boolShift = 0;
   5178         }
   5179         if (ctx->boolShift == 8) {
   5180           read8bPackedBool;
   5181           bo = allocSBool((*ctx->boolAddr) & 0x1);
   5182         }
   5183         else {
   5184           bo = allocSBool((*ctx->boolAddr) & (0x1 << (ctx->boolShift++)));
   5185         }
   5186         sArrayPushTiny(array, (smallt *) bo);
   5187       }
   5188       break;
   5189     case S_DICT:
   5190       loop(arrayCount) {
   5191         d = NULL;
   5192         dictNetDeserial(&d, data, ctx, /*packed=*/true);
   5193         sArrayPushTiny(array, (smallt *) d);
   5194       }
   5195       break;
   5196     case S_DOUBLE:
   5197       loop(arrayCount) {
   5198         D      = (double *)(*data);
   5199         *data += sizeof(double);
   5200         Do     = allocSDouble(*D);
   5201         sArrayPushTiny(array, (smallt *) Do);
   5202       }
   5203       break;
   5204     case S_INT:
   5205       loop(arrayCount) {
   5206         u64 v = varintToUint(data);
   5207         v     = (v >> 1) ^ (~(v & 1) + 1);
   5208         io    = allocSInt(v);
   5209         sArrayPushTiny(array, (smallt *) io);
   5210       }
   5211       break;
   5212     case S_STRING:
   5213       loop(arrayCount) {
   5214         s      = (char *)(*data);
   5215         *data += strlen(s)+1;
   5216         so     = allocSStringTiny(s);
   5217         sArrayPushTiny(array, (smallt *) so);
   5218       }
   5219       break;
   5220     case S_ARRAY:
   5221       loop(arrayCount) {
   5222         a = NULL;
   5223         arrayNetDeserial(&a, data, ctx, /*packed=*/true);
   5224         sArrayPushTiny(array, (smallt *) a);
   5225       }
   5226       break;
   5227     case S_BYTES:
   5228       loop(arrayCount) {
   5229         B      = allocSBytes();
   5230         count  = varintToUint((u8**)data);
   5231         sBytesPushBuffer(&B, *data, count);
   5232         *data += count;
   5233         sArrayPushTiny(array, (smallt *) B);
   5234       }
   5235       break;
   5236     case UNIFORM_DICT:
   5237       loop(arrayCount) {
   5238         d = NULL;
   5239         uniformDictNetDeserial(&d, data, ctx, /*packed=*/true);
   5240         sArrayPushTiny(array, (smallt *) d);
   5241       }
   5242       break;
   5243     case UNIFORM_ARRAY:
   5244       loop(arrayCount) {
   5245         a = NULL;
   5246         uniformArrayNetDeserial(&a, data, ctx, /*packed=*/true);
   5247         sArrayPushTiny(array, (smallt *) a);
   5248       }
   5249       break;
   5250   }
   5251 }
   5252 
   5253 internal smallJsont* deserialNetSerial(smallJsont *self, smallBytest *data) {
   5254 
   5255   if (!data or !data->B or !data->B->count) {
   5256     ret self;
   5257   }
   5258 
   5259   smallt *o = netDeserial(data->B);
   5260 
   5261   if (!o) {
   5262     ret self;
   5263   }
   5264 
   5265   freeG(self);
   5266 
   5267   setsoG(self, o);
   5268 
   5269   ret self;
   5270 }
   5271 
   5272 // vim: set expandtab ts=2 sw=2:
   5273 
   5274 bool checkLibsheepyVersionNetSerial(const char *currentLibsheepyVersion) {
   5275   return eqG(currentLibsheepyVersion, LIBSHEEPY_VERSION);
   5276 }
   5277
	netSerial serializer producing compact bitstreams
	git clone https://noulin.net/git/netSerial.git
	Log \| Files \| Refs \| README \| LICENSE