blockFile

Save data in chunked in blocks in a file for creating databases
git clone https://noulin.net/git/blockFile.git
Log | Files | Refs | LICENSE

blockFile.c (16801B)


      1 
      2 
      3 /* Add class methods and modify the base functions (free, duplicate, ...) where there are the TODOs (TODO)*/
      4 
      5 #include "libsheepyObject.h"
      6 #include "blockFile.h"
      7 #include "blockFileInternal.h"
      8 #include "shpPackages/lz4/lz4.h"
      9 
     10 #include <stdlib.h>
     11 #include <string.h>
     12 #include <stdio.h>
     13 
     14 void initiateBlockFile(blockFilet *self);
     15 void registerMethodsBlockFile(blockFileFunctionst *f);
     16 void initiateAllocateBlockFile(blockFilet **self);
     17 void finalizeBlockFile(void);
     18 blockFilet* allocBlockFile(char *filename);
     19 internal void freeBlockFile(blockFilet *self);
     20 internal void terminateBlockFile(blockFilet **self);
     21 internal char* toStringBlockFile(blockFilet *self);
     22 internal blockFilet* duplicateBlockFile(blockFilet *self);
     23 internal void smashBlockFile(blockFilet **self);
     24 internal void finishBlockFile(blockFilet **self);
     25 internal bool openBlockFile(blockFilet *self, const char *filename);
     26 internal void closeBlockFile(blockFilet *self);
     27 internal void deleteBlockFile(blockFilet *self);
     28 internal void deleteFBlockFile(char *filename);
     29 internal u64  addBlockBlockFile(blockFilet *self, void* buf, i64 len, flagsBlockFilet flags);
     30 internal u64  addBlockFile(blockFilet *self, void* buf, i64 len);
     31 internal bufBlockFilet getBlockFile(blockFilet *self, u64 block);
     32 internal bool removeBlockFile(blockFilet *self, u64 block);
     33 internal bool loadBlockFile(blockFilet *self, void *closure, loadFBlockFileFt callback);
     34 /* TODO add prototypes */
     35 
     36 void initiateBlockFile(blockFilet *self) {
     37 
     38   self->type = "blockFile";
     39   if (!blockFileF) {
     40     blockFileF            = malloc(sizeof(blockFileFunctionst));
     41     registerMethodsBlockFile(blockFileF);
     42     pErrorNot0(atexit(finalizeBlockFile));
     43   }
     44   self->f    = blockFileF;
     45   self->file = malloc(sizeof(privateBlockFilet));
     46   // no database opened: no name
     47   fil.name   = NULL;
     48   /* TODO Initialize object data */
     49 }
     50 
     51 void registerMethodsBlockFile(blockFileFunctionst *f) {
     52 
     53   f->free      = freeBlockFile;
     54   f->terminate = terminateBlockFile;
     55   f->toString  = toStringBlockFile;
     56   f->duplicate = duplicateBlockFile;
     57   f->smash     = smashBlockFile;
     58   f->finish    = finishBlockFile;
     59   f->open      = openBlockFile;
     60   f->close     = closeBlockFile;
     61   f->delete    = deleteBlockFile;
     62   f->deleteF   = deleteFBlockFile;
     63   f->addBlock  = addBlockBlockFile;
     64   f->add       = addBlockFile;
     65   f->get       = getBlockFile;
     66   f->remove    = removeBlockFile;
     67   f->load      = loadBlockFile;
     68   /* TODO add class functions */
     69 }
     70 
     71 void initiateAllocateBlockFile(blockFilet **self) {
     72 
     73   if (self) {
     74     (*self) = malloc(sizeof(blockFilet));
     75     if (*self) {
     76       initiateBlockFile(*self);
     77     }
     78   }
     79 }
     80 
     81 void finalizeBlockFile(void) {
     82 
     83   if (blockFileF) {
     84     free(blockFileF);
     85     blockFileF = NULL;
     86   }
     87 }
     88 
     89 blockFilet* allocBlockFile(char* filename) {
     90   blockFilet *r = NULL;
     91 
     92   initiateAllocateBlockFile(&r);
     93 
     94   if (!openBlockFile(r, filename)) {
     95     terminateBlockFile(&r);
     96   }
     97   return(r);
     98 }
     99 
    100 
    101 internal void freeBlockFile(blockFilet *self) {
    102 
    103   if (fil.name) {
    104     closeBlockFile(self);
    105   }
    106   free(self->file);
    107   /* TODO free internal data (not the structure holding the function pointers) */
    108   return;
    109 }
    110 
    111 internal void terminateBlockFile(blockFilet **self) {
    112 
    113   freeBlockFile(*self);
    114   free(*self);
    115   *self = NULL;
    116 }
    117 
    118 
    119 internal char* toStringBlockFile(blockFilet UNUSED *self) {
    120 
    121   /* TODO convert object data to string */
    122   return(strdup("TODO - blockFile"));
    123 }
    124 
    125 internal blockFilet* duplicateBlockFile(blockFilet UNUSED *self) {
    126 
    127   createAllocateBlockFile(dup);
    128   /* TODO COPY data */
    129   return(dup);
    130 }
    131 
    132 internal void smashBlockFile(blockFilet **self) {
    133 
    134   finishBlockFile(self);
    135 }
    136 
    137 internal void finishBlockFile(blockFilet **self) {
    138 
    139   free(*self);
    140   *self = NULL;
    141 }
    142 
    143 internal bool openBlockFile(blockFilet *self, const char *filename) {
    144   if (!filename || isBlankS(filename))
    145     return false;
    146 
    147   bool dbExists  = true;
    148 
    149   // init
    150   fil.name         = strdup(filename);
    151   fil.blockSize    = BLOCKSIZE;
    152   fil.freeName     = catS(filename, "x");
    153   fil.defaultFlags = COMPRESSED;
    154 
    155   // check if db file exists
    156   if (!fileExists(filename)) {
    157     dbExists = false;
    158 
    159     // save info in block 0
    160     block0 b0;
    161     b0.blockSize      = fil.blockSize;
    162     b0.blockIndexSize = 0;
    163 
    164     char blockBuf[BLOCKSIZE];
    165 
    166     memcpy(blockBuf, &b0, sizeof b0);
    167 
    168     pError0(writeFile(filename, blockBuf, fil.blockSize));
    169   }
    170   fil.f = fopen(filename, "r+");
    171   if (!fil.f) pFuncError;
    172   fil.fmode = READWRITE;
    173 
    174   // TODO read block 0 for configuration: blockSize, blockIndexSize 32/64 bits
    175 
    176   fil.count = fileSize(filename) / fil.blockSize;
    177 
    178   // initialize freeF
    179   if (!fileExists(fil.freeName) || !dbExists) {
    180     dArrayInit(&fil.freeBlocks);
    181     // create empty freeF file
    182     fil.freeF = fopen(fil.freeName, "w");
    183     fclose(fil.freeF);
    184   }
    185   else {
    186     // load freeBlocks
    187     dArrayInit(&fil.freeBlocks);
    188     dArrayReadFilename(&fil.freeBlocks, fil.freeName);
    189   }
    190 
    191   // open freeF - read mode to keep the content (fixes the duplicate test)
    192   // this file is freopen in write mode later
    193   fil.freeF = fopen(fil.freeName, "r");
    194 
    195   return true;
    196 }
    197 
    198 internal void closeBlockFile(blockFilet *self) {
    199   // save free blocks
    200   freopen(NULL, "w", fil.freeF);
    201   dArrayWrite(&fil.freeBlocks, fil.freeF);
    202   // free dArray
    203   dArrayFree(&fil.freeBlocks);
    204 
    205   // close FILES
    206   fclose(fil.f);
    207   fclose(fil.freeF);
    208   freeManyS(fil.name, fil.freeName);
    209 }
    210 
    211 internal void deleteBlockFile(blockFilet *self) {
    212   char *s1 = strdup(fil.name);
    213   char *s2 = strdup(fil.freeName);
    214 
    215   // close file
    216   closeBlockFile(self);
    217 
    218   // remove file and freeF
    219   pError0(rmAll(s1));
    220   pError0(rmAll(s2));
    221   freeManyS(s1, s2);
    222 }
    223 
    224 internal void deleteFBlockFile(char *filename) {
    225   if (!filename || isBlankS(filename))
    226     return;
    227 
    228   char *s = catS(filename, "x");
    229 
    230   // remove file and freeF
    231   pError0(rmAll(filename));
    232   pError0(rmAll(s));
    233   free(s);
    234 }
    235 
    236 /**
    237  * set file mode
    238  * read/write or append
    239  */
    240 internal void setFMode(blockFilet *self, fmodet fmode) {
    241   if ((fil.fmode == APPEND) and (fmode == READWRITE)) {
    242     freopen(NULL, "r+", fil.f);
    243     fil.fmode = READWRITE;
    244   }
    245   if ((fil.fmode == READWRITE) and (fmode == APPEND)) {
    246     freopen(NULL, "a", fil.f);
    247     fil.fmode = APPEND;
    248   }
    249 }
    250 
    251 internal uI insertBlock(blockFilet *self, void *blockData) {
    252   if (!dArrayCount(&fil.freeBlocks)) {
    253     // append new block
    254     //logI("no free blocks");
    255     setFMode(self, APPEND);
    256     fwrite(blockData, 1, fil.blockSize, fil.f);
    257     fil.count++;
    258     //logVarG(fil.count-1);
    259     return fil.count-1;
    260   }
    261   else {
    262     // reuse free block
    263     uI block = dArrayPop(&fil.freeBlocks);
    264     setFMode(self, READWRITE);
    265     fseek(fil.f, block * fil.blockSize, SEEK_SET);
    266     fwrite(blockData, 1, fil.blockSize, fil.f);
    267     //logVarG(block);
    268     return block;
    269   }
    270 }
    271 
    272 internal uI nextBlock(blockFilet *self) {
    273   if (!dArrayCount(&fil.freeBlocks)) {
    274     return fil.count+1;
    275   }
    276   else if (dArrayCount(&fil.freeBlocks) == 1) {
    277     return fil.count;
    278   }
    279   else {
    280     return dArrayAt(&fil.freeBlocks, dArrayLastIndex(&fil.freeBlocks)-1);
    281   }
    282 }
    283 
    284 internal u64  addBlockBlockFile(blockFilet *self, void* buf, i64 len, flagsBlockFilet flags) {
    285   uI firstBlock = 0;
    286   size_t freeB  =  dArrayCount(&fil.freeBlocks);
    287 
    288   //logVarG(len);
    289 
    290   // compress with lz4
    291   uint32_t decompressedSize;
    292   if (flags == COMPRESSED) {
    293     if (len < 192) {
    294       // too small to have benefits from compression
    295       flags = NOT_COMPRESSED;
    296       //AT;
    297     }
    298     else {
    299       decompressedSize         = len;
    300       const char *uncompressed = buf;
    301 
    302       int max_dst_size = LZ4_compressBound(decompressedSize);
    303 
    304       char* compressed_data = malloc(max_dst_size);
    305       if (compressed_data == NULL) {
    306         shEPrintfS("Failed to allocate memory for *compressed_data.");
    307         return 0;
    308       }
    309 
    310       len = LZ4_compress_default(uncompressed, compressed_data, decompressedSize, max_dst_size);
    311 
    312       if (len < 0) {
    313         shEPrintfS("A negative result from LZ4_compress_default indicates a failure trying to compress the data.  Value returned %d\n", len);
    314         return 0;
    315       }
    316       if (len == 0) {
    317         shEPrintfS("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.");
    318         return 0;
    319       }
    320 
    321       buf = compressed_data;
    322     }
    323   }
    324 
    325   // header sizes
    326   u64 firstHeaderSize;
    327   if (flags == COMPRESSED) {
    328     firstHeaderSize = sizeof(struct firstBlockHeaderCompressed);
    329   }
    330   else {
    331     firstHeaderSize = sizeof(struct firstBlockHeader);
    332   }
    333   u64 headerSize    = sizeof(struct blockheader);
    334   i64 firstDataSize = fil.blockSize - firstHeaderSize;
    335   i64 dataSize      = fil.blockSize - headerSize;
    336 
    337   // create block chain
    338   // user data in first block:  blockSize - firstHeaderSize
    339   // user data in other blocks: blockSize - headerSize
    340 
    341   char   blockBuf[fil.blockSize];
    342   blockt *header       = (blockt *)blockBuf;
    343   bool   isFirstBlock  = true;
    344   char   *offset       = buf;
    345   u64    currentDataSize;
    346 
    347   //loghex(buf, len);
    348 
    349   while(len > 0) {
    350     if (isFirstBlock) {
    351       isFirstBlock      = false;
    352       header->chain     = 1;
    353       header->z         = flags == COMPRESSED ? 1 : 0;
    354       header->dataSize  = len;
    355 
    356       if (flags == COMPRESSED) {
    357         header->decompressedSize = decompressedSize;
    358       }
    359 
    360       if (len <= firstDataSize) {
    361         header->nextBlock = 0;
    362         memcpy(blockBuf+firstHeaderSize, offset, len);
    363         firstBlock = insertBlock(self, blockBuf);
    364         //len = 0;
    365         break;
    366        }
    367        else {
    368         // more blocks to save
    369         header->nextBlock = nextBlock(self);
    370         currentDataSize   = firstDataSize;
    371         memcpy(blockBuf+firstHeaderSize, offset, firstDataSize);
    372         firstBlock = insertBlock(self, blockBuf);
    373       }
    374     }
    375     else {
    376       // other blocks
    377       header->chain = 0;
    378       if (len <= dataSize) {
    379         header->nextBlock = 0;
    380         memcpy(blockBuf+headerSize, offset, len);
    381         insertBlock(self, blockBuf);
    382         //len = 0;
    383         break;
    384       }
    385       else {
    386         // more blocks to save
    387         header->nextBlock = nextBlock(self);
    388         currentDataSize   = dataSize;
    389         memcpy(blockBuf+headerSize, offset, currentDataSize);
    390         insertBlock(self, blockBuf);
    391        }
    392     }
    393     //logVarG(header->nextBlock);
    394     len    -= currentDataSize;
    395     offset += currentDataSize;
    396   }
    397   //logVarG(header->nextBlock);
    398 
    399   if (flags == COMPRESSED) {
    400     // free the compressed buffer
    401     free(buf);
    402   }
    403 
    404   // save free blocks if necessary
    405   if (freeB) {
    406     // there were free block when we started, save free blocks
    407     freopen(NULL, "w", fil.freeF);
    408     dArrayWrite(&fil.freeBlocks, fil.freeF);
    409   }
    410   return firstBlock;
    411 }
    412 
    413 internal u64  addBlockFile(blockFilet *self, void* buf, i64 len) {
    414   return addBlockBlockFile(self, buf, len, fil.defaultFlags);
    415 }
    416 
    417 /**
    418  * \return
    419  *  bufBlockFilet {data, len}, data has to be freed
    420  */
    421 internal bufBlockFilet getBlockFile(blockFilet *self, u64 block) {
    422   char   blockBuf[fil.blockSize];
    423   blockt *header       = (blockt *)blockBuf;
    424   bufBlockFilet result = {NULL, 0};
    425 
    426   // block 0 is reserved
    427   if (!block) return result;
    428 
    429   // check if block is free
    430   range(i, (size_t)(dArrayCount(&fil.freeBlocks))) {
    431     if (block == dArrayAt(&fil.freeBlocks, i)) {
    432       return result;
    433     }
    434   }
    435 
    436   setFMode(self, READWRITE);
    437   fseek(fil.f, block * fil.blockSize, SEEK_SET);
    438   fread(blockBuf, 1, fil.blockSize, fil.f);
    439 
    440   if (header->chain != 1) {
    441     // error not first block
    442     return result;
    443   }
    444 
    445   // header sizes
    446   u64 firstHeaderSize;
    447   if (header->z) {
    448     firstHeaderSize = sizeof(struct firstBlockHeaderCompressed);
    449   }
    450   else {
    451     firstHeaderSize = sizeof(struct firstBlockHeader);
    452   }
    453   u64 headerSize    = sizeof(struct blockheader);
    454   u64 firstDataSize = fil.blockSize - firstHeaderSize;
    455   u64 dataSize      = fil.blockSize - headerSize;
    456 
    457   u64 currentDataSize;
    458 
    459   // decompress
    460   uint32_t decompressedSize = 0;
    461   if (header->z) {
    462     decompressedSize = header->decompressedSize;
    463   }
    464 
    465   u64 len                  = header->dataSize;
    466   const u64 compressedSize = header->dataSize;
    467   void *r                  = malloc(len);
    468   if (!r) {
    469     return result;
    470   }
    471 
    472   // set result length: dataSize or decompressedSize
    473   if (header->z) {
    474     result.len = decompressedSize;
    475   }
    476   else {
    477     result.len = len;
    478   }
    479 
    480   /* first block already loaded */
    481   char *offset      = r;
    482 
    483   bool isFirstBlock = true;
    484   while(len > 0) {
    485     if (isFirstBlock) {
    486       isFirstBlock    = false;
    487       if (len <= firstDataSize) {
    488         if (header->nextBlock) {
    489           // error this block should be the last block
    490           goto error;
    491         }
    492         // all the data is in the first block
    493         memcpy(offset, blockBuf+firstHeaderSize, len);
    494         //len = 0;
    495         break;
    496       }
    497       else {
    498         block = header->nextBlock;
    499         if (!block) {
    500           // error there should be a next block
    501           goto error;
    502         }
    503         currentDataSize = firstDataSize;
    504         memcpy(offset, blockBuf+firstHeaderSize, currentDataSize);
    505       }
    506     }
    507     else {
    508       fseek(fil.f, block * fil.blockSize, SEEK_SET);
    509       fread(blockBuf, 1, fil.blockSize, fil.f);
    510 
    511       if (header->chain != 0) {
    512         // error not a block in a chain
    513         goto error;
    514       }
    515 
    516       if (len <= dataSize) {
    517         if (header->nextBlock) {
    518           // error this block should be the last block
    519           goto error;
    520         }
    521         // all the data is in the first block
    522         memcpy(offset, blockBuf+headerSize, len);
    523         //len = 0;
    524         break;
    525       }
    526       else {
    527         block = header->nextBlock;
    528         if (!block) {
    529           // error there should be a next block
    530           goto error;
    531         }
    532         currentDataSize = dataSize;
    533         memcpy(offset, blockBuf+headerSize, currentDataSize);
    534       }
    535     }
    536     //logVarG(header->nextBlock);
    537     len    -= currentDataSize;
    538     offset += currentDataSize;
    539   }
    540 
    541   //loghex(r, compressedSize);
    542 
    543   // decompress
    544   if (decompressedSize) {
    545     // the data is compressed
    546     char *compressed_data = r;
    547     char* regen_buffer    = malloc(decompressedSize);
    548     if (regen_buffer == NULL) {
    549       free(r);
    550       shEPrintfS("Failed to allocate memory for *regen_buffer.");
    551       return result;
    552     }
    553 
    554     const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressedSize, decompressedSize);
    555     free(r);
    556 
    557     if (decompressed_size < 0) {
    558       free(regen_buffer);
    559       shEPrintfS("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data.  Value returned %d\n", decompressed_size);
    560       return result;
    561     }
    562     if (decompressed_size == 0) {
    563       free(regen_buffer);
    564       shEPrintfS("I'm not sure this function can ever return 0.  Documentation in lz4.h doesn't indicate so.");
    565       return result;
    566     }
    567 
    568     r = regen_buffer;
    569   }
    570 
    571   result.data = r;
    572   return result;
    573 
    574 error:
    575   //logVarG(header->nextBlock);
    576   free(r);
    577   return result;
    578 }
    579 
    580 internal bool removeBlockFile(blockFilet *self, u64 block) {
    581   char blockBuf[sizeof(blockt)];
    582   blockt *header = (blockt *)blockBuf;
    583 
    584   // block 0 is reserved
    585   if (!block) return false;
    586 
    587   // check if block is already free
    588   range(i, (size_t)(dArrayCount(&fil.freeBlocks))) {
    589     if (block == dArrayAt(&fil.freeBlocks, i)) {
    590       return false;
    591     }
    592   }
    593 
    594   setFMode(self, READWRITE);
    595   fseek(fil.f, block * fil.blockSize, SEEK_SET);
    596   fread(blockBuf, 1, sizeof(blockt), fil.f);
    597 
    598   if (header->chain != 1) {
    599     // error not first block
    600     return false;
    601   }
    602 
    603   // delete first block
    604   dArrayAppend(&fil.freeBlocks, block);
    605 
    606   while(header->nextBlock) {
    607     block = header->nextBlock;
    608     fseek(fil.f, block * fil.blockSize, SEEK_SET);
    609     fread(blockBuf, 1, sizeof(blockt), fil.f);
    610     if (header->chain != 0) {
    611       // error not other block
    612       return false;
    613     }
    614     // delete block
    615     dArrayAppend(&fil.freeBlocks, block);
    616   }
    617 
    618   // save free blocks
    619   freopen(NULL, "w", fil.freeF);
    620   dArrayWrite(&fil.freeBlocks, fil.freeF);
    621 
    622   return true;
    623 }
    624 
    625 internal bool loadBlockFile(blockFilet *self, void *closure, loadFBlockFileFt callback) {
    626   char blockBuf[sizeof(blockt)];
    627   blockt *header = (blockt *)blockBuf;
    628 
    629   setFMode(self, READWRITE);
    630 
    631   rangeFrom(i, 1, fil.count) {
    632     // check if block i is free
    633     range(j, (size_t)(dArrayCount(&fil.freeBlocks))) {
    634       if (i == dArrayAt(&fil.freeBlocks, j)) goto cont;
    635     }
    636 
    637     fseek(fil.f, i * fil.blockSize, SEEK_SET);
    638     fread(blockBuf, 1, sizeof(blockt), fil.f);
    639 
    640     if (header->chain != 1) {
    641       // not first block, skip
    642       goto cont;
    643     }
    644 
    645     bufBlockFilet data = getBlockFile(self, i);
    646     if (!callback(closure, i, data)) return false;
    647     cont:;
    648   }
    649   return true;
    650 }
    651 
    652 /* TODO add method implementations */
    653 
    654 bool checkLibsheepyVersionBlockFile(const char *currentLibsheepyVersion) {
    655   return eqG(currentLibsheepyVersion, LIBSHEEPY_VERSION);
    656 }
    657