diff --git a/zpaqfranz.cpp b/zpaqfranz.cpp index dd5b1bb..e99e8e9 100644 --- a/zpaqfranz.cpp +++ b/zpaqfranz.cpp @@ -53,8 +53,8 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#define ZPAQ_VERSION "60.4c" -#define ZPAQ_DATE "(2024-07-13)" // cannot use __DATE__ on Debian! +#define ZPAQ_VERSION "60.5e" +#define ZPAQ_DATE "(2024-07-20)" // cannot use __DATE__ on Debian! /// optional align for malloc (sparc64) via -DALIGNMALLOC #define STR(a) #a @@ -6692,6 +6692,8 @@ void stretchKey(char* out, const char* in, const char* salt) { } /// LICENSE_END.2 + + //////////////////////////// random ////////////////////////// // Put n cryptographic random bytes in buf[0..n-1]. // The first byte will not be 'z' or '7' (start of a ZPAQ archive). @@ -36232,6 +36234,25 @@ const char* const WB="wb"; const char* const RBPLUS="rb+"; const char* const ABPLUS="ab+"; const char* const AB="ab"; + +int myfclose(FP* fp) +{ + if (flagdebug) + if ((*fp)==FPNULL) + { + myprintf("06702: fclose on FPNULL ignored\n"); + return EOF; + } + if (flagdebug3) + myprintf("06706: pre closing fp %s\n",migliaia(int64_t(*fp))); + int risultato=fclose(*fp); + if (flagdebug3) + myprintf("06710: post closing fp %s result %d\n",migliaia(int64_t(*fp)),risultato); + *fp=FPNULL; + if (flagdebug3) + myprintf("36253: fp now %s\n",migliaia(int64_t(*fp))); + return risultato; +} #else // Windows typedef HANDLE FP; const FP FPNULL=INVALID_HANDLE_VALUE; @@ -36259,6 +36280,24 @@ int fclose(FP fp) { return CloseHandle(fp) ? 0 : EOF; } +int myfclose(FP* fp) +{ + if (flagdebug) + if ((*fp)==FPNULL) + { + myprintf("36288: win fclose on FPNULL ignored\n"); + return EOF; + } + if (flagdebug3) + myprintf("36292: pre closing fp %s\n",migliaia(int64_t(*fp))); + int risultato=CloseHandle(*fp) ? 0 : EOF; + if (flagdebug3) + myprintf("36295: post closing fp %s result %d\n",migliaia(int64_t(*fp)),risultato); + *fp=FPNULL; + if (flagdebug3) + myprintf("36298: fp now %s\n",migliaia(int64_t(*fp))); + return risultato; +} // Read nobj objects of size size into ptr. Return number of objects read. size_t fread(void* ptr, size_t size, size_t nobj, FP fp) { DWORD r=0; @@ -37162,7 +37201,7 @@ bool close(const char* filename, int64_t date, int64_t attr, FP fp=FPNULL) { if (fp==stdout) return true; #ifdef unix - if (fp!=FPNULL) fclose(fp); + if (fp!=FPNULL) myfclose(&fp); if (date>0) { struct utimbuf ub; ub.actime=time(NULL); @@ -38159,7 +38198,7 @@ class ArchiveBase { ArchiveBase(): aes(0), fp(FPNULL) {} ~ArchiveBase() { if (aes) delete aes; - if (fp!=FPNULL) fclose(fp); + if (fp!=FPNULL) myfclose(&fp); } bool isopen() {return fp!=FPNULL;} FP getthefp() { return fp;} @@ -38241,7 +38280,7 @@ void InputArchive::seek(int64_t p, int whence) { if (sum>off || i+1>=sz.size()) break; } const string next=subpart(fn, i+1); - fclose(fp); + myfclose(&fp); fp=fopen(next.c_str(), RB); if (fp==FPNULL) ioerr(next.c_str()); fseeko(fp, off-sum, SEEK_END); @@ -38271,7 +38310,7 @@ InputArchive::InputArchive(const char* filename):fn(filename) sz.push_back(ftello(fp)); total_sz+=sz.back(); if (fp!=NULL) - fclose(fp); + myfclose(&fp); } // Open first part const string part1=subpart(filename,1); @@ -38487,8 +38526,10 @@ class OutputArchive: public ArchiveBase, public libzpaq::Writer { myfwrite(buf, 1, ptr, firstfp); if (flagdebug3) myprintf("37838: closing the firstp %s\n",migliaia(int64_t(firstfp))); - fclose(firstfp); + myfclose(&firstfp); firstfp=FPNULL; + /// fik + fp=FPNULL; ptr=0; return; } @@ -38593,7 +38634,7 @@ class OutputArchive: public ArchiveBase, public libzpaq::Writer { myprintf("\r"); myprintf("37827: closing __________________________ %s |%s|\n",migliaia(int64_t(fp)),thefilename.c_str()); } - fclose(fp); + myfclose(&fp); } thefilename=format_filename(++chunknumber); if (flagdebug3) @@ -38705,9 +38746,12 @@ class OutputArchive: public ArchiveBase, public libzpaq::Writer { if (flagdebug3) myprintf("37954: calling flush on fp=%s\n",migliaia(int64_t(fp))); flush(); - if (flagdebug3) - myprintf("37957: calling close %s\n",migliaia(int64_t(fp))); - fclose(fp); + if (fp!=FPNULL) + { ///fik + if (flagdebug3) + myprintf("37957: calling close %s\n",migliaia(int64_t(fp))); + myfclose(&fp); + } } fp=FPNULL; } @@ -39531,6 +39575,7 @@ int64_t g_dt_ram; #define WORK_NONE 0 #define WORK_UPDATED 1 #define WORK_ADDED 2 +#define WORK_REMOVED 3 // filename entry struct DT // if you get some warning here, update your compiler! @@ -39549,7 +39594,6 @@ struct DT // if you get some warning here, update your compiler! string hexhash; // for new functions, not for add string hashtype; // for paranoid() string hexcrc32; - /// guilist string outputname; // written filename @@ -39584,7 +39628,7 @@ struct DT // if you get some warning here, update your compiler! franzfs *pramfile; int64_t kompressedsize; - int filework; //0 = nothing; 1=updated; 2=added + int filework; //0 = nothing; 1=updated; 2=added, 3=removed DT(): date(0), size(0), attr(0), data(0),creationdate(0),accessdate(0),written(-1),isordered(false),isselected(false),/*franz_block_size(FRANZOFFSETV3),*/file_crc32(0),hashedsize(0),chunk(-1),expectedsize(0),version(0),forceadd(false),is4(false),red_total(0),red_count(0),red_min(256),red_max(0),red_avg(0),red_candidate(0),kompressedsize(0),filework(0) { /// let's save a bit of RAM (during compression) @@ -45532,6 +45576,7 @@ string help_a(bool i_usage,bool i_example) #endif moreprint("+ : -date Store creation date (slower)"); moreprint("+ : -frugal Use less RAM, but CANNOT WORK with mixed hashes"); + moreprint("+ : -stat Show external files added/updated"); } if (i_usage && i_example) moreprint(" Examples:"); @@ -45921,7 +45966,7 @@ string help_i(bool i_usage,bool i_example) moreprint("CMD i (info)"); moreprint("+ : Directly shows the versions into the archive, with size and comments"); moreprint("+ : -comment Shows comments (if any)"); - moreprint("+ : -stat Count 'weird' files"); + moreprint("+ : -stat Count 'weird' files and show uncompressed size (slow)"); help_range(); } if (i_usage && i_example) moreprint(" Examples:"); @@ -46126,6 +46171,7 @@ string help_t(bool i_usage,bool i_example) moreprint("+ : -paranoid Extract all into -to something, check every file with stored hash"); moreprint("+ : delete every equal files"); moreprint("+ : -collision Test collisions"); + moreprint("+ : -quick Do not check hash, only size/date"); } if (i_usage && i_example) moreprint(" Examples:"); @@ -46142,6 +46188,7 @@ string help_t(bool i_usage,bool i_example) moreprint("Multiple test (*NIX): t \"/copie/*.zpaq\" "); moreprint("All version SHA-1 collisions: t z:\\1.zpaq -collision -all"); moreprint("Compare with string manipulation: t z:\\2 c:\\nz -find k:\\nz -replace c:\\nz -verify"); + moreprint("Quick-and-dirty size/date t z:\\1.zpaq c:\\nz -quick"); } return("Test archive integrity"); @@ -53844,6 +53891,12 @@ string franz_do_hash::filehash(string i_filename,bool i_flagcalccrc32,int64_t i_ // In the meantime calc the crc32 of the entire file bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc32,string i_myhashtype,const string i_myhash,string& o_hash) { + if (flagdebug3) + { + myprintf("\n\n"); + myprintf("53895: equal p first %s\n",p->first.c_str()); + myprintf("53896: equal filename %s\n",filename); + } o_crc32=0; o_hash=""; if (i_myhash=="FAKE") @@ -53862,17 +53915,43 @@ bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc3 } // internal or neither file exists if (p->second.date==0) - return !exists(filename); + { + bool risultato=!exists(filename); + if (flagdebug3) + myprintf("53919: second date zero, risultato %d\n",(int)risultato); + return risultato; + } // directories always match if (p->first!="" && isdirectory(p->first)) - return exists(filename); + { + bool risultato=exists(filename); + if (flagdebug3) + myprintf("53926: p->first isdirectory, check exists %d\n",(int)risultato); + return risultato; + } // compare sizes FP in=fopen(filename, RB); if ((in==NULL) || (in==FPNULL)) + { + if (flagdebug3) + myprintf("53935: failed fopen %s\n",filename); return false; + } fseeko(in, 0, SEEK_END); if (ftello(in)!=p->second.size) + { + if (flagdebug3) + myprintf("53942: ftello(in) %s <> p second size %s\n",migliaia(ftello(in)),migliaia2(p->second.size)); return fclose(in), false; + } + + if (flagquick) + { + fclose(in); + if (flagdebug3) + myprintf("53950: flagquick, return true\n"); + return true; + } // compare hashes chunk by chunk. fseeko(in, 0, SEEK_SET); libzpaq::SHA1 sha1; @@ -55712,6 +55791,83 @@ int Jidac::enumeratecomments() a->second.data='-'; filelist.push_back(a); } + + + + + /* + if (flagstat) + { + bool iserr=false; + + int updates=0, deletes=0, undated=0; + int64_t earliest=0, latest=0; + int errors=0; + for (unsigned i=1; i=0) ++blocks, blockSize=0; + if (ht[i].usize<0) ++unknown; + else { + usize+=ht[i].usize; + if (ht[i].usize>largestFragment) largestFragment=ht[i].usize; + blockSize+=ht[i].usize; + if (blockSize>largestBlock) largestBlock=blockSize; + } + if (ht[i].csize>csize || ht[i].csize<-int(i)) ++errors; + unsigned j; + for (j=0; j<20; ++j) + if (ht[i].sha1[j]) break; + nohash+=j==20; + } + } + + myprintf("55817: Known uncompressed %20s (%s)\n", migliaia(usize),tohuman(usize)); + ///myprintf("55768: Versions %s\n",migliaia(ver.size()-1)); + ///myprintf("55781: +/# %12s -%12s\n",migliaia(updates),migliaia2(deletes)); + myprintf("55783: First version %s last %s\n",dateToString(flagutc,earliest).c_str(),dateToString(flagutc,latest).c_str()); + myprintf("55785: Undated version %s Out of sequence %s",migliaia(undated),migliaia2(errors)); + if ((undated+errors)==0) + myprintf(": this is GOOD\n"); + else + myprintf(": something WRONG\n"); + myprintf("55817: Blocks %s blocks fragments %s highest frag %s\n", migliaia(blocks),migliaia2(used),migliaia3(ht.size()-1)); + if (used>0) + myprintf("55822: Average fragment size %1.3f\n", double(usize)/used); + myprintf("55822: Largest fragment size %20s\n",migliaia(largestFragment)); + myprintf("55822: Largest uncompressed block size %20s\n",migliaia(largestBlock)); + myprintf("55822: Fragments of unknown size %s without hashes %s missing %s",migliaia(unknown),migliaia2(nohash),migliaia3(errors)); + if ((unknown+nohash+errors)==0) + myprintf(": this is GOOD\n"); + else + myprintf(": something WRONG\n"); + + } +*/ + + if (all) { if (g_rangelast>0) @@ -55720,9 +55876,20 @@ int Jidac::enumeratecomments() g_rangefrom =lastver-g_rangelast+1; g_rangeto =lastver; } - myprintf("53244: --------------------------------------------------------------------------\n"); - myprintf("53245: < Ver > < date > < time > < added > < bytes added >\n"); - myprintf("53246: --------------------------------------------------------------------------\n"); + + if (flagstat) + { + myprintf("55792: ----------------------------------------------------------------------------------------------\n"); + myprintf("55793: < Ver > < date > < time > < added > < uncompressed > < compressed >\n"); + myprintf("55794: ----------------------------------------------------------------------------------------------\n"); + } + else + { + myprintf("53244: --------------------------------------------------------------------------\n"); + myprintf("53245: < Ver > < date > < time > < added > < bytes added >\n"); + myprintf("53246: --------------------------------------------------------------------------\n"); + } + for (unsigned fi=0;fisecond.date).c_str()); + + if (flagstat) + myprintf(" +%08d -%08d %20s [%20s]", ver[v].updates, ver[v].deletes, + migliaia2(ver[v].usize),migliaia((int64_t)((v+1 %20s", ver[v].updates, ver[v].deletes, migliaia((int64_t)((v+1::iterator commento; commento=mappacommenti.find(v); if(commento!= mappacommenti.end()) @@ -57137,6 +57311,7 @@ char extract_test4[]={"sFPjBhOWQqV6WzrDglUFbQMK03yqJV1b8bQY72n3KHOSpOrBreBdxIX+k ///DEBIANEND ///char extract_test1[]={"DEBIAN"};char extract_test2[]={""};char extract_test3[]={""};char extract_test4[]={""}; + int Jidac::pause() { int thekey=0; @@ -59707,10 +59882,15 @@ int Jidac::test() { if ((files.size()>0)) { - if (flagchecksum || flagverify || flagparanoid) - myprintf("56960: SHA-1-chunked verify+HASH checksum\n"); + if (flagquick) + myprintf("59884: -quick and dirty (only date/size) test\n"); else - myprintf("56951: SHA-1-chunked verify\n"); + { + if (flagchecksum || flagverify || flagparanoid) + myprintf("56960: SHA-1-chunked verify+HASH checksum\n"); + else + myprintf("56951: SHA-1-chunked verify\n"); + } return testverify(); } @@ -66827,6 +67007,8 @@ int Jidac::add() int64_t start_iblock=out.tell(); #endif + + // Delete from archive int dtcount=0; // index block header name int removed=0; // count @@ -66838,11 +67020,11 @@ int Jidac::add() is.write(p->first.c_str(), p->first.size()); //strlen(p->first.c_str())); is.put(0); ++removed; + p->second.filework=WORK_REMOVED; if (is.size()>16000) { libzpaq::compressBlock(&is, &wp, "1",("jDC"+itos(date)+"i"+itos(++dtcount, 10)).c_str(), "jDC\x01"); is.resize(0); - } } } @@ -66922,6 +67104,7 @@ int Jidac::add() if (currentcrc32!=p->second.file_crc32) myprintf("29604: SOMETHING WRONG ON %s\n",p->first.c_str()); } + ++added; /// date and filename puti(is, p->second.date, 8); @@ -67386,9 +67569,6 @@ int Jidac::add() - - - #if defined(_WIN32) if (flagvss) { @@ -68013,6 +68193,38 @@ int Jidac::add() if ((flagads) && (!flagfasttxt)) fill_ads(g_archive,start_iblock); #endif + + if (flagstat) + { + int work_added =0; + int work_updated =0; + int work_removed =0; + for (DTMap::const_iterator p=edt.begin(); p!=edt.end(); ++p) + { + ///myprintf("68222: filework %d %s\n",p->second.filework,p->first.c_str()); + if (p->second.filework==WORK_ADDED) + { + myprintf("68223: + %08d ",++work_added); + printUTF8(p->first.c_str()); + myprintf("\n"); + } + } + for (DTMap::const_iterator p=edt.begin(); p!=edt.end(); ++p) + if (p->second.filework==WORK_UPDATED) + { + myprintf("68231: # %08d ",++work_updated); + printUTF8(p->first.c_str()); + myprintf("\n"); + } + for (DTMap::const_iterator p=dt.begin(); p!=dt.end(); ++p) + if (p->second.filework==WORK_REMOVED) + { + myprintf("68221: - %08d ",++work_removed); + printUTF8(p->first.c_str()); + myprintf("\n"); + } + } + string temp="67504: Files "; string temp2=""; if (files_added>0) @@ -68032,6 +68244,7 @@ int Jidac::add() } if (files_added+files_updated+removed>0) myprintf("%s\n",temp.c_str()); + return errors; } @@ -91240,9 +91453,14 @@ int Jidac::testbackup() myprintf("84258: Report\n"); bool flagallok=true; +///fika + libzpaq::SHA256 globalesha256; for (unsigned int i=0;i0xffffffff) error("journaling block too big"); } + ///myprintf("_____________________ %s\n",migliaia(usize)); /// hic // Read the date and number in the filename int64_t fdate=0, num=0; for (i=3; i<17 && isdigit(filename.s[i]); ++i) @@ -92905,6 +93131,7 @@ int64_t Jidac::read_archive(callback_function i_advance,const char* arc, int *er error("bad checksum size"); if (sha1result[0] && memcmp(sha1result+1, sha1.result(), 20)) error("bad checksum"); + if ( ++parts % 1000 ==0) if (!flagnoeta) if (!i_quiet) @@ -93203,6 +93430,8 @@ int64_t Jidac::read_archive(callback_function i_advance,const char* arc, int *er k+=ht[j].csize; if (k>0 && k=0) dtr.kompressedsize+=ht[j].estimatedratio*ht[j].usize; + ///fik update version size + ver.back().usize+=ht[j].usize; } } if (flagads) @@ -93769,6 +93998,8 @@ int Jidac::dump() myprintf(">>\n"); return 2; } + ///int64_t csize=in.get_totalsize(); + if (all) { flagverbose =true; @@ -98107,70 +98338,79 @@ int Jidac::testverify() } if (flagdebug2) myprintf("98004: |%s| %s\n",myhashtype.c_str(),p->first.c_str()); + if (equal(p, p1->first.c_str(),crc32fromfile,myhashtype,myhash,hashfromfile)) //crc32 of second parameters { - if (!isfolderp1) + if (flagquick) { - if (hashfromfile!="") + p->second.data='='; + ++fi; + } + else + { + if (!isfolderp1) { - if (flagdebug2) - myprintf("49508: Check hash for %s\n",p->first.c_str()); - hashchecked+=p->second.size; - if (hashfromfile!=myhash) + if (hashfromfile!="") { - p->second.data='#'; - snprintf(linebuffer,sizeof(linebuffer),"\nHASH %s NOT MATCH STORED %s vs FROM FILE %s ",myhashtype.c_str(),myhash.c_str(),hashfromfile.c_str()); - risultati.push_back(linebuffer); - risultati_utf8.push_back(false); - snprintf(linebuffer,sizeof(linebuffer),"%s\n",p1->first.c_str()); - risultati.push_back(linebuffer); - risultati_utf8.push_back(true); + if (flagdebug2) + myprintf("49508: Check hash for %s\n",p->first.c_str()); + hashchecked+=p->second.size; + if (hashfromfile!=myhash) + { + p->second.data='#'; + snprintf(linebuffer,sizeof(linebuffer),"\nHASH %s NOT MATCH STORED %s vs FROM FILE %s ",myhashtype.c_str(),myhash.c_str(),hashfromfile.c_str()); + risultati.push_back(linebuffer); + risultati_utf8.push_back(false); + snprintf(linebuffer,sizeof(linebuffer),"%s\n",p1->first.c_str()); + risultati.push_back(linebuffer); + risultati_utf8.push_back(true); + } + else + { + hashmatches++; + p->second.data='='; + ++fi; + } } else + if (mycrc32!="") { - hashmatches++; - p->second.data='='; - ++fi; - } - } - else - if (mycrc32!="") - { - if (flagdebug2) - myprintf("98035: Check CRC-32 for %s\n",p->first.c_str()); - uint32_t crc32stored=crchex2int(mycrc32.c_str()); - if (crc32stored!=crc32fromfile) - { - p->second.data='#'; - snprintf(linebuffer,sizeof(linebuffer),"\nCRC-32 NOT MATCH STORED %s vs FROM FILE %08X ",mycrc32.c_str(),crc32fromfile); - risultati.push_back(linebuffer); - risultati_utf8.push_back(false); - snprintf(linebuffer,sizeof(linebuffer),"%s\n",p1->first.c_str()); - risultati.push_back(linebuffer); - risultati_utf8.push_back(true); + if (flagdebug2) + myprintf("98035: Check CRC-32 for %s\n",p->first.c_str()); + uint32_t crc32stored=crchex2int(mycrc32.c_str()); + if (crc32stored!=crc32fromfile) + { + p->second.data='#'; + snprintf(linebuffer,sizeof(linebuffer),"\nCRC-32 NOT MATCH STORED %s vs FROM FILE %08X ",mycrc32.c_str(),crc32fromfile); + risultati.push_back(linebuffer); + risultati_utf8.push_back(false); + snprintf(linebuffer,sizeof(linebuffer),"%s\n",p1->first.c_str()); + risultati.push_back(linebuffer); + risultati_utf8.push_back(true); + } + else + { + /// if very, do a full hash verify! + p->second.data='='; + ++fi; + } } else { - /// if very, do a full hash verify! + if (flagdebug2) + myprintf("98057: No zpaqfranz check for %s\n",p->first.c_str()); + /// crc-32 is not stored in the archive, cannot tell anything p->second.data='='; ++fi; } } else { - if (flagdebug2) - myprintf("98057: No zpaqfranz check for %s\n",p->first.c_str()); - /// crc-32 is not stored in the archive, cannot tell anything + /// directory always match p->second.data='='; ++fi; } } - else - { - /// directory always match - p->second.data='='; - ++fi; - } } else { @@ -98210,6 +98450,13 @@ int Jidac::testverify() } } myprintf("\n\n"); + + if (flagquick) + { + color_yellow(); + myprintf("98380: WARNING -quick only compare file size/date, NOT hash!\n"); + color_restore(); + } bool myerror=false; if (menoenne) if ((mismatches+externalfile+externalfolder+internal)>menoenne)