From: Steinar H. Gunderson Date: Mon, 21 Feb 2011 22:38:35 +0000 (+0100) Subject: Modify ChainWalkSet::FindInFile() to keep the precalc index in memory instead of... X-Git-Url: https://git.sesse.net/?p=freerainbowtables;a=commitdiff_plain;h=67081d97eed3fd94f2a109d3d9f51a0a477f9a79 Modify ChainWalkSet::FindInFile() to keep the precalc index in memory instead of reading it in every time. About halves the wall time spent between checking false alarms for each hash, which means a solid (~2x) speedup for anything with 20 or more threads doing false alarm checking. --- diff --git a/Client Applications/rcracki_mt/ChainWalkSet.cpp b/Client Applications/rcracki_mt/ChainWalkSet.cpp index 61a2ec0..9aafcda 100644 --- a/Client Applications/rcracki_mt/ChainWalkSet.cpp +++ b/Client Applications/rcracki_mt/ChainWalkSet.cpp @@ -57,7 +57,7 @@ void CChainWalkSet::DiscardAll() m_lChainWalk.clear(); } -string CChainWalkSet::CheckOrRotatePreCalcFile() +int CChainWalkSet::CheckOrRotatePreCalcFile() { char sPreCalcFileName[255]; @@ -81,19 +81,20 @@ string CChainWalkSet::CheckOrRotatePreCalcFile() // We don't as only newly generated chainwalksets will be stored to this new file, so we don't have to look there if (debug) printf("Debug: Using for precalc: %s\n", sReturnPreCalcPath.c_str()); fclose(file); - return sReturnPreCalcPath; + return preCalcPart; } fclose(file); } } - return string(""); + return -1; } void CChainWalkSet::updateUsedPrecalcFiles() { // we might also use this function to search a wildcard path of precalc files vPrecalcFiles.clear(); + mPrecalcIndex.clear(); char sPreCalcFileName[255]; int i; @@ -103,13 +104,35 @@ void CChainWalkSet::updateUsedPrecalcFiles() sprintf(sPreCalcFileName, "%s.%d", sPrecalcPathName.c_str(), i); string sTryPreCalcPath(sPreCalcFileName); FILE* file = fopen(sTryPreCalcPath.c_str(), "rb"); - if(file!=NULL) { - vPrecalcFiles.push_back(sTryPreCalcPath); - fclose(file); - } - else { + if(file==NULL) { break; } + + vPrecalcFiles.push_back(sTryPreCalcPath); + fclose(file); + + // Load the index file into our cache. + string sCurrentPrecalcIndexPathName = sTryPreCalcPath + ".index"; + vector precalcLines; + if (!ReadLinesFromFile(sCurrentPrecalcIndexPathName.c_str(), precalcLines)) { + continue; + } + + long unsigned int offset = 0; + for (int j = 0; j < (int)precalcLines.size(); j++) + { + // Parse + vector vPart; + if (!SeparateString(precalcLines[j], "___:", vPart)) + { + // corrupt file + printf("Corrupted precalculation file!\n"); + break; + } + + mPrecalcIndex[precalcLines[j]] = make_pair(i, offset); + offset += ((atoi(vPart[3].c_str())-1) * sizeof(uint64)); + } } } @@ -137,92 +160,62 @@ void CChainWalkSet::removePrecalcFiles() if (debug) printf("Debug: Failed removing precalc index file: %s\n", sCurrentPrecalcIndexPathName.c_str()); } + + mPrecalcIndex.clear(); } bool CChainWalkSet::FindInFile(uint64* pIndexE, unsigned char* pHash, int nHashLen) { - int gotPrecalcOnLine = -1; char precalculationLine[255]; - sprintf(precalculationLine, "%s_%s#%d-%d_%d_%d:%s\n", m_sHashRoutineName.c_str(), m_sPlainCharsetName.c_str(), m_nPlainLenMin, m_nPlainLenMax, m_nRainbowTableIndex, m_nRainbowChainLen, HexToStr(pHash, nHashLen).c_str() ); - string precalcString(precalculationLine); + sprintf(precalculationLine, "%s_%s#%d-%d_%d_%d:%s", m_sHashRoutineName.c_str(), m_sPlainCharsetName.c_str(), m_nPlainLenMin, m_nPlainLenMax, m_nRainbowTableIndex, m_nRainbowChainLen, HexToStr(pHash, nHashLen).c_str() ); - string sCurrentPrecalcPathName = ""; - string sCurrentPrecalcIndexPathName = ""; - long unsigned int offset; - - int i; - for (i = 0; i < (int)vPrecalcFiles.size() && gotPrecalcOnLine == -1; i++) - { - sCurrentPrecalcPathName = vPrecalcFiles[i]; - sCurrentPrecalcIndexPathName = sCurrentPrecalcPathName + ".index"; - - offset = 0; - - vector precalcLines; - if (ReadLinesFromFile(sCurrentPrecalcIndexPathName.c_str(), precalcLines)) - { - int j; - for (j = 0; j < (int)precalcLines.size(); j++) - { - if (precalcString.compare(0, precalcString.size()-1, precalcLines[j]) == 0) - { - gotPrecalcOnLine = j; - break; - } - - // Parse - vector vPart; - if (SeparateString(precalcLines[j], "___:", vPart)) - { - // add to offset - offset += ((atoi(vPart[3].c_str())-1) * sizeof(uint64)); - } - else { - // corrupt file - printf("Corrupted precalculation file!\n"); - gotPrecalcOnLine = -1; - break; - } - } - } + map >::const_iterator precalc_it = + mPrecalcIndex.find(precalculationLine); + if (precalc_it == mPrecalcIndex.end()) { + return false; } - if (gotPrecalcOnLine > -1) - { - if (debug) printf("Debug: Reading pre calculations from file, line %d offset %lu\n", gotPrecalcOnLine, offset); - - FILE* fp = fopen(sCurrentPrecalcPathName.c_str(), "rb"); + int fileNumber = precalc_it->second.first; + unsigned long int offset = precalc_it->second.second; + if (debug) printf("Debug: Reading pre calculations from file, file %d offset %lu\n", fileNumber, offset); - if (fp!=NULL) { - fseek(fp, offset, SEEK_SET); + string sCurrentPrecalcPathName = vPrecalcFiles[fileNumber]; + FILE* fp = fopen(sCurrentPrecalcPathName.c_str(), "rb"); - // We should do some verification here, for example by recalculating the middle chain, to catch corrupted files - if(fread(pIndexE, sizeof(uint64), (unsigned long)m_nRainbowChainLen-1, fp) != (unsigned long)m_nRainbowChainLen-1) - printf("File read error."); - fclose(fp); - } - else - printf("Cannot open precalculation file %s.\n", sCurrentPrecalcPathName.c_str()); + if (fp == NULL) { + printf("Cannot open precalculation file %s.\n", sCurrentPrecalcPathName.c_str()); + return false; + } - //printf("\npIndexE[0]: %s\n", uint64tostr(pIndexE[0]).c_str()); - //printf("\npIndexE[nRainbowChainLen-2]: %s\n", uint64tostr(pIndexE[m_nRainbowChainLen-2]).c_str()); + fseek(fp, offset, SEEK_SET); - return true; + // We should do some verification here, for example by recalculating the middle chain, to catch corrupted files + if(fread(pIndexE, sizeof(uint64), (unsigned long)m_nRainbowChainLen-1, fp) != (unsigned long)m_nRainbowChainLen-1) { + printf("File read error."); + return false; } - return false; + fclose(fp); + return true; } void CChainWalkSet::StoreToFile(uint64* pIndexE, unsigned char* pHash, int nHashLen) { if (debug) printf("\nDebug: Storing precalc\n"); - string sCurrentPrecalcPathName = CheckOrRotatePreCalcFile(); - string sCurrentPrecalcIndexPathName = sCurrentPrecalcPathName + ".index"; + int iCurrentPrecalcPart = CheckOrRotatePreCalcFile(); + if (iCurrentPrecalcPart == -1) { + return; + } + char sCurrentPrecalcPathName[256]; + sprintf(sCurrentPrecalcPathName, "%s.%d", sPrecalcPathName.c_str(), preCalcPart); + string sCurrentPrecalcIndexPathName = string(sCurrentPrecalcPathName) + ".index"; - FILE* fp = fopen(sCurrentPrecalcPathName.c_str(), "ab"); + FILE* fp = fopen(sCurrentPrecalcPathName, "ab"); if(fp!=NULL) { + unsigned long int offset = GetFileLen(fp); + if(fwrite(pIndexE, sizeof(uint64), (unsigned long)m_nRainbowChainLen-1, fp) != (unsigned long)m_nRainbowChainLen-1) printf("File write error."); else @@ -234,12 +227,13 @@ void CChainWalkSet::StoreToFile(uint64* pIndexE, unsigned char* pHash, int nHash sprintf(precalculationLine, "%s_%s#%d-%d_%d_%d:%s\n", m_sHashRoutineName.c_str(), m_sPlainCharsetName.c_str(), m_nPlainLenMin, m_nPlainLenMax, m_nRainbowTableIndex, m_nRainbowChainLen, HexToStr(pHash, nHashLen).c_str() ); fputs (precalculationLine, file); fclose (file); + mPrecalcIndex[TrimString(precalculationLine)] = make_pair(iCurrentPrecalcPart, offset); } } fclose(fp); } else - printf("Cannot open precalculation file %s\n", sCurrentPrecalcPathName.c_str()); + printf("Cannot open precalculation file %s\n", sCurrentPrecalcPathName); } uint64* CChainWalkSet::RequestWalk(unsigned char* pHash, int nHashLen, diff --git a/Client Applications/rcracki_mt/ChainWalkSet.h b/Client Applications/rcracki_mt/ChainWalkSet.h index a89c38c..a29fc01 100644 --- a/Client Applications/rcracki_mt/ChainWalkSet.h +++ b/Client Applications/rcracki_mt/ChainWalkSet.h @@ -53,11 +53,12 @@ private: string sPrecalcPathName; int preCalcPart; vector vPrecalcFiles; + map > mPrecalcIndex; // key -> (file number, file offset) private: void DiscardAll(); bool FindInFile(uint64* pIndexE, unsigned char* pHash, int nHashLen); - string CheckOrRotatePreCalcFile(); + int CheckOrRotatePreCalcFile(); void updateUsedPrecalcFiles(); public: diff --git a/Client Applications/rcracki_mt/Public.h b/Client Applications/rcracki_mt/Public.h index a277381..df741b2 100644 --- a/Client Applications/rcracki_mt/Public.h +++ b/Client Applications/rcracki_mt/Public.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "global.h"