15 #include "MemoryPool.h"
20 printf("converti2 - Original to Indexed rainbow table converter\n");
21 printf("by Martin Westergaard <martinwj2005@gmail.com>\n");
22 printf("http://www.freerainbowtables.com\n\n");
24 printf("usage: converti2 rainbow_table_pathname\n");
25 printf("rainbow_table_pathname: pathname of the rainbow table(s), wildchar(*, ?) supported\n");
27 printf("example: converti2 *.rt\n");
28 printf(" converti2 md5_*.rt\n");
32 int GetMaxBits(uint64 highvalue)
56 if(highvalue < 0x1000)
58 if(highvalue < 0x2000)
60 if(highvalue < 0x4000)
62 if(highvalue < 0x8000)
64 if(highvalue < 0x10000)
66 if(highvalue < 0x20000)
68 if(highvalue < 0x40000)
70 if(highvalue < 0x80000)
72 if(highvalue < 0x100000)
74 if(highvalue < 0x200000)
76 if(highvalue < 0x400000)
78 if(highvalue < 0x800000)
80 if(highvalue < 0x1000000)
82 if(highvalue < 0x2000000)
84 if(highvalue < 0x4000000)
86 if(highvalue < 0x8000000)
88 if(highvalue < 0x10000000)
90 if(highvalue < 0x20000000)
92 if(highvalue < 0x40000000)
94 if(highvalue < 0x80000000)
97 if(highvalue < 0x0000000100000000I64)
99 if(highvalue < 0x0000000200000000I64)
101 if(highvalue < 0x0000000400000000I64)
103 if(highvalue < 0x0000000800000000I64)
105 if(highvalue < 0x0000001000000000I64)
107 if(highvalue < 0x0000002000000000I64)
109 if(highvalue < 0x0000004000000000I64)
111 if(highvalue < 0x0000008000000000I64)
113 if(highvalue < 0x0000010000000000I64)
115 if(highvalue < 0x0000020000000000I64)
117 if(highvalue < 0x0000040000000000I64)
119 if(highvalue < 0x0000080000000000I64)
121 if(highvalue < 0x0000100000000000I64)
123 if(highvalue < 0x0000200000000000I64)
125 if(highvalue < 0x0000400000000000I64)
127 if(highvalue < 0x0000800000000000I64)
129 if(highvalue < 0x0001000000000000I64)
131 if(highvalue < 0x0002000000000000I64)
133 if(highvalue < 0x0004000000000000I64)
135 if(highvalue < 0x0008000000000000I64)
137 if(highvalue < 0x0010000000000000I64)
139 if(highvalue < 0x0020000000000000I64)
141 if(highvalue < 0x0040000000000000I64)
143 if(highvalue < 0x0080000000000000I64)
145 if(highvalue < 0x0100000000000000I64)
147 if(highvalue < 0x0200000000000000I64)
149 if(highvalue < 0x0400000000000000I64)
151 if(highvalue < 0x0800000000000000I64)
153 if(highvalue < 0x1000000000000000I64)
155 if(highvalue < 0x2000000000000000I64)
157 if(highvalue < 0x4000000000000000I64)
159 if(highvalue < 0x8000000000000000I64)
162 if(highvalue < 0x0000000100000000LL)
164 if(highvalue < 0x0000000200000000LL)
166 if(highvalue < 0x0000000400000000LL)
168 if(highvalue < 0x0000000800000000LL)
170 if(highvalue < 0x0000001000000000LL)
172 if(highvalue < 0x0000002000000000LL)
174 if(highvalue < 0x0000004000000000LL)
176 if(highvalue < 0x0000008000000000LL)
178 if(highvalue < 0x0000010000000000LL)
180 if(highvalue < 0x0000020000000000LL)
182 if(highvalue < 0x0000040000000000LL)
184 if(highvalue < 0x0000080000000000LL)
186 if(highvalue < 0x0000100000000000LL)
188 if(highvalue < 0x0000200000000000LL)
190 if(highvalue < 0x0000400000000000LL)
192 if(highvalue < 0x0000800000000000LL)
194 if(highvalue < 0x0001000000000000LL)
196 if(highvalue < 0x0002000000000000LL)
198 if(highvalue < 0x0004000000000000LL)
200 if(highvalue < 0x0008000000000000LL)
202 if(highvalue < 0x0010000000000000LL)
204 if(highvalue < 0x0020000000000000LL)
206 if(highvalue < 0x0040000000000000LL)
208 if(highvalue < 0x0080000000000000LL)
210 if(highvalue < 0x0100000000000000LL)
212 if(highvalue < 0x0200000000000000LL)
214 if(highvalue < 0x0400000000000000LL)
216 if(highvalue < 0x0800000000000000LL)
218 if(highvalue < 0x1000000000000000LL)
220 if(highvalue < 0x2000000000000000LL)
222 if(highvalue < 0x4000000000000000LL)
224 if(highvalue < 0x8000000000000000LL)
233 void GetTableList(string sWildCharPathName, vector<string>& vPathName)
238 int n = sWildCharPathName.find_last_of('\\');
240 sPath = sWildCharPathName.substr(0, n + 1);
243 long handle = _findfirst(sWildCharPathName.c_str(), &fd);
248 string sName = fd.name;
249 if (sName != "." && sName != ".." && !(fd.attrib & _A_SUBDIR))
251 string sPathName = sPath + sName;
252 vPathName.push_back(sPathName);
254 } while (_findnext(handle, &fd) == 0);
260 void GetTableList(int argc, char* argv[], vector<string>& vPathName)
265 for (i = 1; i < argc; i++)
267 string sPathName = argv[i];
269 if (lstat(sPathName.c_str(), &buf) == 0)
271 if (S_ISREG(buf.st_mode))
272 vPathName.push_back(sPathName);
280 void ConvertRainbowTable(string sPathName, string sResultFileName, unsigned int rti_startptlength, unsigned int rti_endptlength, int showDistribution, int hascp, int rti_cplength, vector<unsigned int> rti_cppos)
283 int nIndex = sPathName.find_last_of('\\');
285 int nIndex = sPathName.find_last_of('/');
289 sFileName = sPathName.substr(nIndex + 1);
291 sFileName = sPathName;
293 printf("%s:\n", sFileName.c_str());
294 FILE* file = fopen(sPathName.c_str(), "rb");
295 FILE* fileR = fopen(sResultFileName.c_str(), "wb");
296 unsigned int distribution[64] = {0};
297 unsigned int numProcessedChains = 0;
299 if (file != NULL && fileR != NULL)
302 unsigned int nFileLen = GetFileLen(file);
303 unsigned int nTotalChainCount = 0;
304 if(hascp == 0) nTotalChainCount = nFileLen / 16;
305 else nTotalChainCount = nFileLen / 18;
306 if ((hascp == 0 && nFileLen % 16 != 0) || (hascp == 1 && nFileLen % 18 != 0))
308 printf("file length mismatch\n");
312 static CMemoryPool mp;
313 unsigned int nAllocatedSize;
314 RainbowChainCP* pChain = (RainbowChainCP*)mp.Allocate(nFileLen, nAllocatedSize);
316 unsigned int chainrowsize = ceil((float)(rti_startptlength + rti_endptlength + rti_cplength) / 8) * 8 ; // The size in bits (in whole bytes)
317 unsigned int chainrowsizebytes = chainrowsize / 8;
322 nAllocatedSize = nAllocatedSize / sizeof(RainbowChainCP) * sizeof(RainbowChainCP);
323 fseek(file, 0, SEEK_SET);
324 uint64 curPrefix = 0, prefixStart = 0;
325 vector<IndexRow> indexes;
326 int nRainbowChainCountRead = 0;
327 while (true) // Chunk read loop
329 /* if (ftell(file) == nFileLen)
332 memset(pChain, 0x00, nAllocatedSize);
333 printf("reading...\n");
334 clock_t t1 = clock();
335 for(nReadThisRound = 0; nReadThisRound < nAllocatedSize / sizeof(RainbowChainCP) && nRainbowChainCountRead < nTotalChainCount; nReadThisRound++)
337 if(fread(&pChain[nReadThisRound], 16, 1, file) != 1)
339 printf("Error reading file\n"); exit(1);
343 if(fread(&pChain[nReadThisRound].nCheckPoint, 2, 1, file) != 1)
345 printf("Error reading file\n"); exit(2);
348 nRainbowChainCountRead++;
350 clock_t t2 = clock();
351 float fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
352 int nDataRead = nRainbowChainCountRead * 16;
353 if(hascp == 1) nDataRead += nRainbowChainCountRead * 2; // Add the index readings too
354 printf("%u bytes read, disk access time: %.2f s\n", nDataRead , fTime);
357 for(int i = 0; i < nReadThisRound; i++)
359 if(showDistribution == 1)
361 distribution[GetMaxBits(pChain[i].nIndexS)-1]++;
365 uint64 chainrow = pChain[i].nIndexS; // Insert the complete start point
366 chainrow |= ((uint64)pChain[i].nIndexE & (0xffffffff >> (32 - rti_endptlength))) << rti_startptlength; //
367 if(hascp == 1 && rti_cplength > 0)
369 chainrow |= (uint64)pChain[i].nCheckPoint << rti_startptlength + rti_endptlength;
371 fwrite(&chainrow, 1, chainrowsizebytes, fileR);
372 uint64 prefix = pChain[i].nIndexE >> rti_endptlength;
373 if(i == 0) curPrefix = prefix;
374 if(prefix != curPrefix && numProcessedChains - prefixStart > 0)
376 if(prefix < curPrefix)
378 printf("**** Error writeChain(): Prefix is smaller than previous prefix. %llu < %llu****\n", prefix, curPrefix);
381 //unsigned char index[11] = {0}; // [0 - 10]
382 unsigned int numchains = numProcessedChains - prefixStart;
384 index.prefix = curPrefix;
385 // index.prefixstart = prefixStart;
386 index.numchains = numchains;
387 indexes.push_back(index);
388 prefixStart = numProcessedChains;
392 numProcessedChains++;
395 fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
396 printf("conversion time: %.2f s\n", fTime);
397 if(nRainbowChainCountRead == nTotalChainCount)
399 if(showDistribution == 1)
401 for(int i = 0; i < 64; i++)
403 printf("%u - %u\n", (i+1), distribution[i]);
412 // We need to write the last index down
414 index.prefix = curPrefix;
415 index.prefixstart = prefixStart;
416 index.numchains = numProcessedChains - prefixStart;
417 indexes.push_back(index);
419 IndexRow high = {0}; // Used to find the highest numbers. This tells us how much we can pack the index bits
420 for(int i = 0; i < indexes.size(); i++)
422 if(indexes[i].numchains > high.numchains)
423 high.numchains = indexes[i].numchains;
424 /* if(indexes[i].prefixstart > high.prefixstart)
425 high.prefixstart = indexes[i].prefixstart;
426 if(indexes[i].prefix > high.prefix)
427 high.prefix = indexes[i].prefix;
430 high.prefix = indexes[indexes.size()-1].prefix; // The last prefix is always the highest prefix
431 // unsigned int m_rti_index_prefixlength = GetMaxBits(high.prefix);
432 unsigned int m_rti_index_numchainslength = GetMaxBits(high.numchains);
433 // unsigned int m_rti_index_indexlength = GetMaxBits(high.prefixstart);
434 unsigned int m_indexrowsize = ceil((float)(/*m_rti_index_indexlength + */m_rti_index_numchainslength) / 8) * 8; // The size in bits (in whole bytes)
435 unsigned int m_indexrowsizebytes = m_indexrowsize / 8;
436 FILE *pFileIndex = fopen(sResultFileName.append(".index").c_str(), "wb");
437 fwrite("RTI2", 1, 4, pFileIndex);
438 fwrite(&rti_startptlength, 1, 1, pFileIndex);
439 fwrite(&rti_endptlength, 1, 1, pFileIndex);
440 fwrite(&rti_cplength, 1, 1, pFileIndex);
441 // fwrite(&m_rti_index_indexlength , 1, 1, pFileIndex);
442 fwrite(&m_rti_index_numchainslength, 1, 1, pFileIndex);
443 for(int i = 0; i < rti_cppos.size(); i++)
445 fwrite(&rti_cppos[i], 1, 4, pFileIndex); // The position of the checkpoints
447 // fwrite(&m_rti_index_prefixlength, 1, 1, pFileIndex);
449 fwrite(&indexes[0].prefix, 1, 8, pFileIndex); // Write the first prefix
450 unsigned int lastPrefix = 0;
451 for(int i = 0; i < indexes.size(); i++)
454 lastPrefix = indexes[0].prefix;
455 unsigned int indexrow = 0;
456 // Checks how big a distance there is between the current and the next prefix. eg cur is 3 and next is 10 = 7.
457 unsigned int diffSize = indexes[i].prefix - lastPrefix;
458 if(i > 0 && diffSize > 1)
460 //indexrow |= indexes[i].prefixstart;
461 //printf("Diffsize is %u\n", diffSize);
463 // then write the distance amount of 00's
464 for(int j = 1; j < diffSize; j++)
466 fwrite(&zero, 1, m_indexrowsizebytes, pFileIndex);
469 fwrite(&indexes[i].numchains, 1, m_indexrowsizebytes, pFileIndex);
470 lastPrefix = indexes[i].prefix;
474 else printf("memory allocation fail\n");
483 printf("can't open file\n");
487 int main(int argc, char* argv[])
489 int argi = 1, i, argsUsed = 0;
490 unsigned int sptl = 40, eptl = 16;
491 int showDistribution = 0;
492 int usecp = 0;// How many bits to use from the index
494 vector<unsigned int> cppositions;
502 for (; argi < argc; argi++)
504 if (strcmp(argv[argi], "-d") == 0 && (argsUsed & 0x8) == 0)
506 // Enable verbose mode
508 showDistribution = 1;
510 else if (strncmp(argv[argi], "-sptl=", 6) == 0 && (argsUsed & 0x1) == 0)
512 // Maximum index for starting point
515 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
518 sptl += ((int) argv[argi][i]) - 0x30;
520 if (argv[argi][i] != '\0')
522 printf("Error: Invalid number.\n\n");
526 if (i > 23) // i - 3 > 20
528 printf("Error: Number is too large.\n\n");
534 else if (strncmp(argv[argi], "-eptl=", 6) == 0 && (argsUsed & 0x2) == 0)
536 // Maximum index for ending points
539 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
542 eptl += ((int) argv[argi][i]) - 0x30;
544 if (argv[argi][i] != '\0')
546 printf("Error: Invalid number.\n\n");
550 if (i > 23) // i - 3 > 20
552 printf("Error: Number is too large.\n\n");
557 else if(strncmp(argv[argi], "-usecp=", 7) == 0 && (argsUsed & 0x4) == 0)
562 unsigned int cppos = 0;
563 for(i = 7; argv[argi][i] != ' ' && argv[argi][i] != '\n' && argv[argi][i] != 0;)
565 if(cppositions.size() > 0) i++;
566 for (; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
569 cppos += ((int) argv[argi][i]) - 0x30;
571 /* if(argv[argi][i] == ',')
573 cppositions.push_back(cppos);
578 if (argv[argi][i] != '\0')
580 printf("Error: Invalid number.\n\n");
584 if (usecp > 16) // i - 3 > 20
586 printf("Error: Number is too large.\n\n");
590 else printf("Using %i bits of the checkpoints\n", usecp);
595 vector<string> vPathName;
597 string sWildCharPathName = argv[1];
598 GetTableList(sWildCharPathName, vPathName);
600 GetTableList(argc, argv, vPathName);
602 if (vPathName.size() == 0)
604 printf("no rainbow table found\n");
607 for (int i = 0; i < vPathName.size(); i++)
610 int n = vPathName[i].find_last_of('\\');
612 sResultFile = vPathName[i].substr(n+1, vPathName[i].length()) + "i2";
614 sResultFile = vPathName[i] + "i2"; // Resulting file is .rt, not .rti
615 printf("Using %i of 64 bits\n", (sptl + eptl + usecp));
616 if(sptl + eptl + usecp > 64)
620 ConvertRainbowTable(vPathName[i], sResultFile, sptl, eptl, showDistribution, hascp, usecp, cppositions);