]> git.sesse.net Git - freerainbowtables/blob - Client Applications/converti2/converti2.cpp
a73fd95ed05a0bbb52139f6ba01ce1c9f1ffdbac
[freerainbowtables] / Client Applications / converti2 / converti2.cpp
1 #include <string>
2 #include <vector>
3 #ifdef _WIN32
4 #include <io.h>
5 #else
6         #include <sys/types.h>
7         #include <sys/stat.h>
8         #include <unistd.h>
9 #endif
10
11 #include <time.h>
12 #include <math.h>
13 #include <vector>
14 #include <conio.h>
15 #include "Public.h"
16 #include "MemoryPool.h"
17 using namespace std;
18
19 void Usage()
20 {
21         printf("converti2 - Original to Indexed rainbow table converter\n");
22         printf("by Martin Westergaard <martinwj2005@gmail.com>\n");
23         printf("http://www.freerainbowtables.com\n\n");
24
25         printf("usage: converti2 rainbow_table_pathname\n");
26         printf("rainbow_table_pathname: pathname of the rainbow table(s), wildchar(*, ?) supported\n");
27         printf("\n");
28         printf("example: converti2 *.rt\n");
29         printf("         converti2 md5_*.rt\n");
30 }
31
32
33 int GetMaxBits(uint64 highvalue)
34 {
35         if(highvalue < 0x02)
36                 return 1;
37         if(highvalue < 0x04)
38                 return 2;
39         if(highvalue < 0x08)
40                 return 3;
41         if(highvalue < 0x10)
42                 return 4;
43         if(highvalue < 0x20)
44                 return 5;
45         if(highvalue < 0x40)
46                 return 6;
47         if(highvalue < 0x80)
48                 return 7;
49         if(highvalue < 0x100)
50                 return 8;
51         if(highvalue < 0x200)
52                 return 9;
53         if(highvalue < 0x400)
54                 return 10;
55         if(highvalue < 0x800)
56                 return 11;
57         if(highvalue < 0x1000)
58                 return 12;
59         if(highvalue < 0x2000)
60                 return 13;
61         if(highvalue < 0x4000)
62                 return 14;
63         if(highvalue < 0x8000)
64                 return 15;
65         if(highvalue < 0x10000)
66                 return 16;
67         if(highvalue < 0x20000)
68                 return 17;
69         if(highvalue < 0x40000)
70                 return 18;
71         if(highvalue < 0x80000)
72                 return 19;
73         if(highvalue < 0x100000)
74                 return 20;
75         if(highvalue < 0x200000)
76                 return 21;
77         if(highvalue < 0x400000)
78                 return 22;
79         if(highvalue < 0x800000)
80                 return 23;
81         if(highvalue < 0x1000000)
82                 return 24;
83         if(highvalue < 0x2000000)
84                 return 25;
85         if(highvalue < 0x4000000)
86                 return 26;
87         if(highvalue < 0x8000000)
88                 return 27;
89         if(highvalue < 0x10000000)
90                 return 28;
91         if(highvalue < 0x20000000)
92                 return 29;
93         if(highvalue < 0x40000000)
94                 return 30;
95         if(highvalue < 0x80000000)
96                 return 31;
97 #ifdef WIN32
98         if(highvalue < 0x0000000100000000I64)
99                 return 32;
100         if(highvalue < 0x0000000200000000I64)
101                 return 33;
102         if(highvalue < 0x0000000400000000I64)
103                 return 34;
104         if(highvalue < 0x0000000800000000I64)
105                 return 35;
106         if(highvalue < 0x0000001000000000I64)
107                 return 36;
108         if(highvalue < 0x0000002000000000I64)
109                 return 37;
110         if(highvalue < 0x0000004000000000I64)
111                 return 38;
112         if(highvalue < 0x0000008000000000I64)
113                 return 39;
114         if(highvalue < 0x0000010000000000I64)
115                 return 40;
116         if(highvalue < 0x0000020000000000I64)
117                 return 41;
118         if(highvalue < 0x0000040000000000I64)
119                 return 42;
120         if(highvalue < 0x0000080000000000I64)
121                 return 43;
122         if(highvalue < 0x0000100000000000I64)
123                 return 44;
124         if(highvalue < 0x0000200000000000I64)
125                 return 45;
126         if(highvalue < 0x0000400000000000I64)
127                 return 46;
128         if(highvalue < 0x0000800000000000I64)
129                 return 47;
130         if(highvalue < 0x0001000000000000I64)
131                 return 48;
132         if(highvalue < 0x0002000000000000I64)
133                 return 49;
134         if(highvalue < 0x0004000000000000I64)
135                 return 50;
136         if(highvalue < 0x0008000000000000I64)
137                 return 51;
138         if(highvalue < 0x0010000000000000I64)
139                 return 52;
140         if(highvalue < 0x0020000000000000I64)
141                 return 53;
142         if(highvalue < 0x0040000000000000I64)
143                 return 54;
144         if(highvalue < 0x0080000000000000I64)
145                 return 55;
146         if(highvalue < 0x0100000000000000I64)
147                 return 56;
148         if(highvalue < 0x0200000000000000I64)
149                 return 57;
150         if(highvalue < 0x0400000000000000I64)
151                 return 58;
152         if(highvalue < 0x0800000000000000I64)
153                 return 59;
154         if(highvalue < 0x1000000000000000I64)
155                 return 60;
156         if(highvalue < 0x2000000000000000I64)
157                 return 61;
158         if(highvalue < 0x4000000000000000I64)
159                 return 62;
160         if(highvalue < 0x8000000000000000I64)
161                 return 63;
162 #else
163         if(highvalue < 0x0000000100000000LL)
164                 return 32;
165         if(highvalue < 0x0000000200000000LL)
166                 return 33;
167         if(highvalue < 0x0000000400000000LL)
168                 return 34;
169         if(highvalue < 0x0000000800000000LL)
170                 return 35;
171         if(highvalue < 0x0000001000000000LL)
172                 return 36;
173         if(highvalue < 0x0000002000000000LL)
174                 return 37;
175         if(highvalue < 0x0000004000000000LL)
176                 return 38;
177         if(highvalue < 0x0000008000000000LL)
178                 return 39;
179         if(highvalue < 0x0000010000000000LL)
180                 return 40;
181         if(highvalue < 0x0000020000000000LL)
182                 return 41;
183         if(highvalue < 0x0000040000000000LL)
184                 return 42;
185         if(highvalue < 0x0000080000000000LL)
186                 return 43;
187         if(highvalue < 0x0000100000000000LL)
188                 return 44;
189         if(highvalue < 0x0000200000000000LL)
190                 return 45;
191         if(highvalue < 0x0000400000000000LL)
192                 return 46;
193         if(highvalue < 0x0000800000000000LL)
194                 return 47;
195         if(highvalue < 0x0001000000000000LL)
196                 return 48;
197         if(highvalue < 0x0002000000000000LL)
198                 return 49;
199         if(highvalue < 0x0004000000000000LL)
200                 return 50;
201         if(highvalue < 0x0008000000000000LL)
202                 return 51;
203         if(highvalue < 0x0010000000000000LL)
204                 return 52;
205         if(highvalue < 0x0020000000000000LL)
206                 return 53;
207         if(highvalue < 0x0040000000000000LL)
208                 return 54;
209         if(highvalue < 0x0080000000000000LL)
210                 return 55;
211         if(highvalue < 0x0100000000000000LL)
212                 return 56;
213         if(highvalue < 0x0200000000000000LL)
214                 return 57;
215         if(highvalue < 0x0400000000000000LL)
216                 return 58;
217         if(highvalue < 0x0800000000000000LL)
218                 return 59;
219         if(highvalue < 0x1000000000000000LL)
220                 return 60;
221         if(highvalue < 0x2000000000000000LL)
222                 return 61;
223         if(highvalue < 0x4000000000000000LL)
224                 return 62;
225         if(highvalue < 0x8000000000000000LL)
226                 return 63;
227
228 #endif
229         return 64;
230
231 }
232
233 #ifdef _WIN32
234 void GetTableList(string sWildCharPathName, vector<string>& vPathName)
235 {
236         vPathName.clear();
237
238         string sPath;
239         int n = sWildCharPathName.find_last_of('\\');
240         if (n != -1)
241                 sPath = sWildCharPathName.substr(0, n + 1);
242
243         _finddata_t fd;
244         long handle = _findfirst(sWildCharPathName.c_str(), &fd);
245         if (handle != -1)
246         {
247                 do
248                 {
249                         string sName = fd.name;
250                         if (sName != "." && sName != ".." && !(fd.attrib & _A_SUBDIR))
251                         {
252                                 string sPathName = sPath + sName;
253                                 vPathName.push_back(sPathName);
254                         }
255                 } while (_findnext(handle, &fd) == 0);
256
257                 _findclose(handle);
258         }
259 }
260 #else
261 void GetTableList(int argc, char* argv[], vector<string>& vPathName)
262 {
263         vPathName.clear();
264
265         int i;
266         for (i = 1; i < argc; i++)
267         {
268                 string sPathName = argv[i];
269                 struct stat buf;
270                 if (lstat(sPathName.c_str(), &buf) == 0)
271                 {
272                         if (S_ISREG(buf.st_mode))
273                                 vPathName.push_back(sPathName);
274
275                 }
276         }
277 }
278 #endif
279
280
281 void ConvertRainbowTable(string sPathName, string sResultFileName, unsigned int rti_startptlength, unsigned int rti_endptlength, int showDistribution, int hascp, int rti_cplength, vector<unsigned int> rti_cppos)
282 {
283 #ifdef _WIN32
284         int nIndex = sPathName.find_last_of('\\');
285 #else
286         int nIndex = sPathName.find_last_of('/');
287 #endif
288         string sFileName;
289         if (nIndex != -1)
290                 sFileName = sPathName.substr(nIndex + 1);
291         else
292                 sFileName = sPathName;
293         // Info
294         printf("%s:\n", sFileName.c_str());
295         FILE* file = fopen(sPathName.c_str(), "rb");
296         FILE* fileR = fopen(sResultFileName.c_str(), "wb");
297         unsigned int distribution[64] = {0};
298         unsigned int numProcessedChains = 0;
299         
300         if (file != NULL && fileR != NULL)
301         {
302                 // File length check
303                 unsigned int nFileLen = GetFileLen(file);
304                 unsigned int nTotalChainCount = 0;
305                 if(hascp == 0) nTotalChainCount = nFileLen / 16;
306                 else nTotalChainCount = nFileLen / 18;
307                 if ((hascp == 0 && nFileLen % 16 != 0) || (hascp == 1 && nFileLen % 18 != 0))
308                 {
309                         printf("file length mismatch\n");
310                 }
311                 else
312                 {
313                         static CMemoryPool mp;
314                         unsigned int nAllocatedSize;
315                         RainbowChainCP* pChain = (RainbowChainCP*)mp.Allocate(nFileLen, nAllocatedSize);
316                         
317                         unsigned int chainrowsize = ceil((float)(rti_startptlength + rti_endptlength + rti_cplength) / 8) * 8 ; // The size in bits (in whole bytes)
318                         unsigned int chainrowsizebytes = chainrowsize / 8;
319
320
321                         if (pChain != NULL)
322                         {
323                                 nAllocatedSize = nAllocatedSize / sizeof(RainbowChainCP) * sizeof(RainbowChainCP);
324                                 fseek(file, 0, SEEK_SET);
325                                 uint64 curPrefix = 0, prefixStart = 0;
326                                 vector<IndexRow> indexes;
327                                 int nRainbowChainCountRead = 0;
328                                 while (true)    // Chunk read loop
329                                 {
330 /*                                      if (ftell(file) == nFileLen)
331                                                 break;*/
332                                         int nReadThisRound;
333                                         memset(pChain, 0x00, nAllocatedSize);
334                                         printf("reading...\n");
335                                         clock_t t1 = clock();
336                                         for(nReadThisRound = 0; nReadThisRound < nAllocatedSize / sizeof(RainbowChainCP) && nRainbowChainCountRead < nTotalChainCount; nReadThisRound++)
337                                         {                                               
338                                                 if(fread(&pChain[nReadThisRound], 16, 1, file) != 1) 
339                                                 { 
340                                                         printf("Error reading file\n"); exit(1);
341                                                 }
342                                                 if(hascp == 1)
343                                                 {
344                                                         if(fread(&pChain[nReadThisRound].nCheckPoint, 2, 1, file) != 1) 
345                                                         { 
346                                                                 printf("Error reading file\n"); exit(2);
347                                                         }
348                                                 }
349                                                 nRainbowChainCountRead++;
350                                         }
351                                         clock_t t2 = clock();
352                                         float fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
353                                         int nDataRead = nRainbowChainCountRead * 16;
354                                         if(hascp == 1) nDataRead += nRainbowChainCountRead * 2; // Add the index readings too
355                                         printf("%u bytes read, disk access time: %.2f s\n", nDataRead , fTime);
356                                         t1 = clock();
357
358                                         for(int i = 0; i < nReadThisRound; i++)
359                                         {
360                                                 if(showDistribution == 1)
361                                                 {
362                                                         distribution[GetMaxBits(pChain[i].nIndexS)-1]++;
363                                                 }
364                                                 else
365                                                 {
366                                                         uint64 chainrow = pChain[i].nIndexS; // Insert the complete start point                                                          
367                                                         chainrow |= ((uint64)pChain[i].nIndexE & (0xffffffff >> (32 - rti_endptlength))) << rti_startptlength; // 
368                                                         if(hascp == 1 && rti_cplength > 0) 
369                                                         {
370                                                                 chainrow |= (uint64)pChain[i].nCheckPoint << rti_startptlength + rti_endptlength;
371                                                         }
372                                                         fwrite(&chainrow, 1, chainrowsizebytes, fileR);                 
373                                                         uint64 prefix = pChain[i].nIndexE >> rti_endptlength;
374                                                         if(i == 0) curPrefix = prefix;
375                                                         if(prefix != curPrefix && numProcessedChains - prefixStart > 0)
376                                                         {
377                                                                         if(prefix < curPrefix)
378                                                                         {
379                                                                                 printf("**** Error writeChain(): Prefix is smaller than previous prefix. %llu < %llu****\n", prefix, curPrefix);
380                                                                                 exit(1);                                                                        
381                                                                         }
382                                                                         //unsigned char index[11] = {0}; // [0 - 10]
383                                                                         unsigned int numchains = numProcessedChains - prefixStart;
384                                                                         IndexRow index;
385                                                                         index.prefix = curPrefix;
386 //                                                                              index.prefixstart = prefixStart;
387                                                                         index.numchains = numchains;
388                                                                         indexes.push_back(index);
389                                                                         prefixStart = numProcessedChains;
390                                                                         curPrefix = prefix; 
391                                                         }
392                                                 }
393                                                 numProcessedChains++;
394                                         }               
395                                         t2 = clock();
396                                         fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
397                                         printf("conversion time: %.2f s\n", fTime);             
398                                         if(nRainbowChainCountRead == nTotalChainCount)
399                                                 break;
400                                         if(showDistribution == 1)
401                                         {
402                                                 for(int i = 0; i < 64; i++)
403                                                 {
404                                                         printf("%u - %u\n", (i+1), distribution[i]);
405                                                 }
406                                                 return;
407                                         }
408
409                                 }
410         
411                 
412         
413                                 // We need to write the last index down
414                                 IndexRow index;
415                                 index.prefix = curPrefix;
416                                 index.prefixstart = prefixStart;
417                                 index.numchains = numProcessedChains - prefixStart;
418                                 indexes.push_back(index);
419
420                                 IndexRow high = {0}; // Used to find the highest numbers. This tells us how much we can pack the index bits
421                                 for(int i = 0; i < indexes.size(); i++)
422                                 {
423                                         if(indexes[i].numchains > high.numchains)
424                                                 high.numchains = indexes[i].numchains;
425 /*                                              if(indexes[i].prefixstart > high.prefixstart)
426                                                 high.prefixstart = indexes[i].prefixstart;
427                                         if(indexes[i].prefix > high.prefix)
428                                                 high.prefix = indexes[i].prefix;
429 */
430                                 }
431                                 high.prefix = indexes[indexes.size()-1].prefix; // The last prefix is always the highest prefix
432 //                                      unsigned int m_rti_index_prefixlength = GetMaxBits(high.prefix);
433                                 unsigned int m_rti_index_numchainslength = GetMaxBits(high.numchains);
434 //                                      unsigned int m_rti_index_indexlength = GetMaxBits(high.prefixstart);
435                                 unsigned int m_indexrowsize = ceil((float)(/*m_rti_index_indexlength + */m_rti_index_numchainslength) / 8) * 8; // The size in bits (in whole bytes)    
436                                 unsigned int m_indexrowsizebytes = m_indexrowsize / 8;
437                                 FILE *pFileIndex = fopen(sResultFileName.append(".index").c_str(), "wb");
438                                 fwrite("RTI2", 1, 4, pFileIndex);
439                                 fwrite(&rti_startptlength, 1, 1, pFileIndex);
440                                 fwrite(&rti_endptlength, 1, 1, pFileIndex);
441                                 fwrite(&rti_cplength, 1, 1, pFileIndex);
442 //                                      fwrite(&m_rti_index_indexlength , 1, 1, pFileIndex);
443                                 fwrite(&m_rti_index_numchainslength, 1, 1, pFileIndex);
444                                 for(int i = 0; i < rti_cppos.size(); i++)
445                                 {
446                                         fwrite(&rti_cppos[i], 1, 4, pFileIndex); // The position of the checkpoints
447                                 }
448 //                                      fwrite(&m_rti_index_prefixlength, 1, 1, pFileIndex);
449                                 int zero = 0;
450                                 fwrite(&indexes[0].prefix, 1, 8, pFileIndex); // Write the first prefix
451                                 unsigned int lastPrefix = 0;
452                                 for(int i = 0; i < indexes.size(); i++)
453                                 {
454                                         if(i == 0)
455                                                 lastPrefix = indexes[0].prefix;
456                                         unsigned int indexrow = 0;
457                                         // Checks how big a distance there is between the current and the next prefix. eg cur is 3 and next is 10 = 7.
458                                         unsigned int diffSize = indexes[i].prefix - lastPrefix; 
459                                         if(i > 0 && diffSize > 1)
460                                         {
461                                                 //indexrow |= indexes[i].prefixstart;
462                                                 //printf("Diffsize is %u\n", diffSize);
463
464                                                 // then write the distance amount of 00's
465                                                 if(diffSize > 1000) {
466                                                         printf("WARNING! The distance to the next prefix is %i. Do you want to continue writing %i bytes of 0x00? Press y to continue", diffSize, diffSize);
467                                                         if(getch() != 'y') {
468                                                                 printf("Aborting...");
469                                                                 exit(1);
470                                                         }
471                                                 }
472                                                 for(int j = 1; j < diffSize; j++)
473                                                 {                                                               
474                                                         fwrite(&zero, 1, m_indexrowsizebytes, pFileIndex);
475                                                 }
476                                         }                                       
477                                         fwrite(&indexes[i].numchains, 1, m_indexrowsizebytes, pFileIndex);
478                                         lastPrefix = indexes[i].prefix;
479                                 }
480                                 fclose(pFileIndex);
481                         }
482                         else printf("memory allocation fail\n");
483
484                         
485                                         // Already finished?
486
487                 }
488                 fclose(file);
489         }
490         else
491                 printf("can't open file\n");
492
493 }
494
495 int main(int argc, char* argv[])
496 {
497         int argi = 1, i, argsUsed = 0;
498         unsigned int sptl = 40, eptl = 16;
499         int showDistribution = 0;
500         int usecp = 0;// How many bits to use from the index
501         int hascp = 0; 
502         vector<unsigned int> cppositions;
503         if (argc == 1)
504         {
505                 Usage();                
506                 return 0;
507         }
508         else if(argc > 2)
509         {
510                 for (; argi < argc; argi++)
511                 {
512                         if (strcmp(argv[argi], "-d") == 0 && (argsUsed & 0x8) == 0)
513                         {
514                                 // Enable verbose mode
515                                 argsUsed |= 0x8;                                
516                                 showDistribution = 1;
517                         }                       
518                         else if (strncmp(argv[argi], "-sptl=", 6) == 0 && (argsUsed & 0x1) == 0)
519                         {
520                                 // Maximum index for starting point
521                                 argsUsed |= 0x1;
522                                 sptl = 0;
523                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
524                                 {
525                                         sptl *= 10;
526                                         sptl += ((int) argv[argi][i]) - 0x30;
527                                 }
528                                 if (argv[argi][i] != '\0')
529                                 {
530                                         printf("Error: Invalid number.\n\n");
531                                         Usage();
532                                         return 1;
533                                 }
534                                 if (i > 23) // i - 3 > 20
535                                 {
536                                         printf("Error: Number is too large.\n\n");
537                                         Usage();
538                                         return 1;
539                                 }                       
540                         }
541
542                         else if (strncmp(argv[argi], "-eptl=", 6) == 0 && (argsUsed & 0x2) == 0)
543                         {
544                                 // Maximum index for ending points
545                                 argsUsed |= 0x2;
546                                 eptl = 0;
547                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
548                                 {
549                                         eptl *= 10;
550                                         eptl += ((int) argv[argi][i]) - 0x30;
551                                 }
552                                 if (argv[argi][i] != '\0')
553                                 {
554                                         printf("Error: Invalid number.\n\n");
555                                         Usage();
556                                         return 1;
557                                 }
558                                 if (i > 23) // i - 3 > 20
559                                 {
560                                         printf("Error: Number is too large.\n\n");
561                                         Usage();
562                                         return 1;
563                                 }                       
564                         }
565                         else if(strncmp(argv[argi], "-usecp=", 7) == 0 && (argsUsed & 0x4) == 0)
566                         {
567                                 argsUsed |= 0x4;
568                                 hascp = 1;
569                                 usecp = 0;
570                                 unsigned int cppos = 0;
571                                 for(i = 7; argv[argi][i] != ' ' && argv[argi][i] != '\n' && argv[argi][i] != 0;)
572                                 {
573                                         if(cppositions.size() > 0) i++;
574                                         for (; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
575                                         {
576                                                 cppos *= 10;
577                                                 cppos += ((int) argv[argi][i]) - 0x30;
578                                         }
579 /*                                      if(argv[argi][i] == ',')
580                                         {*/
581                                                 cppositions.push_back(cppos);
582                                                 usecp++;
583                                                 cppos = 0;
584                                         //}
585                                 }
586                                 if (argv[argi][i] != '\0')
587                                 {
588                                         printf("Error: Invalid number.\n\n");
589                                         Usage();
590                                         return 1;
591                                 }
592                                 if (usecp > 16) // i - 3 > 20
593                                 {
594                                         printf("Error: Number is too large.\n\n");
595                                         Usage();
596                                         return 1;
597                                 }                               
598                                 else printf("Using %i bits of the checkpoints\n", usecp);
599                         }
600
601                 }               
602         }
603         vector<string> vPathName;
604 #ifdef WIN32
605         string sWildCharPathName = argv[1];
606         GetTableList(sWildCharPathName, vPathName);
607 #else
608         GetTableList(argc, argv, vPathName);
609 #endif
610         if (vPathName.size() == 0)
611         {
612                 printf("no rainbow table found\n");
613                 return 0;
614         }
615         for (int i = 0; i < vPathName.size(); i++)
616         {
617                 string sResultFile;
618                 int n = vPathName[i].find_last_of('\\');
619                 if (n != -1)
620                         sResultFile = vPathName[i].substr(n+1, vPathName[i].length()) + "i2";
621                 else 
622                         sResultFile = vPathName[i] + "i2"; // Resulting file is .rt, not .rti
623                 printf("Using %i of 64 bits. sptl: %i, eptl: %i, cp: %i. Chains will be %i bytes in size\n", (sptl + eptl + usecp), sptl, eptl, usecp, ((sptl + eptl + usecp) / 8));
624                 if(sptl + eptl + usecp > 64)
625                 {
626                         exit(1);
627                 }
628                 ConvertRainbowTable(vPathName[i], sResultFile, sptl, eptl, showDistribution, hascp, usecp, cppositions);
629                 printf("\n");
630         }
631         return 0;
632 }