]> git.sesse.net Git - freerainbowtables/blob - Client Applications/converti2/converti2.cpp
13b492e64c78cb9d304b1fb86aa71d12ae4140b2
[freerainbowtables] / Client Applications / converti2 / converti2.cpp
1 #include <string>
2 #include <vector>
3 #ifdef _WIN32
4         #include <io.h>
5         #include <conio.h>
6 #else
7         #include <sys/types.h>
8         #include <sys/stat.h>
9         #include <unistd.h>
10 #endif
11
12 #include <stdlib.h>
13 #include <string.h>
14 #include <time.h>
15 #include <math.h>
16 #include <vector>
17 #include <sstream>
18 #include "Public.h"
19 #include "MemoryPool.h"
20 #include "RTIReader.h"
21 #include "RTReader.h"
22
23 using namespace std;
24
25 void Usage()
26 {
27         printf("converti2 - Original to Indexed rainbow table converter\n");
28         printf("by Martin Westergaard <martinwj2005@gmail.com>\n");
29         printf("http://www.freerainbowtables.com\n\n");
30
31         printf("usage: converti2 rainbow_table_pathname\n");
32         printf("rainbow_table_pathname: pathname of the rainbow table(s), wildchar(*, ?) supported\n");
33         printf("\n");
34         printf("example: converti2 *.rt\n");
35         printf("         converti2 md5_*.rt\n");
36 }
37
38
39 int GetMaxBits(uint64 highvalue)
40 {
41         if(highvalue < 0x02)
42                 return 1;
43         if(highvalue < 0x04)
44                 return 2;
45         if(highvalue < 0x08)
46                 return 3;
47         if(highvalue < 0x10)
48                 return 4;
49         if(highvalue < 0x20)
50                 return 5;
51         if(highvalue < 0x40)
52                 return 6;
53         if(highvalue < 0x80)
54                 return 7;
55         if(highvalue < 0x100)
56                 return 8;
57         if(highvalue < 0x200)
58                 return 9;
59         if(highvalue < 0x400)
60                 return 10;
61         if(highvalue < 0x800)
62                 return 11;
63         if(highvalue < 0x1000)
64                 return 12;
65         if(highvalue < 0x2000)
66                 return 13;
67         if(highvalue < 0x4000)
68                 return 14;
69         if(highvalue < 0x8000)
70                 return 15;
71         if(highvalue < 0x10000)
72                 return 16;
73         if(highvalue < 0x20000)
74                 return 17;
75         if(highvalue < 0x40000)
76                 return 18;
77         if(highvalue < 0x80000)
78                 return 19;
79         if(highvalue < 0x100000)
80                 return 20;
81         if(highvalue < 0x200000)
82                 return 21;
83         if(highvalue < 0x400000)
84                 return 22;
85         if(highvalue < 0x800000)
86                 return 23;
87         if(highvalue < 0x1000000)
88                 return 24;
89         if(highvalue < 0x2000000)
90                 return 25;
91         if(highvalue < 0x4000000)
92                 return 26;
93         if(highvalue < 0x8000000)
94                 return 27;
95         if(highvalue < 0x10000000)
96                 return 28;
97         if(highvalue < 0x20000000)
98                 return 29;
99         if(highvalue < 0x40000000)
100                 return 30;
101         if(highvalue < 0x80000000)
102                 return 31;
103 #if defined(_WIN32) && !defined(__GNUC__)
104         if(highvalue < 0x0000000100000000I64)
105                 return 32;
106         if(highvalue < 0x0000000200000000I64)
107                 return 33;
108         if(highvalue < 0x0000000400000000I64)
109                 return 34;
110         if(highvalue < 0x0000000800000000I64)
111                 return 35;
112         if(highvalue < 0x0000001000000000I64)
113                 return 36;
114         if(highvalue < 0x0000002000000000I64)
115                 return 37;
116         if(highvalue < 0x0000004000000000I64)
117                 return 38;
118         if(highvalue < 0x0000008000000000I64)
119                 return 39;
120         if(highvalue < 0x0000010000000000I64)
121                 return 40;
122         if(highvalue < 0x0000020000000000I64)
123                 return 41;
124         if(highvalue < 0x0000040000000000I64)
125                 return 42;
126         if(highvalue < 0x0000080000000000I64)
127                 return 43;
128         if(highvalue < 0x0000100000000000I64)
129                 return 44;
130         if(highvalue < 0x0000200000000000I64)
131                 return 45;
132         if(highvalue < 0x0000400000000000I64)
133                 return 46;
134         if(highvalue < 0x0000800000000000I64)
135                 return 47;
136         if(highvalue < 0x0001000000000000I64)
137                 return 48;
138         if(highvalue < 0x0002000000000000I64)
139                 return 49;
140         if(highvalue < 0x0004000000000000I64)
141                 return 50;
142         if(highvalue < 0x0008000000000000I64)
143                 return 51;
144         if(highvalue < 0x0010000000000000I64)
145                 return 52;
146         if(highvalue < 0x0020000000000000I64)
147                 return 53;
148         if(highvalue < 0x0040000000000000I64)
149                 return 54;
150         if(highvalue < 0x0080000000000000I64)
151                 return 55;
152         if(highvalue < 0x0100000000000000I64)
153                 return 56;
154         if(highvalue < 0x0200000000000000I64)
155                 return 57;
156         if(highvalue < 0x0400000000000000I64)
157                 return 58;
158         if(highvalue < 0x0800000000000000I64)
159                 return 59;
160         if(highvalue < 0x1000000000000000I64)
161                 return 60;
162         if(highvalue < 0x2000000000000000I64)
163                 return 61;
164         if(highvalue < 0x4000000000000000I64)
165                 return 62;
166         if(highvalue < 0x8000000000000000I64)
167                 return 63;
168 #else
169         if(highvalue < 0x0000000100000000LL)
170                 return 32;
171         if(highvalue < 0x0000000200000000LL)
172                 return 33;
173         if(highvalue < 0x0000000400000000LL)
174                 return 34;
175         if(highvalue < 0x0000000800000000LL)
176                 return 35;
177         if(highvalue < 0x0000001000000000LL)
178                 return 36;
179         if(highvalue < 0x0000002000000000LL)
180                 return 37;
181         if(highvalue < 0x0000004000000000LL)
182                 return 38;
183         if(highvalue < 0x0000008000000000LL)
184                 return 39;
185         if(highvalue < 0x0000010000000000LL)
186                 return 40;
187         if(highvalue < 0x0000020000000000LL)
188                 return 41;
189         if(highvalue < 0x0000040000000000LL)
190                 return 42;
191         if(highvalue < 0x0000080000000000LL)
192                 return 43;
193         if(highvalue < 0x0000100000000000LL)
194                 return 44;
195         if(highvalue < 0x0000200000000000LL)
196                 return 45;
197         if(highvalue < 0x0000400000000000LL)
198                 return 46;
199         if(highvalue < 0x0000800000000000LL)
200                 return 47;
201         if(highvalue < 0x0001000000000000LL)
202                 return 48;
203         if(highvalue < 0x0002000000000000LL)
204                 return 49;
205         if(highvalue < 0x0004000000000000LL)
206                 return 50;
207         if(highvalue < 0x0008000000000000LL)
208                 return 51;
209         if(highvalue < 0x0010000000000000LL)
210                 return 52;
211         if(highvalue < 0x0020000000000000LL)
212                 return 53;
213         if(highvalue < 0x0040000000000000LL)
214                 return 54;
215         if(highvalue < 0x0080000000000000LL)
216                 return 55;
217         if(highvalue < 0x0100000000000000LL)
218                 return 56;
219         if(highvalue < 0x0200000000000000LL)
220                 return 57;
221         if(highvalue < 0x0400000000000000LL)
222                 return 58;
223         if(highvalue < 0x0800000000000000LL)
224                 return 59;
225         if(highvalue < 0x1000000000000000LL)
226                 return 60;
227         if(highvalue < 0x2000000000000000LL)
228                 return 61;
229         if(highvalue < 0x4000000000000000LL)
230                 return 62;
231         if(highvalue < 0x8000000000000000LL)
232                 return 63;
233
234 #endif
235         return 64;
236
237 }
238
239 #ifdef _WIN32
240 void GetTableList(string sWildCharPathName, vector<string>& vPathName)
241 {
242         vPathName.clear();
243
244         string sPath;
245         int n = sWildCharPathName.find_last_of('\\');
246         if (n != -1)
247                 sPath = sWildCharPathName.substr(0, n + 1);
248
249         _finddata_t fd;
250         long handle = _findfirst(sWildCharPathName.c_str(), &fd);
251         if (handle != -1) {
252                 do      {
253                         string sName = fd.name;
254                         if (sName != "." && sName != ".." && !(fd.attrib & _A_SUBDIR))  {
255                                 string sPathName = sPath + sName;
256                                 vPathName.push_back(sPathName);
257                         }
258                 } while (_findnext(handle, &fd) == 0);
259
260                 _findclose(handle);
261         }
262 }
263 #else
264 void GetTableList(int argc, char* argv[], vector<string>& vPathName)
265 {
266         vPathName.clear();
267
268         int i;
269         for (i = 1; i < argc; i++)
270         {
271                 string sPathName = argv[i];
272                 struct stat buf;
273                 if (lstat(sPathName.c_str(), &buf) == 0)
274                 {
275                         if (S_ISREG(buf.st_mode))
276                                 vPathName.push_back(sPathName);
277
278                 }
279         }
280 }
281 #endif
282
283
284 void ConvertRainbowTable(string sPathName, string sResultFileName, unsigned int rti_startptlength, unsigned int rti_endptlength, int showDistribution, int hascp, int rti_cplength, vector<unsigned int> rti_cppos)
285 {
286 #ifdef _WIN32
287         int nIndex = sPathName.find_last_of('\\');
288 #else
289         int nIndex = sPathName.find_last_of('/');
290 #endif
291         string sFileName;
292         if (nIndex != -1) {
293                 sFileName = sPathName.substr(nIndex + 1);
294         }
295         else {
296                 sFileName = sPathName;
297         }
298         unsigned int distribution[64] = {0};
299         unsigned int numProcessedChains = 0;
300         FILE* fileR = NULL;
301         BaseRTReader *reader = NULL;
302         if(sPathName.substr(sPathName.length() - 2, sPathName.length()) == "rt")
303                 reader = (BaseRTReader*)new RTReader(sPathName);
304         else if(sPathName.substr(sPathName.length() - 3, sPathName.length()) == "rti")
305                 reader = (BaseRTReader*)new RTIReader(sPathName);
306         if(reader == NULL) {
307                 printf("%s is not a supported file (Only RT and RTI is supported)\n", sPathName.c_str());
308                 return;
309         }
310         // Info
311         printf("%s:\n", sFileName.c_str());
312         if(showDistribution == 0) {
313                 fileR = fopen(sResultFileName.c_str(), "wb");
314         }
315         if (fileR != NULL || showDistribution == 1) {
316                 // File length check
317
318                 int size = reader->GetChainsLeft() * sizeof(RainbowChain);
319                         static CMemoryPool mp;
320                         unsigned int nAllocatedSize;
321                         RainbowChain* pChain = (RainbowChain*)mp.Allocate(size, nAllocatedSize);                        
322                         uint32 chainrowsize = (uint32)ceil((float)(rti_startptlength + rti_endptlength + rti_cplength) / 8) * 8 ; // The size in bits (in whole bytes)
323                         unsigned int chainrowsizebytes = chainrowsize / 8;
324
325                         if (pChain != NULL)     {
326                                 nAllocatedSize = nAllocatedSize / sizeof(RainbowChain) * sizeof(RainbowChain);
327                                 unsigned int nChains = nAllocatedSize / sizeof(RainbowChain);
328                                 uint64 curPrefix = 0, prefixStart = 0;
329                                 vector<IndexRow> indexes;
330                                 unsigned int chainsLeft;
331                                 while((chainsLeft = reader->GetChainsLeft()) > 0) {
332                                         
333 /*                                      if (ftell(file) == nFileLen)
334                                                 break;*/
335                                         printf("%u chains left to read\n", chainsLeft);
336                                         //int nReadThisRound;
337                                         clock_t t1 = clock();
338                                         printf("reading...\n");
339 #ifdef _MEMORYDEBUG
340                         printf("Grabbing %i chains from file\n", nChains);
341 #endif
342                                         reader->ReadChains(nChains, pChain);
343 #ifdef _MEMORYDEBUG
344                         printf("Recieved %i chains from file\n", nChains);
345 #endif
346                                         clock_t t2 = clock();
347                                         float fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
348                                         printf("reading time: %.2f s\n", fTime);                
349                                         printf("converting %i chains...\n", nChains);
350                                         t1 = clock();
351                                         for(unsigned int i = 0; i < nChains; i++)       {
352                                                 if(showDistribution == 1) {
353                                                         distribution[GetMaxBits(pChain[i].nIndexS)-1]++;
354                                                 }
355                                                 else
356                                                 {
357                                                         uint64 chainrow = pChain[i].nIndexS; // Insert the complete start point                                                          
358                                                         chainrow |= ((uint64)pChain[i].nIndexE & (0xffffffff >> (32 - rti_endptlength))) << rti_startptlength; // 
359 /*                                                      if(hascp == 1 && rti_cplength > 0) {
360                                                                 chainrow |= (uint64)pChain[i].nCheckPoint << rti_startptlength + rti_endptlength;
361                                                         }*/
362                                                         fwrite(&chainrow, 1, chainrowsizebytes, fileR);                 
363                                                         uint64 prefix = pChain[i].nIndexE >> rti_endptlength;
364                                                         if(i == 0) curPrefix = prefix;
365                                                         if(prefix != curPrefix && numProcessedChains - prefixStart > 0) {
366                                                                 if(prefix < curPrefix) {
367                                                                         printf("**** Error writeChain(): Prefix is smaller than previous prefix. %llu < %llu****\n", prefix, curPrefix);
368                                                                         exit(1);                                                                        
369                                                                 }
370                                                                 //unsigned char index[11] = {0}; // [0 - 10]
371                                                                 unsigned int numchains = numProcessedChains - prefixStart;
372                                                                 IndexRow index;
373                                                                 index.prefix = curPrefix;
374 //                                                                              index.prefixstart = prefixStart;
375                                                                 index.numchains = numchains;
376                                                                 indexes.push_back(index);
377                                                                 prefixStart = numProcessedChains;
378                                                                 curPrefix = prefix; 
379                                                         }
380                                                 }
381                                                 numProcessedChains++;
382                                         }               
383                                         t2 = clock();
384                                         fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
385                                         printf("conversion time: %.2f s\n", fTime);             
386                                         if(showDistribution == 1) {
387                                                 for(int i = 0; i < 64; i++)     {
388                                                         printf("%u - %u\n", (i+1), distribution[i]);
389                                                 }
390                                                 delete reader;
391                                                 return;
392                                         }
393
394                                 }
395         
396                 
397         
398                                 // We need to write the last index down
399                                 IndexRow index;
400                                 index.prefix = curPrefix;
401                                 index.prefixstart = prefixStart;
402                                 index.numchains = numProcessedChains - prefixStart;
403                                 indexes.push_back(index);
404
405                                 IndexRow high = {0}; // Used to find the highest numbers. This tells us how much we can pack the index bits
406                                 for(UINT4 i = 0; i < indexes.size(); i++)
407                                 {
408                                         if(indexes[i].numchains > high.numchains)
409                                                 high.numchains = indexes[i].numchains;
410 /*                                              if(indexes[i].prefixstart > high.prefixstart)
411                                                 high.prefixstart = indexes[i].prefixstart;
412                                         if(indexes[i].prefix > high.prefix)
413                                                 high.prefix = indexes[i].prefix;
414 */
415                                 }
416                                 high.prefix = indexes[indexes.size()-1].prefix; // The last prefix is always the highest prefix
417 //                                      unsigned int m_rti_index_prefixlength = GetMaxBits(high.prefix);
418                                 unsigned int m_rti_index_numchainslength = GetMaxBits(high.numchains);
419 //                                      unsigned int m_rti_index_indexlength = GetMaxBits(high.prefixstart);
420                                 uint32 m_indexrowsize = (uint32)ceil((float)(/*m_rti_index_indexlength + */m_rti_index_numchainslength) / 8) * 8; // The size in bits (in whole bytes)  
421                                 unsigned int m_indexrowsizebytes = m_indexrowsize / 8;
422                                 FILE *pFileIndex = fopen(sResultFileName.append(".index").c_str(), "wb");
423                                 fwrite("RTI2", 1, 4, pFileIndex);
424                                 fwrite(&rti_startptlength, 1, 1, pFileIndex);
425                                 fwrite(&rti_endptlength, 1, 1, pFileIndex);
426                                 fwrite(&rti_cplength, 1, 1, pFileIndex);
427 //                                      fwrite(&m_rti_index_indexlength , 1, 1, pFileIndex);
428
429                                 fwrite(&m_rti_index_numchainslength, 1, 1, pFileIndex);
430                                 for(UINT4 i = 0; i < rti_cppos.size(); i++)     {
431                                         fwrite(&rti_cppos[i], 1, 4, pFileIndex); // The position of the checkpoints
432                                 }
433 //                                      fwrite(&m_rti_index_prefixlength, 1, 1, pFileIndex);
434                                 int zero = 0;
435                                 fwrite(&indexes[0].prefix, 1, 8, pFileIndex); // Write the first prefix
436                                 unsigned int lastPrefix = 0;
437                                 for(UINT4 i = 0; i < indexes.size(); i++)       {
438                                         if(i == 0) {
439                                                 lastPrefix = indexes[0].prefix;
440                                         }
441                                         //unsigned int indexrow = 0;
442                                         // Checks how big a distance there is between the current and the next prefix. eg cur is 3 and next is 10 = 7.
443                                         unsigned int diffSize = indexes[i].prefix - lastPrefix; 
444                                         if(i > 0 && diffSize > 1) {
445                                                 //indexrow |= indexes[i].prefixstart;
446                                                 //printf("Diffsize is %u\n", diffSize);
447
448                                                 // then write the distance amount of 00's
449                                                 if(diffSize > 1000) {
450                                                         printf("WARNING! The distance to the next prefix is %i. Do you want to continue writing %i bytes of 0x00? Press y to continue", diffSize, (diffSize*m_indexrowsizebytes));
451                                                         #ifdef _WIN32
452                                                         if ( _getch() != 'y' ) {
453                                                         #else
454                                                         if ( tty_getchar() != 'y' ) {
455                                                         #endif
456                                                                 printf("Aborting...");
457                                                                 exit(1);
458                                                         }
459                                                 }
460                                                 for(UINT4 j = 1; j < diffSize; j++)
461                                                 {                                                               
462                                                         fwrite(&zero, 1, m_indexrowsizebytes, pFileIndex);
463                                                 }
464                                         }                                       
465                                         fwrite(&indexes[i].numchains, 1, m_indexrowsizebytes, pFileIndex);
466                                         lastPrefix = indexes[i].prefix;
467                                 }
468                                 fclose(pFileIndex);
469                         }
470                         else {
471                                 printf("memory allocation fail\n");
472                         }                       
473                         // Already finished?
474         }
475         else {
476                 printf("can't open file\n");
477         }
478         if(reader != NULL)
479                 delete reader;
480         if(fileR != NULL) {
481                 fclose(fileR);
482         }
483                 
484 }
485
486 int main(int argc, char* argv[])
487 {
488         int argi = 1, i, argsUsed = 0;
489         unsigned int sptl = 40, eptl = 16;
490         int showDistribution = 0;
491         int usecp = 0;// How many bits to use from the index
492         int hascp = 0; 
493         vector<unsigned int> cppositions;
494         if (argc == 1) {
495                 Usage();                
496                 return 0;
497         }
498         else if(argc > 2) {
499                 for (; argi < argc; argi++)
500                 {
501                         if(strcmp(argv[argi], "-d") == 0 && (argsUsed & 0x8) == 0) {
502                                 // Enable verbose mode
503                                 argsUsed |= 0x8;                                
504                                 showDistribution = 1;
505                         }                       
506                         else if (strncmp(argv[argi], "-sptl=", 6) == 0 && (argsUsed & 0x1) == 0) {
507                                 // Maximum index for starting point
508                                 argsUsed |= 0x1;
509                                 sptl = 0;
510                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++) {
511                                         sptl *= 10;
512                                         sptl += ((int) argv[argi][i]) - 0x30;
513                                 }
514                                 if (argv[argi][i] != '\0') {
515                                         printf("Error: Invalid number.\n\n");
516                                         Usage();
517                                         return 1;
518                                 }
519                                 if (i > 23) { // i - 3 > 20                             
520                                         printf("Error: Number is too large.\n\n");
521                                         Usage();
522                                         return 1;
523                                 }                       
524                         }
525
526                         else if (strncmp(argv[argi], "-eptl=", 6) == 0 && (argsUsed & 0x2) == 0) {
527                                 // Maximum index for ending points
528                                 argsUsed |= 0x2;
529                                 eptl = 0;
530                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++) {
531                                         eptl *= 10;
532                                         eptl += ((int) argv[argi][i]) - 0x30;
533                                 }
534                                 if (argv[argi][i] != '\0') {
535                                         printf("Error: Invalid number.\n\n");
536                                         Usage();
537                                         return 1;
538                                 }
539                                 if (i > 23) { // i - 3 > 20                             
540                                         printf("Error: Number is too large.\n\n");
541                                         Usage();
542                                         return 1;
543                                 }                       
544                         }
545                         else if(strncmp(argv[argi], "-usecp=", 7) == 0 && (argsUsed & 0x4) == 0) {
546                                 argsUsed |= 0x4;
547                                 hascp = 1;
548                                 usecp = 0;
549                                 unsigned int cppos = 0;
550                                 for(i = 7; argv[argi][i] != ' ' && argv[argi][i] != '\n' && argv[argi][i] != 0;) {
551                                         if(cppositions.size() > 0) i++;
552                                         for (; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
553                                         {
554                                                 cppos *= 10;
555                                                 cppos += ((int) argv[argi][i]) - 0x30;
556                                         }
557 /*                                      if(argv[argi][i] == ',')
558                                         {*/
559                                                 cppositions.push_back(cppos);
560                                                 usecp++;
561                                                 cppos = 0;
562                                         //}
563                                 }
564                                 if (argv[argi][i] != '\0') {
565                                         printf("Error: Invalid number.\n\n");
566                                         Usage();
567                                         return 1;
568                                 }
569                                 if (usecp > 16) { // i - 3 > 20
570                                         printf("Error: Number is too large.\n\n");
571                                         Usage();
572                                         return 1;
573                                 }                               
574                                 else {
575                                         printf("Using %i bits of the checkpoints\n", usecp);
576                                 }
577                         }
578
579                 }               
580         }
581         vector<string> vPathName;
582 #ifdef _WIN32
583         string sWildCharPathName = argv[1];
584         GetTableList(sWildCharPathName, vPathName);
585 #else
586         GetTableList(argc, argv, vPathName);
587 #endif
588         if (vPathName.size() == 0) {
589                 printf("no rainbow table found\n");
590                 return 0;
591         }
592         for (UINT4 i = 0; i < vPathName.size(); i++) {
593                 string sResultFile;
594                 int n = vPathName[i].find_last_of('\\');
595                 if (n != -1) {
596                         if(vPathName[i].substr(vPathName[i].length() - 3, vPathName[i].length()) == "rti")      {
597                                 sResultFile = vPathName[i].substr(n+1, vPathName[i].length()) + "2";                            
598                         }
599                         else {
600                                 sResultFile = vPathName[i].substr(n+1, vPathName[i].length()) + "i2";
601                         }
602                 }
603                 else {
604                         if(vPathName[i].substr(vPathName[i].length() - 3, vPathName[i].length()) == "rti")      {
605                                 sResultFile = vPathName[i] + "2";                               
606                         } else {
607                                 sResultFile = vPathName[i] + "i2"; // Resulting file is .rt, not .rti
608                         }
609                 }
610                 if(usecp == 0 && showDistribution == 0) {
611                         printf("Using %i of 64 bits. sptl: %i, eptl: %i, cp: %i. Chains will be %i bytes in size\n", (sptl + eptl + usecp), sptl, eptl, usecp, ((sptl + eptl + usecp) / 8));
612                 }
613                 if(sptl + eptl + usecp > 64) {
614                         exit(1);
615                 }
616                 ConvertRainbowTable(vPathName[i], sResultFile, sptl, eptl, showDistribution, hascp, usecp, cppositions);
617                 printf("\n");
618         }
619         return 0;
620 }