]> git.sesse.net Git - freerainbowtables/blob - Client Applications/converti2/converti2.cpp
initial
[freerainbowtables] / Client Applications / converti2 / converti2.cpp
1 #include <string>
2 #include <vector>
3 #ifdef _WIN32
4 #include <io.h>
5 #else
6         #include <sys/types.h>
7         #include <sys/stat.h>
8         #include <unistd.h>
9 #endif
10
11 #include <time.h>
12 #include <math.h>
13 #include <vector>
14 #include "Public.h"
15 #include "MemoryPool.h"
16 using namespace std;
17
18 void Usage()
19 {
20         printf("converti2 - Original to Indexed rainbow table converter\n");
21         printf("by Martin Westergaard <martinwj2005@gmail.com>\n");
22         printf("http://www.freerainbowtables.com\n\n");
23
24         printf("usage: converti2 rainbow_table_pathname\n");
25         printf("rainbow_table_pathname: pathname of the rainbow table(s), wildchar(*, ?) supported\n");
26         printf("\n");
27         printf("example: converti2 *.rt\n");
28         printf("         converti2 md5_*.rt\n");
29 }
30
31
32 int GetMaxBits(uint64 highvalue)
33 {
34         if(highvalue < 0x02)
35                 return 1;
36         if(highvalue < 0x04)
37                 return 2;
38         if(highvalue < 0x08)
39                 return 3;
40         if(highvalue < 0x10)
41                 return 4;
42         if(highvalue < 0x20)
43                 return 5;
44         if(highvalue < 0x40)
45                 return 6;
46         if(highvalue < 0x80)
47                 return 7;
48         if(highvalue < 0x100)
49                 return 8;
50         if(highvalue < 0x200)
51                 return 9;
52         if(highvalue < 0x400)
53                 return 10;
54         if(highvalue < 0x800)
55                 return 11;
56         if(highvalue < 0x1000)
57                 return 12;
58         if(highvalue < 0x2000)
59                 return 13;
60         if(highvalue < 0x4000)
61                 return 14;
62         if(highvalue < 0x8000)
63                 return 15;
64         if(highvalue < 0x10000)
65                 return 16;
66         if(highvalue < 0x20000)
67                 return 17;
68         if(highvalue < 0x40000)
69                 return 18;
70         if(highvalue < 0x80000)
71                 return 19;
72         if(highvalue < 0x100000)
73                 return 20;
74         if(highvalue < 0x200000)
75                 return 21;
76         if(highvalue < 0x400000)
77                 return 22;
78         if(highvalue < 0x800000)
79                 return 23;
80         if(highvalue < 0x1000000)
81                 return 24;
82         if(highvalue < 0x2000000)
83                 return 25;
84         if(highvalue < 0x4000000)
85                 return 26;
86         if(highvalue < 0x8000000)
87                 return 27;
88         if(highvalue < 0x10000000)
89                 return 28;
90         if(highvalue < 0x20000000)
91                 return 29;
92         if(highvalue < 0x40000000)
93                 return 30;
94         if(highvalue < 0x80000000)
95                 return 31;
96 #ifdef WIN32
97         if(highvalue < 0x0000000100000000I64)
98                 return 32;
99         if(highvalue < 0x0000000200000000I64)
100                 return 33;
101         if(highvalue < 0x0000000400000000I64)
102                 return 34;
103         if(highvalue < 0x0000000800000000I64)
104                 return 35;
105         if(highvalue < 0x0000001000000000I64)
106                 return 36;
107         if(highvalue < 0x0000002000000000I64)
108                 return 37;
109         if(highvalue < 0x0000004000000000I64)
110                 return 38;
111         if(highvalue < 0x0000008000000000I64)
112                 return 39;
113         if(highvalue < 0x0000010000000000I64)
114                 return 40;
115         if(highvalue < 0x0000020000000000I64)
116                 return 41;
117         if(highvalue < 0x0000040000000000I64)
118                 return 42;
119         if(highvalue < 0x0000080000000000I64)
120                 return 43;
121         if(highvalue < 0x0000100000000000I64)
122                 return 44;
123         if(highvalue < 0x0000200000000000I64)
124                 return 45;
125         if(highvalue < 0x0000400000000000I64)
126                 return 46;
127         if(highvalue < 0x0000800000000000I64)
128                 return 47;
129         if(highvalue < 0x0001000000000000I64)
130                 return 48;
131         if(highvalue < 0x0002000000000000I64)
132                 return 49;
133         if(highvalue < 0x0004000000000000I64)
134                 return 50;
135         if(highvalue < 0x0008000000000000I64)
136                 return 51;
137         if(highvalue < 0x0010000000000000I64)
138                 return 52;
139         if(highvalue < 0x0020000000000000I64)
140                 return 53;
141         if(highvalue < 0x0040000000000000I64)
142                 return 54;
143         if(highvalue < 0x0080000000000000I64)
144                 return 55;
145         if(highvalue < 0x0100000000000000I64)
146                 return 56;
147         if(highvalue < 0x0200000000000000I64)
148                 return 57;
149         if(highvalue < 0x0400000000000000I64)
150                 return 58;
151         if(highvalue < 0x0800000000000000I64)
152                 return 59;
153         if(highvalue < 0x1000000000000000I64)
154                 return 60;
155         if(highvalue < 0x2000000000000000I64)
156                 return 61;
157         if(highvalue < 0x4000000000000000I64)
158                 return 62;
159         if(highvalue < 0x8000000000000000I64)
160                 return 63;
161 #else
162         if(highvalue < 0x0000000100000000LL)
163                 return 32;
164         if(highvalue < 0x0000000200000000LL)
165                 return 33;
166         if(highvalue < 0x0000000400000000LL)
167                 return 34;
168         if(highvalue < 0x0000000800000000LL)
169                 return 35;
170         if(highvalue < 0x0000001000000000LL)
171                 return 36;
172         if(highvalue < 0x0000002000000000LL)
173                 return 37;
174         if(highvalue < 0x0000004000000000LL)
175                 return 38;
176         if(highvalue < 0x0000008000000000LL)
177                 return 39;
178         if(highvalue < 0x0000010000000000LL)
179                 return 40;
180         if(highvalue < 0x0000020000000000LL)
181                 return 41;
182         if(highvalue < 0x0000040000000000LL)
183                 return 42;
184         if(highvalue < 0x0000080000000000LL)
185                 return 43;
186         if(highvalue < 0x0000100000000000LL)
187                 return 44;
188         if(highvalue < 0x0000200000000000LL)
189                 return 45;
190         if(highvalue < 0x0000400000000000LL)
191                 return 46;
192         if(highvalue < 0x0000800000000000LL)
193                 return 47;
194         if(highvalue < 0x0001000000000000LL)
195                 return 48;
196         if(highvalue < 0x0002000000000000LL)
197                 return 49;
198         if(highvalue < 0x0004000000000000LL)
199                 return 50;
200         if(highvalue < 0x0008000000000000LL)
201                 return 51;
202         if(highvalue < 0x0010000000000000LL)
203                 return 52;
204         if(highvalue < 0x0020000000000000LL)
205                 return 53;
206         if(highvalue < 0x0040000000000000LL)
207                 return 54;
208         if(highvalue < 0x0080000000000000LL)
209                 return 55;
210         if(highvalue < 0x0100000000000000LL)
211                 return 56;
212         if(highvalue < 0x0200000000000000LL)
213                 return 57;
214         if(highvalue < 0x0400000000000000LL)
215                 return 58;
216         if(highvalue < 0x0800000000000000LL)
217                 return 59;
218         if(highvalue < 0x1000000000000000LL)
219                 return 60;
220         if(highvalue < 0x2000000000000000LL)
221                 return 61;
222         if(highvalue < 0x4000000000000000LL)
223                 return 62;
224         if(highvalue < 0x8000000000000000LL)
225                 return 63;
226
227 #endif
228         return 64;
229
230 }
231
232 #ifdef _WIN32
233 void GetTableList(string sWildCharPathName, vector<string>& vPathName)
234 {
235         vPathName.clear();
236
237         string sPath;
238         int n = sWildCharPathName.find_last_of('\\');
239         if (n != -1)
240                 sPath = sWildCharPathName.substr(0, n + 1);
241
242         _finddata_t fd;
243         long handle = _findfirst(sWildCharPathName.c_str(), &fd);
244         if (handle != -1)
245         {
246                 do
247                 {
248                         string sName = fd.name;
249                         if (sName != "." && sName != ".." && !(fd.attrib & _A_SUBDIR))
250                         {
251                                 string sPathName = sPath + sName;
252                                 vPathName.push_back(sPathName);
253                         }
254                 } while (_findnext(handle, &fd) == 0);
255
256                 _findclose(handle);
257         }
258 }
259 #else
260 void GetTableList(int argc, char* argv[], vector<string>& vPathName)
261 {
262         vPathName.clear();
263
264         int i;
265         for (i = 1; i < argc; i++)
266         {
267                 string sPathName = argv[i];
268                 struct stat buf;
269                 if (lstat(sPathName.c_str(), &buf) == 0)
270                 {
271                         if (S_ISREG(buf.st_mode))
272                                 vPathName.push_back(sPathName);
273
274                 }
275         }
276 }
277 #endif
278
279
280 void ConvertRainbowTable(string sPathName, string sResultFileName, unsigned int rti_startptlength, unsigned int rti_endptlength, int showDistribution, int hascp, int rti_cplength, vector<unsigned int> rti_cppos)
281 {
282 #ifdef _WIN32
283         int nIndex = sPathName.find_last_of('\\');
284 #else
285         int nIndex = sPathName.find_last_of('/');
286 #endif
287         string sFileName;
288         if (nIndex != -1)
289                 sFileName = sPathName.substr(nIndex + 1);
290         else
291                 sFileName = sPathName;
292         // Info
293         printf("%s:\n", sFileName.c_str());
294         FILE* file = fopen(sPathName.c_str(), "rb");
295         FILE* fileR = fopen(sResultFileName.c_str(), "wb");
296         unsigned int distribution[64] = {0};
297         unsigned int numProcessedChains = 0;
298         
299         if (file != NULL && fileR != NULL)
300         {
301                 // File length check
302                 unsigned int nFileLen = GetFileLen(file);
303                 unsigned int nTotalChainCount = 0;
304                 if(hascp == 0) nTotalChainCount = nFileLen / 16;
305                 else nTotalChainCount = nFileLen / 18;
306                 if ((hascp == 0 && nFileLen % 16 != 0) || (hascp == 1 && nFileLen % 18 != 0))
307                 {
308                         printf("file length mismatch\n");
309                 }
310                 else
311                 {
312                         static CMemoryPool mp;
313                         unsigned int nAllocatedSize;
314                         RainbowChainCP* pChain = (RainbowChainCP*)mp.Allocate(nFileLen, nAllocatedSize);
315                         
316                         unsigned int chainrowsize = ceil((float)(rti_startptlength + rti_endptlength + rti_cplength) / 8) * 8 ; // The size in bits (in whole bytes)
317                         unsigned int chainrowsizebytes = chainrowsize / 8;
318
319
320                         if (pChain != NULL)
321                         {
322                                 nAllocatedSize = nAllocatedSize / sizeof(RainbowChainCP) * sizeof(RainbowChainCP);
323                                 fseek(file, 0, SEEK_SET);
324                                 uint64 curPrefix = 0, prefixStart = 0;
325                                 vector<IndexRow> indexes;
326                                 int nRainbowChainCountRead = 0;
327                                 while (true)    // Chunk read loop
328                                 {
329 /*                                      if (ftell(file) == nFileLen)
330                                                 break;*/
331                                         int nReadThisRound;
332                                         memset(pChain, 0x00, nAllocatedSize);
333                                         printf("reading...\n");
334                                         clock_t t1 = clock();
335                                         for(nReadThisRound = 0; nReadThisRound < nAllocatedSize / sizeof(RainbowChainCP) && nRainbowChainCountRead < nTotalChainCount; nReadThisRound++)
336                                         {                                               
337                                                 if(fread(&pChain[nReadThisRound], 16, 1, file) != 1) 
338                                                 { 
339                                                         printf("Error reading file\n"); exit(1);
340                                                 }
341                                                 if(hascp == 1)
342                                                 {
343                                                         if(fread(&pChain[nReadThisRound].nCheckPoint, 2, 1, file) != 1) 
344                                                         { 
345                                                                 printf("Error reading file\n"); exit(2);
346                                                         }
347                                                 }
348                                                 nRainbowChainCountRead++;
349                                         }
350                                         clock_t t2 = clock();
351                                         float fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
352                                         int nDataRead = nRainbowChainCountRead * 16;
353                                         if(hascp == 1) nDataRead += nRainbowChainCountRead * 2; // Add the index readings too
354                                         printf("%u bytes read, disk access time: %.2f s\n", nDataRead , fTime);
355                                         t1 = clock();
356
357                                         for(int i = 0; i < nReadThisRound; i++)
358                                         {
359                                                 if(showDistribution == 1)
360                                                 {
361                                                         distribution[GetMaxBits(pChain[i].nIndexS)-1]++;
362                                                 }
363                                                 else
364                                                 {
365                                                         uint64 chainrow = pChain[i].nIndexS; // Insert the complete start point                                                          
366                                                         chainrow |= ((uint64)pChain[i].nIndexE & (0xffffffff >> (32 - rti_endptlength))) << rti_startptlength; // 
367                                                         if(hascp == 1 && rti_cplength > 0) 
368                                                         {
369                                                                 chainrow |= (uint64)pChain[i].nCheckPoint << rti_startptlength + rti_endptlength;
370                                                         }
371                                                         fwrite(&chainrow, 1, chainrowsizebytes, fileR);                 
372                                                         uint64 prefix = pChain[i].nIndexE >> rti_endptlength;
373                                                         if(i == 0) curPrefix = prefix;
374                                                         if(prefix != curPrefix && numProcessedChains - prefixStart > 0)
375                                                         {
376                                                                         if(prefix < curPrefix)
377                                                                         {
378                                                                                 printf("**** Error writeChain(): Prefix is smaller than previous prefix. %llu < %llu****\n", prefix, curPrefix);
379                                                                                 exit(1);                                                                        
380                                                                         }
381                                                                         //unsigned char index[11] = {0}; // [0 - 10]
382                                                                         unsigned int numchains = numProcessedChains - prefixStart;
383                                                                         IndexRow index;
384                                                                         index.prefix = curPrefix;
385 //                                                                              index.prefixstart = prefixStart;
386                                                                         index.numchains = numchains;
387                                                                         indexes.push_back(index);
388                                                                         prefixStart = numProcessedChains;
389                                                                         curPrefix = prefix; 
390                                                         }
391                                                 }
392                                                 numProcessedChains++;
393                                         }               
394                                         t2 = clock();
395                                         fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
396                                         printf("conversion time: %.2f s\n", fTime);             
397                                         if(nRainbowChainCountRead == nTotalChainCount)
398                                                 break;
399                                         if(showDistribution == 1)
400                                         {
401                                                 for(int i = 0; i < 64; i++)
402                                                 {
403                                                         printf("%u - %u\n", (i+1), distribution[i]);
404                                                 }
405                                                 return;
406                                         }
407
408                                 }
409         
410                 
411         
412                                 // We need to write the last index down
413                                 IndexRow index;
414                                 index.prefix = curPrefix;
415                                 index.prefixstart = prefixStart;
416                                 index.numchains = numProcessedChains - prefixStart;
417                                 indexes.push_back(index);
418
419                                 IndexRow high = {0}; // Used to find the highest numbers. This tells us how much we can pack the index bits
420                                 for(int i = 0; i < indexes.size(); i++)
421                                 {
422                                         if(indexes[i].numchains > high.numchains)
423                                                 high.numchains = indexes[i].numchains;
424 /*                                              if(indexes[i].prefixstart > high.prefixstart)
425                                                 high.prefixstart = indexes[i].prefixstart;
426                                         if(indexes[i].prefix > high.prefix)
427                                                 high.prefix = indexes[i].prefix;
428 */
429                                 }
430                                 high.prefix = indexes[indexes.size()-1].prefix; // The last prefix is always the highest prefix
431 //                                      unsigned int m_rti_index_prefixlength = GetMaxBits(high.prefix);
432                                 unsigned int m_rti_index_numchainslength = GetMaxBits(high.numchains);
433 //                                      unsigned int m_rti_index_indexlength = GetMaxBits(high.prefixstart);
434                                 unsigned int m_indexrowsize = ceil((float)(/*m_rti_index_indexlength + */m_rti_index_numchainslength) / 8) * 8; // The size in bits (in whole bytes)    
435                                 unsigned int m_indexrowsizebytes = m_indexrowsize / 8;
436                                 FILE *pFileIndex = fopen(sResultFileName.append(".index").c_str(), "wb");
437                                 fwrite("RTI2", 1, 4, pFileIndex);
438                                 fwrite(&rti_startptlength, 1, 1, pFileIndex);
439                                 fwrite(&rti_endptlength, 1, 1, pFileIndex);
440                                 fwrite(&rti_cplength, 1, 1, pFileIndex);
441 //                                      fwrite(&m_rti_index_indexlength , 1, 1, pFileIndex);
442                                 fwrite(&m_rti_index_numchainslength, 1, 1, pFileIndex);
443                                 for(int i = 0; i < rti_cppos.size(); i++)
444                                 {
445                                         fwrite(&rti_cppos[i], 1, 4, pFileIndex); // The position of the checkpoints
446                                 }
447 //                                      fwrite(&m_rti_index_prefixlength, 1, 1, pFileIndex);
448                                 int zero = 0;
449                                 fwrite(&indexes[0].prefix, 1, 8, pFileIndex); // Write the first prefix
450                                 unsigned int lastPrefix = 0;
451                                 for(int i = 0; i < indexes.size(); i++)
452                                 {
453                                         if(i == 0)
454                                                 lastPrefix = indexes[0].prefix;
455                                         unsigned int indexrow = 0;
456                                         // Checks how big a distance there is between the current and the next prefix. eg cur is 3 and next is 10 = 7.
457                                         unsigned int diffSize = indexes[i].prefix - lastPrefix; 
458                                         if(i > 0 && diffSize > 1)
459                                         {
460                                                 //indexrow |= indexes[i].prefixstart;
461                                                 //printf("Diffsize is %u\n", diffSize);
462
463                                                 // then write the distance amount of 00's
464                                                 for(int j = 1; j < diffSize; j++)
465                                                 {                                                               
466                                                         fwrite(&zero, 1, m_indexrowsizebytes, pFileIndex);
467                                                 }
468                                         }                                       
469                                         fwrite(&indexes[i].numchains, 1, m_indexrowsizebytes, pFileIndex);
470                                         lastPrefix = indexes[i].prefix;
471                                 }
472                                 fclose(pFileIndex);
473                         }
474                         else printf("memory allocation fail\n");
475
476                         
477                                         // Already finished?
478
479                 }
480                 fclose(file);
481         }
482         else
483                 printf("can't open file\n");
484
485 }
486
487 int main(int argc, char* argv[])
488 {
489         int argi = 1, i, argsUsed = 0;
490         unsigned int sptl = 40, eptl = 16;
491         int showDistribution = 0;
492         int usecp = 0;// How many bits to use from the index
493         int hascp = 0; 
494         vector<unsigned int> cppositions;
495         if (argc == 1)
496         {
497                 Usage();                
498                 return 0;
499         }
500         else if(argc > 2)
501         {
502                 for (; argi < argc; argi++)
503                 {
504                         if (strcmp(argv[argi], "-d") == 0 && (argsUsed & 0x8) == 0)
505                         {
506                                 // Enable verbose mode
507                                 argsUsed |= 0x8;                                
508                                 showDistribution = 1;
509                         }                       
510                         else if (strncmp(argv[argi], "-sptl=", 6) == 0 && (argsUsed & 0x1) == 0)
511                         {
512                                 // Maximum index for starting point
513                                 argsUsed |= 0x1;
514                                 sptl = 0;
515                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
516                                 {
517                                         sptl *= 10;
518                                         sptl += ((int) argv[argi][i]) - 0x30;
519                                 }
520                                 if (argv[argi][i] != '\0')
521                                 {
522                                         printf("Error: Invalid number.\n\n");
523                                         Usage();
524                                         return 1;
525                                 }
526                                 if (i > 23) // i - 3 > 20
527                                 {
528                                         printf("Error: Number is too large.\n\n");
529                                         Usage();
530                                         return 1;
531                                 }                       
532                         }
533
534                         else if (strncmp(argv[argi], "-eptl=", 6) == 0 && (argsUsed & 0x2) == 0)
535                         {
536                                 // Maximum index for ending points
537                                 argsUsed |= 0x2;
538                                 eptl = 0;
539                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
540                                 {
541                                         eptl *= 10;
542                                         eptl += ((int) argv[argi][i]) - 0x30;
543                                 }
544                                 if (argv[argi][i] != '\0')
545                                 {
546                                         printf("Error: Invalid number.\n\n");
547                                         Usage();
548                                         return 1;
549                                 }
550                                 if (i > 23) // i - 3 > 20
551                                 {
552                                         printf("Error: Number is too large.\n\n");
553                                         Usage();
554                                         return 1;
555                                 }                       
556                         }
557                         else if(strncmp(argv[argi], "-usecp=", 7) == 0 && (argsUsed & 0x4) == 0)
558                         {
559                                 argsUsed |= 0x4;
560                                 hascp = 1;
561                                 usecp = 0;
562                                 unsigned int cppos = 0;
563                                 for(i = 7; argv[argi][i] != ' ' && argv[argi][i] != '\n' && argv[argi][i] != 0;)
564                                 {
565                                         if(cppositions.size() > 0) i++;
566                                         for (; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
567                                         {
568                                                 cppos *= 10;
569                                                 cppos += ((int) argv[argi][i]) - 0x30;
570                                         }
571 /*                                      if(argv[argi][i] == ',')
572                                         {*/
573                                                 cppositions.push_back(cppos);
574                                                 usecp++;
575                                                 cppos = 0;
576                                         //}
577                                 }
578                                 if (argv[argi][i] != '\0')
579                                 {
580                                         printf("Error: Invalid number.\n\n");
581                                         Usage();
582                                         return 1;
583                                 }
584                                 if (usecp > 16) // i - 3 > 20
585                                 {
586                                         printf("Error: Number is too large.\n\n");
587                                         Usage();
588                                         return 1;
589                                 }                               
590                                 else printf("Using %i bits of the checkpoints\n", usecp);
591                         }
592
593                 }               
594         }
595         vector<string> vPathName;
596 #ifdef WIN32
597         string sWildCharPathName = argv[1];
598         GetTableList(sWildCharPathName, vPathName);
599 #else
600         GetTableList(argc, argv, vPathName);
601 #endif
602         if (vPathName.size() == 0)
603         {
604                 printf("no rainbow table found\n");
605                 return 0;
606         }
607         for (int i = 0; i < vPathName.size(); i++)
608         {
609                 string sResultFile;
610                 int n = vPathName[i].find_last_of('\\');
611                 if (n != -1)
612                         sResultFile = vPathName[i].substr(n+1, vPathName[i].length()) + "i2";
613                 else 
614                         sResultFile = vPathName[i] + "i2"; // Resulting file is .rt, not .rti
615                 printf("Using %i of 64 bits\n", (sptl + eptl + usecp));
616                 if(sptl + eptl + usecp > 64)
617                 {
618                         exit(1);
619                 }
620                 ConvertRainbowTable(vPathName[i], sResultFile, sptl, eptl, showDistribution, hascp, usecp, cppositions);
621                 printf("\n");
622         }
623         return 0;
624 }