]> git.sesse.net Git - freerainbowtables/blob - Client Applications/converti2/converti2.cpp
actually commit everything
[freerainbowtables] / Client Applications / converti2 / converti2.cpp
1 #include <string>
2 #include <vector>
3 #ifdef _WIN32
4         #include <io.h>
5         #include <conio.h>
6 #else
7         #include <sys/types.h>
8         #include <sys/stat.h>
9         #include <unistd.h>
10 #endif
11
12 #include <stdlib.h>
13 #include <string.h>
14 #include <time.h>
15 #include <math.h>
16 #include <vector>
17
18 #include "Public.h"
19 #include "MemoryPool.h"
20
21 using namespace std;
22
23 void Usage()
24 {
25         printf("converti2 - Original to Indexed rainbow table converter\n");
26         printf("by Martin Westergaard <martinwj2005@gmail.com>\n");
27         printf("http://www.freerainbowtables.com\n\n");
28
29         printf("usage: converti2 rainbow_table_pathname\n");
30         printf("rainbow_table_pathname: pathname of the rainbow table(s), wildchar(*, ?) supported\n");
31         printf("\n");
32         printf("example: converti2 *.rt\n");
33         printf("         converti2 md5_*.rt\n");
34 }
35
36
37 int GetMaxBits(uint64 highvalue)
38 {
39         if(highvalue < 0x02)
40                 return 1;
41         if(highvalue < 0x04)
42                 return 2;
43         if(highvalue < 0x08)
44                 return 3;
45         if(highvalue < 0x10)
46                 return 4;
47         if(highvalue < 0x20)
48                 return 5;
49         if(highvalue < 0x40)
50                 return 6;
51         if(highvalue < 0x80)
52                 return 7;
53         if(highvalue < 0x100)
54                 return 8;
55         if(highvalue < 0x200)
56                 return 9;
57         if(highvalue < 0x400)
58                 return 10;
59         if(highvalue < 0x800)
60                 return 11;
61         if(highvalue < 0x1000)
62                 return 12;
63         if(highvalue < 0x2000)
64                 return 13;
65         if(highvalue < 0x4000)
66                 return 14;
67         if(highvalue < 0x8000)
68                 return 15;
69         if(highvalue < 0x10000)
70                 return 16;
71         if(highvalue < 0x20000)
72                 return 17;
73         if(highvalue < 0x40000)
74                 return 18;
75         if(highvalue < 0x80000)
76                 return 19;
77         if(highvalue < 0x100000)
78                 return 20;
79         if(highvalue < 0x200000)
80                 return 21;
81         if(highvalue < 0x400000)
82                 return 22;
83         if(highvalue < 0x800000)
84                 return 23;
85         if(highvalue < 0x1000000)
86                 return 24;
87         if(highvalue < 0x2000000)
88                 return 25;
89         if(highvalue < 0x4000000)
90                 return 26;
91         if(highvalue < 0x8000000)
92                 return 27;
93         if(highvalue < 0x10000000)
94                 return 28;
95         if(highvalue < 0x20000000)
96                 return 29;
97         if(highvalue < 0x40000000)
98                 return 30;
99         if(highvalue < 0x80000000)
100                 return 31;
101 #ifdef WIN32
102         if(highvalue < 0x0000000100000000I64)
103                 return 32;
104         if(highvalue < 0x0000000200000000I64)
105                 return 33;
106         if(highvalue < 0x0000000400000000I64)
107                 return 34;
108         if(highvalue < 0x0000000800000000I64)
109                 return 35;
110         if(highvalue < 0x0000001000000000I64)
111                 return 36;
112         if(highvalue < 0x0000002000000000I64)
113                 return 37;
114         if(highvalue < 0x0000004000000000I64)
115                 return 38;
116         if(highvalue < 0x0000008000000000I64)
117                 return 39;
118         if(highvalue < 0x0000010000000000I64)
119                 return 40;
120         if(highvalue < 0x0000020000000000I64)
121                 return 41;
122         if(highvalue < 0x0000040000000000I64)
123                 return 42;
124         if(highvalue < 0x0000080000000000I64)
125                 return 43;
126         if(highvalue < 0x0000100000000000I64)
127                 return 44;
128         if(highvalue < 0x0000200000000000I64)
129                 return 45;
130         if(highvalue < 0x0000400000000000I64)
131                 return 46;
132         if(highvalue < 0x0000800000000000I64)
133                 return 47;
134         if(highvalue < 0x0001000000000000I64)
135                 return 48;
136         if(highvalue < 0x0002000000000000I64)
137                 return 49;
138         if(highvalue < 0x0004000000000000I64)
139                 return 50;
140         if(highvalue < 0x0008000000000000I64)
141                 return 51;
142         if(highvalue < 0x0010000000000000I64)
143                 return 52;
144         if(highvalue < 0x0020000000000000I64)
145                 return 53;
146         if(highvalue < 0x0040000000000000I64)
147                 return 54;
148         if(highvalue < 0x0080000000000000I64)
149                 return 55;
150         if(highvalue < 0x0100000000000000I64)
151                 return 56;
152         if(highvalue < 0x0200000000000000I64)
153                 return 57;
154         if(highvalue < 0x0400000000000000I64)
155                 return 58;
156         if(highvalue < 0x0800000000000000I64)
157                 return 59;
158         if(highvalue < 0x1000000000000000I64)
159                 return 60;
160         if(highvalue < 0x2000000000000000I64)
161                 return 61;
162         if(highvalue < 0x4000000000000000I64)
163                 return 62;
164         if(highvalue < 0x8000000000000000I64)
165                 return 63;
166 #else
167         if(highvalue < 0x0000000100000000LL)
168                 return 32;
169         if(highvalue < 0x0000000200000000LL)
170                 return 33;
171         if(highvalue < 0x0000000400000000LL)
172                 return 34;
173         if(highvalue < 0x0000000800000000LL)
174                 return 35;
175         if(highvalue < 0x0000001000000000LL)
176                 return 36;
177         if(highvalue < 0x0000002000000000LL)
178                 return 37;
179         if(highvalue < 0x0000004000000000LL)
180                 return 38;
181         if(highvalue < 0x0000008000000000LL)
182                 return 39;
183         if(highvalue < 0x0000010000000000LL)
184                 return 40;
185         if(highvalue < 0x0000020000000000LL)
186                 return 41;
187         if(highvalue < 0x0000040000000000LL)
188                 return 42;
189         if(highvalue < 0x0000080000000000LL)
190                 return 43;
191         if(highvalue < 0x0000100000000000LL)
192                 return 44;
193         if(highvalue < 0x0000200000000000LL)
194                 return 45;
195         if(highvalue < 0x0000400000000000LL)
196                 return 46;
197         if(highvalue < 0x0000800000000000LL)
198                 return 47;
199         if(highvalue < 0x0001000000000000LL)
200                 return 48;
201         if(highvalue < 0x0002000000000000LL)
202                 return 49;
203         if(highvalue < 0x0004000000000000LL)
204                 return 50;
205         if(highvalue < 0x0008000000000000LL)
206                 return 51;
207         if(highvalue < 0x0010000000000000LL)
208                 return 52;
209         if(highvalue < 0x0020000000000000LL)
210                 return 53;
211         if(highvalue < 0x0040000000000000LL)
212                 return 54;
213         if(highvalue < 0x0080000000000000LL)
214                 return 55;
215         if(highvalue < 0x0100000000000000LL)
216                 return 56;
217         if(highvalue < 0x0200000000000000LL)
218                 return 57;
219         if(highvalue < 0x0400000000000000LL)
220                 return 58;
221         if(highvalue < 0x0800000000000000LL)
222                 return 59;
223         if(highvalue < 0x1000000000000000LL)
224                 return 60;
225         if(highvalue < 0x2000000000000000LL)
226                 return 61;
227         if(highvalue < 0x4000000000000000LL)
228                 return 62;
229         if(highvalue < 0x8000000000000000LL)
230                 return 63;
231
232 #endif
233         return 64;
234
235 }
236
237 #ifdef _WIN32
238 void GetTableList(string sWildCharPathName, vector<string>& vPathName)
239 {
240         vPathName.clear();
241
242         string sPath;
243         int n = sWildCharPathName.find_last_of('\\');
244         if (n != -1)
245                 sPath = sWildCharPathName.substr(0, n + 1);
246
247         _finddata_t fd;
248         long handle = _findfirst(sWildCharPathName.c_str(), &fd);
249         if (handle != -1)
250         {
251                 do
252                 {
253                         string sName = fd.name;
254                         if (sName != "." && sName != ".." && !(fd.attrib & _A_SUBDIR))
255                         {
256                                 string sPathName = sPath + sName;
257                                 vPathName.push_back(sPathName);
258                         }
259                 } while (_findnext(handle, &fd) == 0);
260
261                 _findclose(handle);
262         }
263 }
264 #else
265 void GetTableList(int argc, char* argv[], vector<string>& vPathName)
266 {
267         vPathName.clear();
268
269         int i;
270         for (i = 1; i < argc; i++)
271         {
272                 string sPathName = argv[i];
273                 struct stat buf;
274                 if (lstat(sPathName.c_str(), &buf) == 0)
275                 {
276                         if (S_ISREG(buf.st_mode))
277                                 vPathName.push_back(sPathName);
278
279                 }
280         }
281 }
282 #endif
283
284
285 void ConvertRainbowTable(string sPathName, string sResultFileName, unsigned int rti_startptlength, unsigned int rti_endptlength, int showDistribution, int hascp, int rti_cplength, vector<unsigned int> rti_cppos)
286 {
287 #ifdef _WIN32
288         int nIndex = sPathName.find_last_of('\\');
289 #else
290         int nIndex = sPathName.find_last_of('/');
291 #endif
292         string sFileName;
293         if (nIndex != -1)
294                 sFileName = sPathName.substr(nIndex + 1);
295         else
296                 sFileName = sPathName;
297         // Info
298         printf("%s:\n", sFileName.c_str());
299         FILE* file = fopen(sPathName.c_str(), "rb");
300         FILE* fileR = fopen(sResultFileName.c_str(), "wb");
301         unsigned int distribution[64] = {0};
302         unsigned int numProcessedChains = 0;
303         
304         if (file != NULL && fileR != NULL)
305         {
306                 // File length check
307                 UINT4 nFileLen = GetFileLen(file);
308                 UINT4 nTotalChainCount = 0;
309                 if(hascp == 0) nTotalChainCount = nFileLen / 16;
310                 else nTotalChainCount = nFileLen / 18;
311                 if ((hascp == 0 && nFileLen % 16 != 0) || (hascp == 1 && nFileLen % 18 != 0))
312                 {
313                         printf("file length mismatch\n");
314                 }
315                 else
316                 {
317                         static CMemoryPool mp;
318                         unsigned int nAllocatedSize;
319                         RainbowChainCP* pChain = (RainbowChainCP*)mp.Allocate(nFileLen, nAllocatedSize);
320                         
321                         unsigned int chainrowsize = ceil((float)(rti_startptlength + rti_endptlength + rti_cplength) / 8) * 8 ; // The size in bits (in whole bytes)
322                         unsigned int chainrowsizebytes = chainrowsize / 8;
323
324
325                         if (pChain != NULL)
326                         {
327                                 nAllocatedSize = nAllocatedSize / sizeof(RainbowChainCP) * sizeof(RainbowChainCP);
328                                 fseek(file, 0, SEEK_SET);
329                                 uint64 curPrefix = 0, prefixStart = 0;
330                                 vector<IndexRow> indexes;
331                                 UINT4 nRainbowChainCountRead = 0;
332                                 while (true)    // Chunk read loop
333                                 {
334 /*                                      if (ftell(file) == nFileLen)
335                                                 break;*/
336                                         UINT4 nReadThisRound;
337                                         memset(pChain, 0x00, nAllocatedSize);
338                                         printf("reading...\n");
339                                         clock_t t1 = clock();
340                                         for(nReadThisRound = 0; nReadThisRound < nAllocatedSize / sizeof(RainbowChainCP) && nRainbowChainCountRead < nTotalChainCount; nReadThisRound++)
341                                         {                                               
342                                                 if(fread(&pChain[nReadThisRound], 16, 1, file) != 1) 
343                                                 { 
344                                                         printf("Error reading file\n"); exit(1);
345                                                 }
346                                                 if(hascp == 1)
347                                                 {
348                                                         if(fread(&pChain[nReadThisRound].nCheckPoint, 2, 1, file) != 1) 
349                                                         { 
350                                                                 printf("Error reading file\n"); exit(2);
351                                                         }
352                                                 }
353                                                 nRainbowChainCountRead++;
354                                         }
355                                         clock_t t2 = clock();
356                                         float fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
357                                         int nDataRead = nRainbowChainCountRead * 16;
358                                         if(hascp == 1) nDataRead += nRainbowChainCountRead * 2; // Add the index readings too
359                                         printf("%u bytes read, disk access time: %.2f s\n", nDataRead , fTime);
360                                         t1 = clock();
361
362                                         for(UINT4 i = 0; i < nReadThisRound; i++)
363                                         {
364                                                 if(showDistribution == 1)
365                                                 {
366                                                         distribution[GetMaxBits(pChain[i].nIndexS)-1]++;
367                                                 }
368                                                 else
369                                                 {
370                                                         uint64 chainrow = pChain[i].nIndexS; // Insert the complete start point                                                          
371                                                         chainrow |= ((uint64)pChain[i].nIndexE & (0xffffffff >> (32 - rti_endptlength))) << rti_startptlength; // 
372                                                         if(hascp == 1 && rti_cplength > 0) 
373                                                         {
374                                                                 chainrow |= (uint64)pChain[i].nCheckPoint << rti_startptlength + rti_endptlength;
375                                                         }
376                                                         fwrite(&chainrow, 1, chainrowsizebytes, fileR);                 
377                                                         uint64 prefix = pChain[i].nIndexE >> rti_endptlength;
378                                                         if(i == 0) curPrefix = prefix;
379                                                         if(prefix != curPrefix && numProcessedChains - prefixStart > 0)
380                                                         {
381                                                                         if(prefix < curPrefix)
382                                                                         {
383                                                                                 printf("**** Error writeChain(): Prefix is smaller than previous prefix. %llu < %llu****\n", prefix, curPrefix);
384                                                                                 exit(1);                                                                        
385                                                                         }
386                                                                         //unsigned char index[11] = {0}; // [0 - 10]
387                                                                         unsigned int numchains = numProcessedChains - prefixStart;
388                                                                         IndexRow index;
389                                                                         index.prefix = curPrefix;
390 //                                                                              index.prefixstart = prefixStart;
391                                                                         index.numchains = numchains;
392                                                                         indexes.push_back(index);
393                                                                         prefixStart = numProcessedChains;
394                                                                         curPrefix = prefix; 
395                                                         }
396                                                 }
397                                                 numProcessedChains++;
398                                         }               
399                                         t2 = clock();
400                                         fTime = 1.0f * (t2 - t1) / CLOCKS_PER_SEC;
401                                         printf("conversion time: %.2f s\n", fTime);             
402                                         if(nRainbowChainCountRead == nTotalChainCount)
403                                                 break;
404                                         if(showDistribution == 1)
405                                         {
406                                                 for(int i = 0; i < 64; i++)
407                                                 {
408                                                         printf("%u - %u\n", (i+1), distribution[i]);
409                                                 }
410                                                 return;
411                                         }
412
413                                 }
414         
415                 
416         
417                                 // We need to write the last index down
418                                 IndexRow index;
419                                 index.prefix = curPrefix;
420                                 index.prefixstart = prefixStart;
421                                 index.numchains = numProcessedChains - prefixStart;
422                                 indexes.push_back(index);
423
424                                 IndexRow high = {0}; // Used to find the highest numbers. This tells us how much we can pack the index bits
425                                 for(UINT4 i = 0; i < indexes.size(); i++)
426                                 {
427                                         if(indexes[i].numchains > high.numchains)
428                                                 high.numchains = indexes[i].numchains;
429 /*                                              if(indexes[i].prefixstart > high.prefixstart)
430                                                 high.prefixstart = indexes[i].prefixstart;
431                                         if(indexes[i].prefix > high.prefix)
432                                                 high.prefix = indexes[i].prefix;
433 */
434                                 }
435                                 high.prefix = indexes[indexes.size()-1].prefix; // The last prefix is always the highest prefix
436 //                                      unsigned int m_rti_index_prefixlength = GetMaxBits(high.prefix);
437                                 unsigned int m_rti_index_numchainslength = GetMaxBits(high.numchains);
438 //                                      unsigned int m_rti_index_indexlength = GetMaxBits(high.prefixstart);
439                                 unsigned int m_indexrowsize = ceil((float)(/*m_rti_index_indexlength + */m_rti_index_numchainslength) / 8) * 8; // The size in bits (in whole bytes)    
440                                 unsigned int m_indexrowsizebytes = m_indexrowsize / 8;
441                                 FILE *pFileIndex = fopen(sResultFileName.append(".index").c_str(), "wb");
442                                 fwrite("RTI2", 1, 4, pFileIndex);
443                                 fwrite(&rti_startptlength, 1, 1, pFileIndex);
444                                 fwrite(&rti_endptlength, 1, 1, pFileIndex);
445                                 fwrite(&rti_cplength, 1, 1, pFileIndex);
446 //                                      fwrite(&m_rti_index_indexlength , 1, 1, pFileIndex);
447                                 fwrite(&m_rti_index_numchainslength, 1, 1, pFileIndex);
448                                 for(UINT4 i = 0; i < rti_cppos.size(); i++)
449                                 {
450                                         fwrite(&rti_cppos[i], 1, 4, pFileIndex); // The position of the checkpoints
451                                 }
452 //                                      fwrite(&m_rti_index_prefixlength, 1, 1, pFileIndex);
453                                 int zero = 0;
454                                 fwrite(&indexes[0].prefix, 1, 8, pFileIndex); // Write the first prefix
455                                 unsigned int lastPrefix = 0;
456                                 for(UINT4 i = 0; i < indexes.size(); i++)
457                                 {
458                                         if(i == 0)
459                                                 lastPrefix = indexes[0].prefix;
460                                         // Checks how big a distance there is between the current and the next prefix. eg cur is 3 and next is 10 = 7.
461                                         unsigned int diffSize = indexes[i].prefix - lastPrefix; 
462                                         if(i > 0 && diffSize > 1)
463                                         {
464                                                 //indexrow |= indexes[i].prefixstart;
465                                                 //printf("Diffsize is %u\n", diffSize);
466
467                                                 // then write the distance amount of 00's
468                                                 if(diffSize > 1000) {
469                                                         printf("WARNING! The distance to the next prefix is %i. Do you want to continue writing %i bytes of 0x00? Press y to continue", diffSize, diffSize);
470                                                         #ifdef _WIN32
471                                                         if ( _getch() != 'y' ) {
472                                                         #else
473                                                         if ( tty_getchar() != 'y' ) {
474                                                         #endif
475                                                                 printf("Aborting...");
476                                                                 exit(1);
477                                                         }
478                                                 }
479                                                 for(UINT4 j = 1; j < diffSize; j++)
480                                                 {                                                               
481                                                         fwrite(&zero, 1, m_indexrowsizebytes, pFileIndex);
482                                                 }
483                                         }                                       
484                                         fwrite(&indexes[i].numchains, 1, m_indexrowsizebytes, pFileIndex);
485                                         lastPrefix = indexes[i].prefix;
486                                 }
487                                 fclose(pFileIndex);
488                         }
489                         else printf("memory allocation fail\n");
490
491                         
492                                         // Already finished?
493
494                 }
495                 fclose(file);
496         }
497         else
498                 printf("can't open file\n");
499
500 }
501
502 int main(int argc, char* argv[])
503 {
504         int argi = 1, i, argsUsed = 0;
505         unsigned int sptl = 40, eptl = 16;
506         int showDistribution = 0;
507         int usecp = 0;// How many bits to use from the index
508         int hascp = 0; 
509         vector<unsigned int> cppositions;
510         if (argc == 1)
511         {
512                 Usage();                
513                 return 0;
514         }
515         else if(argc > 2)
516         {
517                 for (; argi < argc; argi++)
518                 {
519                         if (strcmp(argv[argi], "-d") == 0 && (argsUsed & 0x8) == 0)
520                         {
521                                 // Enable verbose mode
522                                 argsUsed |= 0x8;                                
523                                 showDistribution = 1;
524                         }                       
525                         else if (strncmp(argv[argi], "-sptl=", 6) == 0 && (argsUsed & 0x1) == 0)
526                         {
527                                 // Maximum index for starting point
528                                 argsUsed |= 0x1;
529                                 sptl = 0;
530                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
531                                 {
532                                         sptl *= 10;
533                                         sptl += ((int) argv[argi][i]) - 0x30;
534                                 }
535                                 if (argv[argi][i] != '\0')
536                                 {
537                                         printf("Error: Invalid number.\n\n");
538                                         Usage();
539                                         return 1;
540                                 }
541                                 if (i > 23) // i - 3 > 20
542                                 {
543                                         printf("Error: Number is too large.\n\n");
544                                         Usage();
545                                         return 1;
546                                 }                       
547                         }
548
549                         else if (strncmp(argv[argi], "-eptl=", 6) == 0 && (argsUsed & 0x2) == 0)
550                         {
551                                 // Maximum index for ending points
552                                 argsUsed |= 0x2;
553                                 eptl = 0;
554                                 for (i = 6; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
555                                 {
556                                         eptl *= 10;
557                                         eptl += ((int) argv[argi][i]) - 0x30;
558                                 }
559                                 if (argv[argi][i] != '\0')
560                                 {
561                                         printf("Error: Invalid number.\n\n");
562                                         Usage();
563                                         return 1;
564                                 }
565                                 if (i > 23) // i - 3 > 20
566                                 {
567                                         printf("Error: Number is too large.\n\n");
568                                         Usage();
569                                         return 1;
570                                 }                       
571                         }
572                         else if(strncmp(argv[argi], "-usecp=", 7) == 0 && (argsUsed & 0x4) == 0)
573                         {
574                                 argsUsed |= 0x4;
575                                 hascp = 1;
576                                 usecp = 0;
577                                 unsigned int cppos = 0;
578                                 for(i = 7; argv[argi][i] != ' ' && argv[argi][i] != '\n' && argv[argi][i] != 0;)
579                                 {
580                                         if(cppositions.size() > 0) i++;
581                                         for (; argv[argi][i] >= '0' && argv[argi][i] <= '9'; i++)
582                                         {
583                                                 cppos *= 10;
584                                                 cppos += ((int) argv[argi][i]) - 0x30;
585                                         }
586 /*                                      if(argv[argi][i] == ',')
587                                         {*/
588                                                 cppositions.push_back(cppos);
589                                                 usecp++;
590                                                 cppos = 0;
591                                         //}
592                                 }
593                                 if (argv[argi][i] != '\0')
594                                 {
595                                         printf("Error: Invalid number.\n\n");
596                                         Usage();
597                                         return 1;
598                                 }
599                                 if (usecp > 16) // i - 3 > 20
600                                 {
601                                         printf("Error: Number is too large.\n\n");
602                                         Usage();
603                                         return 1;
604                                 }                               
605                                 else printf("Using %i bits of the checkpoints\n", usecp);
606                         }
607
608                 }               
609         }
610         vector<string> vPathName;
611 #ifdef WIN32
612         string sWildCharPathName = argv[1];
613         GetTableList(sWildCharPathName, vPathName);
614 #else
615         GetTableList(argc, argv, vPathName);
616 #endif
617         if (vPathName.size() == 0)
618         {
619                 printf("no rainbow table found\n");
620                 return 0;
621         }
622         for (UINT4 i = 0; i < vPathName.size(); i++)
623         {
624                 string sResultFile;
625                 int n = vPathName[i].find_last_of('\\');
626                 if (n != -1)
627                         sResultFile = vPathName[i].substr(n+1, vPathName[i].length()) + "i2";
628                 else 
629                         sResultFile = vPathName[i] + "i2"; // Resulting file is .rt, not .rti
630                 printf("Using %i of 64 bits. sptl: %i, eptl: %i, cp: %i. Chains will be %i bytes in size\n", (sptl + eptl + usecp), sptl, eptl, usecp, ((sptl + eptl + usecp) / 8));
631                 if(sptl + eptl + usecp > 64)
632                 {
633                         exit(1);
634                 }
635                 ConvertRainbowTable(vPathName[i], sResultFile, sptl, eptl, showDistribution, hascp, usecp, cppositions);
636                 printf("\n");
637         }
638         return 0;
639 }