windows-tests-torture-20090131
[openafs.git] / src / WINNT / tests / torture / Source / WinTorture.c
1 //#define _WIN32_WINNT 0x0500
2 //#include <windows.h>
3
4 #include "includes.h"
5 #include <Psapi.h>
6 #include "common.h"
7 #ifdef HAVE_HESOID
8     #include "ResolveLocker.h"
9     extern int ResolveLocker(USER_OPTIONS *attachOption);
10     extern void GetLockerInfo(char *Locker, char *Path);
11 #endif /* HAVE_HESOID */
12
13 #define MAX_THREADS 100
14
15 int opterr = 1;
16 int optind = 1;
17 int optopt;
18 int optreset;
19 char *optarg;
20
21 extern void LogStats(char *FileName, int ToLog, int Iteration,  int NumberOfProcesses, int NumberOfThreads, 
22                      char *HostName, int ProcessNumber, struct cmd_struct CommandInfo[],
23                      char *CommandLine, char *TargetDirectory);
24 extern int UpdateMasterLog(char *FileName, struct cmd_struct CommandInfo[]);
25 extern int BuildMasterStatLog(char *FileName, char *MoveFileName, int NumberOfProcesses, 
26                               int NumberOfThreads, char *CommandLine, int LoopCount, 
27                               char *TargetDirectory, int ProcessNumber);
28 extern void LogMessage(int ProcessNumber, char *HostName, char *FileName, char *message, int LogID);
29
30 DWORD WINAPI StressTestThread(LPVOID lpThreadParameter);
31 int   getopt(int, char**, char*);
32 DWORD FindProcessCount(char *ProcessName, HANDLE JobHandle);
33 void  show_results(char *CommandLine, char *TargetDirectory, struct cmd_struct CommandInfo[], 
34                    char *HostName, int NumberOfThreads, int CurrentLoop, int LogID);
35
36 char    *ClientText = "streamfiles.txt";
37 char    PathToSecondDir[256];
38 int     ThreadStatus[MAX_HANDLES];
39 int     verbose;
40 static int     BufferSize = 256*1024;
41 int     UseLocker = 0;
42 int     EndOnError;
43 int     AfsTrace;
44 int     ChronLog;
45 BOOL    PrintStats;
46 HANDLE  MutexHandle;
47 HANDLE  FileMutexHandle;
48 HANDLE  ChronMutexHandle;
49 HANDLE  OSMutexHandle;
50 HANDLE  ShutDownEventHandle;
51 HANDLE  PauseEventHandle;
52 HANDLE  ContinueEventHandle;
53 EXIT_STATUS ExitStatus[MAX_HANDLES];
54
55 double create_procs(char *Hostname, char *CommandLine, char *TargetDirectory, 
56                     char *AfsLocker, char *Locker, char *HostName, 
57                     int NumberOfThreads, int CurrentLoop, int LogID)
58 {
59     int     i;
60     int     status;
61     int     count;
62     int     ProcessID;
63     char    EventName[512];
64     HANDLE  hEventHandle[MAX_HANDLES];
65     HANDLE  hThreadHandle[MAX_HANDLES];
66     DWORD   dwThreadID[MAX_HANDLES];
67     struct cmd_struct *CommandInfo;
68     CRITICAL_SECTION CriticalSection;
69     PARAMETERLIST    *pParameterList[MAX_HANDLES];
70 #ifdef HAVE_HESOID
71     USER_OPTIONS     attachOption;
72 #endif
73
74     InitializeCriticalSection(&CriticalSection); 
75     for (i = 0; i < MAX_HANDLES; i++)
76     {
77         hEventHandle[i] = NULL;
78         hThreadHandle[i] = NULL;
79         pParameterList[i] = NULL;
80         dwThreadID[i] = 0;
81     }
82
83     count = 0;
84     CommandInfo = calloc(1, sizeof(struct cmd_struct) * (CMD_MAX_CMD + 1) * NumberOfThreads);
85     ProcessID = getpid();
86
87     for (i = 0; i < NumberOfThreads; i++) 
88     {
89         if (EndOnError)
90         {
91             if (ThreadStatus[count] == 0)
92                 continue;
93         }
94         sprintf(EventName, "%d%sEvent%05d", ProcessID, HostName, count);
95         hEventHandle[count] = CreateEvent(NULL, FALSE, FALSE, EventName);
96         if (hEventHandle[count] == NULL)
97             continue;
98         ResetEvent(hEventHandle[count]);
99         
100         pParameterList[count] = calloc(1, sizeof(PARAMETERLIST));
101         pParameterList[count]->ProcessNumber = count;
102         pParameterList[count]->CommandInfo = (struct cmd_struct *)(CommandInfo + (i * (CMD_MAX_CMD + 1)));
103         pParameterList[count]->BufferSize = BufferSize;
104         pParameterList[count]->PrintStats = PrintStats;
105         pParameterList[count]->CurrentLoop = CurrentLoop;
106         pParameterList[count]->AfsTrace = AfsTrace;
107         pParameterList[count]->TargetDirectory = TargetDirectory;
108         pParameterList[count]->pExitStatus = &ExitStatus[i];
109         pParameterList[count]->pThreadStatus = &ThreadStatus[i];
110         pParameterList[count]->CommandLine = CommandLine;
111         pParameterList[count]->ClientText = ClientText;
112         pParameterList[count]->PathToSecondDir = PathToSecondDir;
113         pParameterList[count]->AfsLocker = AfsLocker;
114         pParameterList[count]->HostName = HostName;
115         pParameterList[count]->ProcessID = ProcessID;;
116         pParameterList[count]->LogID = LogID;;
117
118         ThreadStatus[count] = 0;
119         hThreadHandle[count] = CreateThread(NULL, 0, &StressTestThread, (LPVOID)pParameterList[count], CREATE_SUSPENDED, &dwThreadID[count]);
120         if (hThreadHandle[count] != NULL)
121         {
122             ResumeThread(hThreadHandle[count]);
123             ThreadStatus[count] = 1;
124             ++count;
125         }
126         else
127         {
128             CloseHandle(hEventHandle[count]);
129             if (pParameterList[count] != NULL)
130                 free(pParameterList[count]);
131             pParameterList[count] = NULL;
132             hThreadHandle[count] = NULL;
133             hEventHandle[count] = NULL;
134         }
135     }
136
137     count = 0;
138     for (i = 0; i < MAX_HANDLES; i++)
139     {
140         if (hEventHandle[i] != NULL)
141             ++count;
142     }
143 #ifdef HAVE_HESOID
144     if (UseLocker)
145     {
146         int          rc;
147
148         memset(&attachOption, '\0', sizeof(attachOption));
149         strcpy(attachOption.Locker, Locker);
150         strcpy(attachOption.type, "locker");
151         if (rc = ResolveLocker(&attachOption))
152         {
153             if (!stricmp(attachOption.type, "AFS"))
154             {
155                 printf("Unable to attach locker %s - AFS is not supported\n", Locker);
156                 exit(1);
157             }
158             strcpy(AfsLocker, attachOption.SubMount);
159             memset(&attachOption, '\0', sizeof(attachOption));
160             strcpy(attachOption.Locker, Locker);
161             strcpy(attachOption.type, "locker");
162             if (rc = attach(attachOption, 0, 0, Locker))
163             {
164                 printf("Unable to attach locker %s\n", Locker);
165                 exit(1);
166             }
167         }
168     }
169 #endif /* HAVE_HESOID */
170
171     status = WaitForMultipleObjects(count, hEventHandle, TRUE, INFINITE);
172     for (i = 0; i < MAX_HANDLES; i++)
173     {
174         if (hEventHandle[i] != NULL)
175             CloseHandle(hEventHandle[i]);
176         if (pParameterList[i] != NULL)
177             free(pParameterList[i]);
178         pParameterList[i] = NULL;
179         hEventHandle[i] = NULL;
180     }
181
182     for (i = 0; i < NumberOfThreads; i++)
183     {
184         char    FileName[128];
185         char    temp[512];
186
187         if (strlen(ExitStatus[i].Reason))
188         {
189             sprintf(temp, "Thread %0d exited with reason: %s", i, ExitStatus[i].Reason);
190         }
191         else
192         {
193             sprintf(temp, "Thread %0d completed\n", i);
194         }
195         if (verbose)
196             printf("%s", temp);
197         sprintf(FileName, "Thread_%05d.log", i);
198         LogMessage(i, HostName, FileName, temp, LogID);
199         sprintf(temp, "Ended Iteration %0d\n\n", CurrentLoop);
200         LogMessage(i, HostName, FileName, temp, LogID);
201         CloseHandle(hEventHandle[i]);
202     }
203     show_results(CommandLine, TargetDirectory, CommandInfo, HostName, NumberOfThreads, CurrentLoop, LogID);
204     free(CommandInfo);
205     DeleteCriticalSection(&CriticalSection);
206
207     return(0);
208 }
209
210 static void usage(void)
211 {
212
213     fprintf(stderr, "usage: wintorture [options]\n");
214     fprintf(stderr, "where options can be:\n");
215     fprintf(stderr, "\t-b        Create a chronological log.\n");
216     fprintf(stderr, "\t-c <txt>  Specifies the script txt file to use.\n");
217     fprintf(stderr, "\t-e        End thread processing on an error.\n");
218     fprintf(stderr, "\t-f <name> Target directory name.\n");
219     fprintf(stderr, "\t-i <num>  Number of iterations of the stress test to run.\n");
220     fprintf(stderr, "\t            This option will override the -m option.\n");
221 #ifdef HAVE_HESOID
222     fprintf(stderr, "\t-l <path> AFS locker or AFS submount in which to create the target directory.\n");
223 #endif /* HAVE_HESOID */
224     fprintf(stderr, "\t-m <num>  The number of minutes to run the stress test.\n");
225     fprintf(stderr, "\t            This option will override the -i option.\n");
226     fprintf(stderr, "\t-n <num>  The number of threads to run.\n");
227     fprintf(stderr, "\t-p <path> UNC path to second directory.\n");
228     fprintf(stderr, "\t-s        Output stats.\n");
229     fprintf(stderr, "\t-t        Do AFS trace logging.\n");
230     fprintf(stderr, "\t-u <UNC>  UNC path to target directory.\n");
231     fprintf(stderr, "\t-v        Turn on verbose mode.\n");
232     fprintf(stderr, "\nNOTE: The switches are not case sensitive.  You\n");
233     fprintf(stderr, "\n      may use either upper or lower case letters.\n\n");
234 }
235
236 void show_results(char *CommandLine, char *TargetDirectory, struct cmd_struct *CommandInfo, 
237                   char *HostName, int NumberOfThreads, int CurrentLoop, int LogID)
238 {
239         struct cmd_struct TotalCommandInfo[CMD_MAX_CMD + 1];
240         int         i;
241     int         j;
242         unsigned    grand_total = 0;
243     char        FileName[256];
244     char        WorkingDirectory[512];
245     struct cmd_struct *FinalCmdInfo;
246
247         for (j = 0; j <= CMD_MAX_CMD; j++) {
248                 TotalCommandInfo[j].count = 0;
249                 TotalCommandInfo[j].min_sec = 0;
250                 TotalCommandInfo[j].max_sec = 0;
251                 TotalCommandInfo[j].MilliSeconds = 0;
252                 TotalCommandInfo[j].total_sec = 0;
253                 TotalCommandInfo[j].total_sum_of_squares = 0;
254                 TotalCommandInfo[j].ErrorCount = 0;
255                 TotalCommandInfo[j].ErrorTime = 0;
256         }
257
258     memset(ExitStatus, '\0', sizeof(ExitStatus[0]) * MAX_HANDLES);
259
260         for (j = 0; j < NumberOfThreads; j++) 
261     { 
262             FinalCmdInfo = CommandInfo + (j * (CMD_MAX_CMD + 1));
263
264                 for (i = 0; i <= CMD_MAX_CMD; i++)
265         {
266                     TotalCommandInfo[i].count += FinalCmdInfo[i].count;
267             TotalCommandInfo[i].total_sec += FinalCmdInfo[i].total_sec; 
268             TotalCommandInfo[i].total_sum_of_squares += FinalCmdInfo[i].total_sum_of_squares; 
269             TotalCommandInfo[i].ErrorCount += FinalCmdInfo[i].ErrorCount; 
270             TotalCommandInfo[i].ErrorTime += FinalCmdInfo[i].ErrorTime; 
271             grand_total += FinalCmdInfo[j].total_sec;
272             if (!TotalCommandInfo[i].min_sec || (TotalCommandInfo[i].min_sec > FinalCmdInfo[i].min_sec))
273                 TotalCommandInfo[i].min_sec = FinalCmdInfo[i].min_sec;
274             if (TotalCommandInfo[i].max_sec < FinalCmdInfo[i].max_sec)
275                 TotalCommandInfo[i].max_sec = FinalCmdInfo[i].max_sec;
276         }
277         }
278
279
280     memset(WorkingDirectory, '\0', sizeof(WorkingDirectory));
281     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
282     sprintf(FileName, "%s\\log%05d\\%s\\ProcessStats.log", WorkingDirectory, LogID, HostName);
283
284     if (PrintStats)
285         LogStats(FileName, 0, CurrentLoop, 1, NumberOfThreads, HostName, -1, TotalCommandInfo, 
286                  CommandLine, TargetDirectory);
287     LogStats(FileName, 1, CurrentLoop, 1, NumberOfThreads, HostName, -1, TotalCommandInfo, 
288              CommandLine, TargetDirectory);
289
290     sprintf(FileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, "MasterStatLog.log");
291     UpdateMasterLog(FileName, TotalCommandInfo);
292     sprintf(FileName, "%s\\log%05d\\%s\\%s", WorkingDirectory, LogID, HostName, "MasterProcessStatLog.log");
293     UpdateMasterLog(FileName, TotalCommandInfo);
294 }
295
296
297 int main(int argc, char *argv[])
298 {
299     int         i;
300     int         LoopCount;
301     time_t      SecondsToRun;
302     int         NumberOfIterations;
303     int         StressTestUsed = 0;
304     int         rc;
305     int         opt;
306     int         ErrorState;
307     int         NumberOfProcesses;
308     int         NumberOfThreads;
309     int         IterationCount;
310     int         CurrentLoop = 0;
311     int         LogID;
312     time_t      TotalTime;
313     time_t      StartTime;
314     time_t      EndTime;
315     extern char *optarg;
316     extern int  optind;
317     FILE        *Mfp;
318     FILE        *fp;
319     char        *p;
320     char        tbuffer[10];
321     char        buffer[512];
322     char        FileName[256];
323     char        MoveFileName[256];
324     char        Locker[256];
325     char        command[512];
326     char        DateTime[512];
327     char        WorkingDirectory[512];
328     char        CommandLine[512];
329     char        TargetDirectory[512];
330     char        AfsLocker[256];
331     char        HostName[128];
332     char        JobName[128];
333     SYSTEMTIME  SystemTime;
334     SYSTEMTIME  LocalTime;
335     TIME_ZONE_INFORMATION TimeZoneInformation;
336     HANDLE      ExitMutexHandle;
337     HANDLE      JobHandle;
338
339
340     memset(HostName, '\0', sizeof(HostName));
341     memset(PathToSecondDir, '\0', sizeof(PathToSecondDir));
342     memset(Locker, '\0', sizeof(Locker));
343     SecondsToRun = 0;
344     NumberOfIterations = 0;
345     EndOnError = 0;
346     UseLocker = 0;
347     ChronLog = 0;
348     AfsTrace = 0;
349     verbose = 0;
350     NumberOfThreads = 1;
351     NumberOfProcesses = 1;
352     NumberOfThreads = 1;
353     LogID = 0;
354     PrintStats = FALSE;
355
356     while ((opt = getopt(argc, argv, "A:a:BbC:c:D:d:EeF:f:G:g:I:i:L:l:M:m:N:n:P:p:SsTtU:u:Vv")) != EOF)
357     {
358
359         switch (opt)
360         {
361             case 'a':
362             case 'A':
363                 break;
364             case 'b':
365             case 'B':
366                 ChronLog = 1;
367                 break;
368             case 'c':
369             case 'C':
370                 ClientText = optarg;
371                 break;
372             case 'd':
373             case 'D':
374                 StressTestUsed = 1;
375                 NumberOfProcesses = atoi(optarg);
376                 break;
377             case 'e':
378             case 'E':
379                 EndOnError = 1;
380                 break;
381             case 'f':
382             case 'F':
383                 strcpy(HostName, optarg);
384                 for (i = 0; i < (int)strlen(HostName); i++)
385                 {
386                     if ((HostName[i] == '\\') || (HostName[i] == '/'))
387                     {
388                         printf("\nInvalid -F usage...Subdirectories not allowed\n\n");
389                         usage();
390                         exit(1);
391                     }
392                 }
393                 break;
394             case 'g':
395             case 'G':
396                 StressTestUsed = 1;
397                 LogID = atoi(optarg);
398                 break;
399             case 'i':
400             case 'I':
401                 SecondsToRun = 0;
402                 NumberOfIterations = atoi(optarg);
403                 if (NumberOfIterations < 0)
404                     NumberOfIterations = 0;
405                 break;
406 #ifdef HAVE_HESOID
407             case 'l':
408             case 'L':
409                 strcpy(Locker, optarg);
410                 UseLocker = 1;
411                 break;
412 #endif /* HAVE_HESOID */
413             case 'm':
414             case 'M':
415                 NumberOfIterations = 0;
416                 SecondsToRun = atoi(optarg) * 60;
417                 if (SecondsToRun < 0)
418                     SecondsToRun = 0;
419                 break;
420             case 'n':
421             case 'N':
422                 NumberOfThreads = atoi(optarg);
423                 break;
424             case 'p':
425             case 'P':
426                 strcpy(PathToSecondDir, optarg);
427                 for(p = PathToSecondDir; *p; p++)
428                 {
429                     if(*p == '/')
430                         *p = '\\';
431                 }
432                 break;
433             case 's':
434             case 'S':
435                 PrintStats = TRUE;
436                 break;
437             case 't':
438             case 'T':
439                 AfsTrace = 1;
440                 break;
441             case 'u':
442             case 'U':
443                 UseLocker = 0;
444                 strcpy(Locker, optarg);
445                 break;
446             case 'v':
447             case 'V':
448                 verbose = 1;
449                 break;
450             default:
451                 usage();
452                 exit(1);
453                 break;
454         }
455     }
456
457     if (strlen(HostName) == 0)
458     {
459         printf("You must use the -f option to specify a target directory\n\n");
460         usage();
461         exit(1);
462     }
463     if (strlen(Locker) == 0)
464     {
465         printf("You must use either the -u or the -l option\n\n");
466         usage();
467         exit(1);
468     }
469
470     memset(CommandLine, '\0', sizeof(CommandLine));
471     for (i = 1; i < argc; i++)
472     {
473         if (StressTestUsed)
474         {
475             if (!stricmp(argv[i], "-f"))
476             {
477                 char temp[64];
478
479                 strcpy(temp, argv[i + 1]);
480                 temp[strlen(temp) - 5] = '\0';
481                 strcat(CommandLine, argv[i]);
482                 strcat(CommandLine, " ");
483                 strcat(CommandLine, temp);
484                 strcat(CommandLine, " ");
485                 ++i;
486                 continue;
487             }
488         }
489         strcat(CommandLine, argv[i]);
490         strcat(CommandLine, " ");
491     }
492
493     argc -= optind;
494     argv += optind;
495
496     if (strlen(Locker) == 0)
497     {
498         usage();
499         exit(1);
500     }
501
502
503     for(p = Locker; *p; p++)
504     {
505         if(*p == '/')
506             *p = '\\';
507     }
508  
509 #ifdef HAVE_HESOID
510     if (UseLocker)
511     {
512         char *sPtr;
513
514         sprintf(AfsLocker, "\\\\afs\\%s", Locker);
515         memset(buffer, '\0', sizeof(buffer));
516         GetLockerInfo(Locker, buffer);
517         if (strlen(buffer) != 0)
518         {
519             sPtr = strstr(buffer, "/afs/");
520             sPtr += strlen("/afs/");
521             strcpy(TargetDirectory, sPtr);
522             sPtr = strchr(TargetDirectory, ' ');
523             if (sPtr != NULL)
524                 (*sPtr) = '\0';
525             while ((sPtr = strchr(TargetDirectory, '/')) != NULL)
526                 (*sPtr) = '\\';
527         }
528         else
529         {
530             strcpy(TargetDirectory, Locker);
531         }
532     }
533     else
534     {
535 #endif /* HAVE_HESOID */
536         strcpy(AfsLocker, Locker);
537 #if 0
538         /* jaltman - this makes no sense.  why strip off the server name and leave
539          * an absolute path on the local disk?
540          */
541         if (!strnicmp(Locker, "\\\\afs\\", strlen("\\\\afs\\")))
542             strcpy(TargetDirectory, &Locker[strlen("\\\\afs\\")]);
543         else
544 #endif
545             strcpy(TargetDirectory, Locker);
546 #ifdef HAVE_HESOID
547     }
548 #endif /* HAVE_HESOID */
549
550
551     TotalTime = 0;
552     LoopCount = 0;
553     CurrentLoop = 0;
554
555     ExitMutexHandle = CreateMutex(NULL, FALSE, "AfsExitEvent");
556     ChronMutexHandle = CreateMutex(NULL, FALSE, "WinTortureChronMutex");
557     MutexHandle = CreateMutex(NULL, FALSE, "WinTortureMutex");
558     FileMutexHandle = CreateMutex(NULL, FALSE, "WinTortureFileMutex");
559     OSMutexHandle = CreateMutex(NULL, FALSE, "WinTortureOSMutex");
560
561     for (i = 0; i < MAX_THREADS; i++)
562     {
563         ThreadStatus[i] = 2;
564     }
565
566     sprintf(JobName, "%s%05d", "JOB", LogID);
567     JobHandle = CreateJobObject(NULL, JobName);
568     rc = AssignProcessToJobObject(JobHandle, GetCurrentProcess());
569
570     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
571     sprintf(FileName, "%s\\log%05d", WorkingDirectory, LogID);
572     CreateDirectory(FileName, NULL);
573     sprintf(FileName, "%s\\test", WorkingDirectory);
574     CreateDirectory(FileName, NULL);
575     if (!StressTestUsed)
576     {
577         sprintf(FileName, "%s\\log%05d\\Chron.log", WorkingDirectory, LogID);
578         DeleteFile(FileName);
579     }
580
581     sprintf(FileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, HostName);
582     sprintf(command, "rmdir /S /Q %s > %s\\test\\test", FileName, WorkingDirectory);
583     system(command);
584
585     ShutDownEventHandle = CreateEvent(NULL, TRUE, FALSE, "AfsShutdownEvent");
586     PauseEventHandle = CreateEvent(NULL, TRUE, FALSE, "AfsPauseEvent");
587     ContinueEventHandle = CreateEvent(NULL, TRUE, FALSE, "AfsContinueEvent");
588
589     IterationCount = 0;
590     time(&StartTime);
591     if ((NumberOfIterations == 0) && (SecondsToRun == 0))
592         NumberOfIterations = 1;
593     else if (SecondsToRun != 0)
594     {
595         SecondsToRun += StartTime;
596     }
597
598     while (1)
599     {
600         if (SecondsToRun != 0)
601         {
602             time(&StartTime);
603             if (StartTime > SecondsToRun)
604             {
605                 break;
606             }
607         }
608         if (NumberOfIterations != 0)
609         {
610             if (LoopCount >= NumberOfIterations)
611             {
612                 break;
613             }
614         }
615         if (rc = WaitForSingleObject(ShutDownEventHandle, 0) == WAIT_OBJECT_0)
616         {
617             break;
618         }
619         ++LoopCount;
620         CurrentLoop = LoopCount;
621
622         _strtime(tbuffer);
623         printf("\nIteration %d started at: %s\n", LoopCount, tbuffer);
624         create_procs(HostName, CommandLine, TargetDirectory, AfsLocker, Locker, 
625                      HostName, NumberOfThreads, CurrentLoop, LogID);
626         _strtime(tbuffer);
627         printf("Iteration %d ended at: %s\n", LoopCount, tbuffer);
628         time(&EndTime);
629         printf("Iteration %d lapse time: %ld seconds\n", LoopCount, EndTime - StartTime);
630         TotalTime += EndTime - StartTime;
631         sprintf(FileName, "%s\\log%05d\\IterationCount", WorkingDirectory, LogID);
632         WaitForSingleObject(MutexHandle, 20 * 1000);
633         if ((fp = fopen(FileName, "r")) != NULL)
634         {
635             fgets(buffer, sizeof(buffer), fp);
636             IterationCount = atoi(buffer);
637             fclose(fp);
638         }
639         ++IterationCount;
640         fp = fopen(FileName, "w");
641         fprintf(fp, "%d\n", IterationCount);
642         fclose(fp);
643         ReleaseMutex(MutexHandle);
644         if (EndOnError)
645         {
646             for (i = 0; i < MAX_THREADS; i++)
647             {
648                 if (ThreadStatus[i] == 1)
649                 {
650                     break;
651                 }
652             }
653             if (i >= MAX_THREADS)
654                 break;
655         }
656         ErrorState = 0;
657         for (i = 0; i < MAX_THREADS; i++)
658         {
659             if (ThreadStatus[i] == 0)
660             {
661                 ErrorState = 1;
662                 ThreadStatus[i] = 1;
663             }
664         }
665         if (ErrorState)
666         {
667             printf("\nSleeping for 3 minutes for error recovery\n\n");
668             Sleep(3 * 60 * 1000);
669         }
670     }
671     if (LoopCount == 0)
672         return(0);
673
674     sprintf(FileName, ".\\log%05d\\%s\\Master.log", LogID, HostName);
675     Mfp = fopen(FileName, "w+");
676     fprintf(Mfp, "Average Iteration Time = %.02f minutes\n\n", ((float)TotalTime/(float)(LoopCount))/60.0);
677     for (i = 0; i < NumberOfThreads; i++)
678     {
679         sprintf(FileName, ".\\log%05d\\%s\\Thread_%05d.log", LogID, HostName, i);
680         fp = fopen(FileName, "r");
681         if (fp != NULL)
682         {
683             fprintf(Mfp, "START OF THREAD %d\n\n", i);
684             while (fgets(buffer, 512, fp) != NULL)
685             {
686                 fprintf(Mfp, "%s", buffer);
687             }
688             fclose(fp);
689             fprintf(Mfp, "END OF THREAD %d\n\n", i);
690         }
691     }
692     fclose(Mfp);
693
694     memset(WorkingDirectory, '\0', sizeof(WorkingDirectory));
695     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
696
697     sprintf(FileName, "%s\\log%05d\\%s\\%s", WorkingDirectory, LogID, HostName, "MasterProcessStatLog.log");
698     sprintf(MoveFileName, "%s\\log%05d\\%s\\%s", WorkingDirectory, LogID, HostName, "MasterProcessStatLogRaw.log");
699     BuildMasterStatLog(FileName, MoveFileName, NumberOfProcesses, NumberOfThreads, CommandLine, LoopCount, 
700                        TargetDirectory, -1);
701
702 //    sprintf(DateTime, "%s-%04d%02d%02d-%02d%02d%02d", HostName,
703 //                                                      LocalTime.wYear, 
704 //                                                      LocalTime.wMonth, 
705 //                                                      LocalTime.wDay,
706 //                                                      LocalTime.wHour,
707 //                                                      LocalTime.wMinute,
708 //                                                      LocalTime.wSecond);
709 //    sprintf(command, "rename %s\\log%05d\\%s %s", WorkingDirectory, LogID, HostName, DateTime);
710 //    rc = system(command);
711
712     WaitForSingleObject(ExitMutexHandle, 20 * 1000);
713     Sleep(3 * 1000);
714
715     GetSystemTime(&SystemTime);
716     GetTimeZoneInformation(&TimeZoneInformation);
717     SystemTimeToTzSpecificLocalTime(&TimeZoneInformation, &SystemTime, &LocalTime);
718
719     NumberOfProcesses = 0;
720     sprintf(FileName, "%s\\log%05d\\ProcessCount", WorkingDirectory, LogID);
721     if ((fp = fopen(FileName, "r")) != NULL)
722     {
723         fgets(buffer, sizeof(buffer), fp);
724         NumberOfProcesses = atoi(buffer);
725         fclose(fp);
726     }
727     ++NumberOfProcesses;
728     fp = fopen(FileName, "w");
729     fprintf(fp, "%d\n", NumberOfProcesses);
730     fclose(fp);
731
732     if (FindProcessCount("wintorture.exe", JobHandle) == 1)
733     {
734         NumberOfProcesses = 0;
735         sprintf(FileName, "%s\\log%05d\\ProcessCount", WorkingDirectory, LogID);
736         sprintf(MoveFileName, "%s\\log%05d\\ProcessCountRaw", WorkingDirectory, LogID);
737         if ((fp = fopen(FileName, "r")) != NULL)
738         {
739             fgets(buffer, sizeof(buffer), fp);
740             NumberOfProcesses = atoi(buffer);
741             fclose(fp);
742             MoveFile(FileName, MoveFileName);
743         }
744
745         IterationCount = 0;
746         sprintf(FileName, "%s\\log%05d\\IterationCount", WorkingDirectory, LogID);
747         sprintf(MoveFileName, "%s\\log%05d\\IterationCountRaw", WorkingDirectory, LogID);
748         if ((fp = fopen(FileName, "r")) != NULL)
749         {
750             fgets(buffer, sizeof(buffer), fp);
751             IterationCount = atoi(buffer);
752             fclose(fp);
753             MoveFile(FileName, MoveFileName);
754         }
755
756         sprintf(FileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, "MasterStatLog.log");
757         sprintf(MoveFileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, "MasterStatLogRaw.log");
758         BuildMasterStatLog(FileName, MoveFileName, NumberOfProcesses, NumberOfThreads, 
759                            CommandLine, IterationCount, TargetDirectory, -2);
760         sprintf(DateTime, "%s%05d-%04d%02d%02d-%02d%02d%02d", "log",
761                                                           LogID,
762                                                           LocalTime.wYear, 
763                                                           LocalTime.wMonth, 
764                                                           LocalTime.wDay,
765                                                           LocalTime.wHour,
766                                                           LocalTime.wMinute,
767                                                           LocalTime.wSecond);
768         sprintf(command, "rename %s\\log%05d %s", WorkingDirectory, LogID, DateTime);
769         rc = system(command);
770         ResetEvent(ShutDownEventHandle);
771         ResetEvent(PauseEventHandle);
772         ResetEvent(ContinueEventHandle);
773         CloseHandle(ShutDownEventHandle);
774         CloseHandle(PauseEventHandle);
775         CloseHandle(ContinueEventHandle);
776
777     }
778
779     ReleaseMutex(ExitMutexHandle);
780     CloseHandle(JobHandle);
781     return 0;
782 }
783
784 int ProcessNameAndID(DWORD processID, char *ProcessName, HANDLE JobHandle)
785 {
786     char    szProcessName[1024] = "unknown";
787     char    FileName[1024];
788     char    WorkingDirectory[512];
789     HANDLE  hProcess;
790     int     Count;
791
792     memset(WorkingDirectory, '\0', sizeof(WorkingDirectory));
793     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
794     strcat(WorkingDirectory, "\\");
795     Count = 0;
796     hProcess = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID);
797     if (hProcess)
798     {
799         HMODULE hMod;
800         DWORD cbNeeded;
801
802         if (EnumProcessModules(hProcess, &hMod, sizeof(hMod), &cbNeeded))
803         {
804             memset(szProcessName, '\0', sizeof(szProcessName));
805             GetModuleBaseName(hProcess, hMod, szProcessName, sizeof(szProcessName));
806             if (!stricmp(szProcessName, ProcessName))
807             {
808                 memset(FileName, '\0', sizeof(FileName));
809                 if (GetModuleFileNameEx(hProcess, hMod, FileName, sizeof(FileName) - 1))
810                 {
811                     if (!strnicmp(WorkingDirectory, FileName, strlen(WorkingDirectory)))
812                     {
813                         ++Count;
814                     }
815                 }
816             }
817         }
818     }
819     CloseHandle(hProcess);
820     return(Count);
821 }
822
823 DWORD FindProcessCount(char *ProcessName, HANDLE JobHandle)
824 {
825     DWORD       aProcesses[8092];
826     DWORD       cbNeeded;
827     DWORD       cProcesses;
828     int         Count;
829     int         rc;
830     unsigned    int i;
831     JOBOBJECT_BASIC_PROCESS_ID_LIST IdList;
832
833     rc = QueryInformationJobObject(JobHandle, JobObjectBasicProcessIdList, &IdList, sizeof(IdList), NULL);
834     return(IdList.NumberOfAssignedProcesses);
835
836     if (!EnumProcesses(aProcesses, sizeof(aProcesses), &cbNeeded))
837         return(1);
838
839     Count = 0;
840     cProcesses = cbNeeded / sizeof(DWORD);
841
842     for (i = 0; i < cProcesses; i++)
843     {
844
845 //        Count += ProcessNameAndID(aProcesses[i], ProcessName, JobHandle);
846     }
847
848     return(Count);
849 }
850
851 char *_progname(char *nargv0)
852 {
853     char    *tmp;
854
855     tmp = strrchr(nargv0, '/');
856     if (tmp)
857         tmp++;
858     else
859         tmp = nargv0;
860     return(tmp);
861 }
862
863 #define BADCH   (int)'?'
864 #define BADARG  (int)':'
865 #define EMSG    ""
866
867 int getopt(int nargc, char *nargv[], char *ostr)
868 {
869     static char *__progname = 0;
870     static char *place = EMSG;          /* option letter processing */
871     char *oli;                          /* option letter list index */
872
873     __progname = __progname?__progname:_progname(*nargv);
874
875     if (optreset || !*place)
876     {
877         optreset = 0;
878         if (optind >= nargc || *(place = nargv[optind]) != '-')
879         {
880             place = EMSG;
881             return (-1);
882         }
883         if (place[1] && *++place == '-' && place[1] == '\0')
884         {
885             ++optind;
886             place = EMSG;
887             return (-1);
888         }
889     }
890     if ((optopt = (int)*place++) == (int)':' || !(oli = strchr(ostr, optopt))) 
891     {
892         if (optopt == (int)'-')
893             return (-1);
894         if (!*place)
895             ++optind;
896         if (opterr && *ostr != ':')
897             (void)fprintf(stderr, "%s: illegal option -- %c\n", __progname, optopt);
898         return (BADCH);
899     }
900     if (*++oli != ':') 
901     {
902         optarg = NULL;
903         if (!*place)
904             ++optind;
905     }
906     else
907     {
908         if (*place)
909             optarg = place;
910         else if (nargc <= ++optind) 
911         {
912             place = EMSG;
913             if (*ostr == ':')
914                 return (BADARG);
915             if (opterr)
916                 (void)fprintf(stderr, "%s: option requires an argument -- %c\n", __progname, optopt);
917             return (BADCH);
918         }
919         else
920             optarg = nargv[optind];
921         place = EMSG;
922         ++optind;
923     }
924     return (optopt);
925 }