windows-tests-20090130
[openafs.git] / src / WINNT / tests / torture / Source / WinTorture.c
1 //#define _WIN32_WINNT 0x0500
2 //#include <windows.h>
3
4 #include "includes.h"
5 #include <Psapi.h>
6 #include "common.h"
7 #ifdef HAVE_HESOID
8     #include "ResolveLocker.h"
9     extern int ResolveLocker(USER_OPTIONS *attachOption);
10     extern void GetLockerInfo(char *Locker, char *Path);
11 #endif /* HAVE_HESOID */
12
13 #define MAX_THREADS 100
14
15 int opterr = 1;
16 int optind = 1;
17 int optopt;
18 int optreset;
19 char *optarg;
20
21 extern void LogStats(char *FileName, int ToLog, int Iteration,  int NumberOfProcesses, int NumberOfThreads, 
22                      char *HostName, int ProcessNumber, struct cmd_struct CommandInfo[],
23                      char *CommandLine, char *TargetDirectory);
24 extern int UpdateMasterLog(char *FileName, struct cmd_struct CommandInfo[]);
25 extern int BuildMasterStatLog(char *FileName, char *MoveFileName, int NumberOfProcesses, 
26                               int NumberOfThreads, char *CommandLine, int LoopCount, 
27                               char *TargetDirectory, int ProcessNumber);
28 extern void LogMessage(int ProcessNumber, char *HostName, char *FileName, char *message, int LogID);
29
30 DWORD WINAPI StressTestThread(LPVOID lpThreadParameter);
31 int   getopt(int, char**, char*);
32 DWORD FindProcessCount(char *ProcessName, HANDLE JobHandle);
33 void  show_results(char *CommandLine, char *TargetDirectory, struct cmd_struct CommandInfo[], 
34                    char *HostName, int NumberOfThreads, int CurrentLoop, int LogID);
35
36 char    *ClientText = "streamfiles.txt";
37 char    PathToSecondDir[256];
38 int     ThreadStatus[MAX_HANDLES];
39 int     verbose;
40 static int     BufferSize = 256*1024;
41 int     UseLocker = 0;
42 int     EndOnError;
43 int     AfsTrace;
44 int     ChronLog;
45 BOOL    PrintStats;
46 HANDLE  MutexHandle;
47 HANDLE  FileMutexHandle;
48 HANDLE  ChronMutexHandle;
49 HANDLE  OSMutexHandle;
50 HANDLE  ShutDownEventHandle;
51 HANDLE  PauseEventHandle;
52 HANDLE  ContinueEventHandle;
53 EXIT_STATUS ExitStatus[MAX_HANDLES];
54
55 double create_procs(char *Hostname, char *CommandLine, char *TargetDirectory, 
56                     char *AfsLocker, char *Locker, char *HostName, 
57                     int NumberOfThreads, int CurrentLoop, int LogID)
58 {
59     int     i;
60     int     status;
61     int     count;
62     int     ProcessID;
63     char    EventName[512];
64     HANDLE  hEventHandle[MAX_HANDLES];
65     HANDLE  hThreadHandle[MAX_HANDLES];
66     DWORD   dwThreadID[MAX_HANDLES];
67     struct cmd_struct *CommandInfo;
68     CRITICAL_SECTION CriticalSection;
69     PARAMETERLIST    *pParameterList[MAX_HANDLES];
70 #ifdef HAVE_HESOID
71     USER_OPTIONS     attachOption;
72 #endif
73
74     InitializeCriticalSection(&CriticalSection); 
75     for (i = 0; i < MAX_HANDLES; i++)
76     {
77         hEventHandle[i] = NULL;
78         hThreadHandle[i] = NULL;
79         pParameterList[i] = NULL;
80         dwThreadID[i] = 0;
81     }
82
83     count = 0;
84     CommandInfo = calloc(1, sizeof(struct cmd_struct) * (CMD_MAX_CMD + 1) * NumberOfThreads);
85     ProcessID = getpid();
86
87     for (i = 0; i < NumberOfThreads; i++) 
88     {
89         if (EndOnError)
90         {
91             if (ThreadStatus[count] == 0)
92                 continue;
93         }
94         sprintf(EventName, "%d%sEvent%05d", ProcessID, HostName, count);
95         hEventHandle[count] = CreateEvent(NULL, FALSE, FALSE, EventName);
96         if (hEventHandle[count] == NULL)
97             continue;
98         ResetEvent(hEventHandle[count]);
99         
100         pParameterList[count] = calloc(1, sizeof(PARAMETERLIST));
101         pParameterList[count]->ProcessNumber = count;
102         pParameterList[count]->CommandInfo = (struct cmd_struct *)(CommandInfo + (i * (CMD_MAX_CMD + 1)));
103         pParameterList[count]->BufferSize = BufferSize;
104         pParameterList[count]->PrintStats = PrintStats;
105         pParameterList[count]->CurrentLoop = CurrentLoop;
106         pParameterList[count]->AfsTrace = AfsTrace;
107         pParameterList[count]->TargetDirectory = TargetDirectory;
108         pParameterList[count]->pExitStatus = &ExitStatus[i];
109         pParameterList[count]->pThreadStatus = &ThreadStatus[i];
110         pParameterList[count]->CommandLine = CommandLine;
111         pParameterList[count]->ClientText = ClientText;
112         pParameterList[count]->PathToSecondDir = PathToSecondDir;
113         pParameterList[count]->AfsLocker = AfsLocker;
114         pParameterList[count]->HostName = HostName;
115         pParameterList[count]->ProcessID = ProcessID;;
116         pParameterList[count]->LogID = LogID;;
117
118         ThreadStatus[count] = 0;
119         hThreadHandle[count] = CreateThread(NULL, 0, &StressTestThread, (LPVOID)pParameterList[count], CREATE_SUSPENDED, &dwThreadID[count]);
120         if (hThreadHandle[count] != NULL)
121         {
122             ResumeThread(hThreadHandle[count]);
123             ThreadStatus[count] = 1;
124             ++count;
125         }
126         else
127         {
128             CloseHandle(hEventHandle[count]);
129             if (pParameterList[count] != NULL)
130                 free(pParameterList[count]);
131             pParameterList[count] = NULL;
132             hThreadHandle[count] = NULL;
133             hEventHandle[count] = NULL;
134         }
135     }
136
137     count = 0;
138     for (i = 0; i < MAX_HANDLES; i++)
139     {
140         if (hEventHandle[i] != NULL)
141             ++count;
142     }
143 #ifdef HAVE_HESOID
144     if (UseLocker)
145     {
146         int          rc;
147
148         memset(&attachOption, '\0', sizeof(attachOption));
149         strcpy(attachOption.Locker, Locker);
150         strcpy(attachOption.type, "locker");
151         if (rc = ResolveLocker(&attachOption))
152         {
153             if (!stricmp(attachOption.type, "AFS"))
154             {
155                 printf("Unable to attach locker %s - AFS is not supported\n", Locker);
156                 exit(1);
157             }
158             strcpy(AfsLocker, attachOption.SubMount);
159             memset(&attachOption, '\0', sizeof(attachOption));
160             strcpy(attachOption.Locker, Locker);
161             strcpy(attachOption.type, "locker");
162             if (rc = attach(attachOption, 0, 0, Locker))
163             {
164                 printf("Unable to attach locker %s\n", Locker);
165                 exit(1);
166             }
167         }
168     }
169 #endif /* HAVE_HESOID */
170
171     status = WaitForMultipleObjects(count, hEventHandle, TRUE, INFINITE);
172     for (i = 0; i < MAX_HANDLES; i++)
173     {
174         if (hEventHandle[i] != NULL)
175             CloseHandle(hEventHandle[i]);
176         if (pParameterList[i] != NULL)
177             free(pParameterList[i]);
178         pParameterList[i] = NULL;
179         hEventHandle[i] = NULL;
180     }
181
182     for (i = 0; i < NumberOfThreads; i++)
183     {
184         char    FileName[128];
185         char    temp[512];
186
187         if (strlen(ExitStatus[i].Reason))
188         {
189             sprintf(temp, "Thread %0d exited with reason: %s", i, ExitStatus[i].Reason);
190         }
191         else
192         {
193             sprintf(temp, "Thread %0d completed\n", i);
194         }
195         if (verbose)
196             printf("%s", temp);
197         sprintf(FileName, "Thread_%05d.log", i);
198         LogMessage(i, HostName, FileName, temp, LogID);
199         sprintf(temp, "Ended Iteration %0d\n\n", CurrentLoop);
200         LogMessage(i, HostName, FileName, temp, LogID);
201         CloseHandle(hEventHandle[i]);
202     }
203     show_results(CommandLine, TargetDirectory, CommandInfo, HostName, NumberOfThreads, CurrentLoop, LogID);
204     free(CommandInfo);
205     DeleteCriticalSection(&CriticalSection);
206
207     return(0);
208 }
209
210 static void usage(void)
211 {
212
213     fprintf(stderr, "usage: wintorture [options]\n");
214     fprintf(stderr, "where options can be:\n");
215     fprintf(stderr, "\t-b        Create a chronological log.\n");
216     fprintf(stderr, "\t-c <txt>  Specifies the script txt file to use.\n");
217     fprintf(stderr, "\t-e        End thread processing on an error.\n");
218     fprintf(stderr, "\t-f <name> Target directory name.\n");
219     fprintf(stderr, "\t-i <num>  Number of iterations of the stress test to run.\n");
220     fprintf(stderr, "\t            This option will override the -m option.\n");
221 #ifdef HAVE_HESOID
222     fprintf(stderr, "\t-l <path> AFS locker or AFS submount in which to create the target directory.\n");
223 #endif /* HAVE_HESOID */
224     fprintf(stderr, "\t-m <num>  The number of minutes to run the stress test.\n");
225     fprintf(stderr, "\t            This option will override the -i option.\n");
226     fprintf(stderr, "\t-n <num>  The number of threads to run.\n");
227     fprintf(stderr, "\t-p <path> UNC path to second directory.\n");
228     fprintf(stderr, "\t-s        Output stats.\n");
229     fprintf(stderr, "\t-t        Do AFS trace logging.\n");
230     fprintf(stderr, "\t-u <UNC>  UNC path to target directory.\n");
231     fprintf(stderr, "\t-v        Turn on verbose mode.\n");
232     fprintf(stderr, "\nNOTE: The switches are not case sensitive.  You\n");
233     fprintf(stderr, "\n      may use either upper or lower case letters.\n\n");
234 }
235
236 void show_results(char *CommandLine, char *TargetDirectory, struct cmd_struct *CommandInfo, 
237                   char *HostName, int NumberOfThreads, int CurrentLoop, int LogID)
238 {
239         struct cmd_struct TotalCommandInfo[CMD_MAX_CMD + 1];
240         int         i;
241     int         j;
242         unsigned    grand_total = 0;
243     char        FileName[256];
244     char        WorkingDirectory[512];
245     struct cmd_struct *FinalCmdInfo;
246
247         for (j = 0; j <= CMD_MAX_CMD; j++) {
248                 TotalCommandInfo[j].count = 0;
249                 TotalCommandInfo[j].min_sec = 0;
250                 TotalCommandInfo[j].max_sec = 0;
251                 TotalCommandInfo[j].MilliSeconds = 0;
252                 TotalCommandInfo[j].total_sec = 0;
253                 TotalCommandInfo[j].total_sum_of_squares = 0;
254                 TotalCommandInfo[j].ErrorCount = 0;
255                 TotalCommandInfo[j].ErrorTime = 0;
256         }
257
258     memset(ExitStatus, '\0', sizeof(ExitStatus[0]) * MAX_HANDLES);
259
260         for (j = 0; j < NumberOfThreads; j++) 
261     { 
262             FinalCmdInfo = CommandInfo + (j * (CMD_MAX_CMD + 1));
263
264                 for (i = 0; i <= CMD_MAX_CMD; i++)
265         {
266                     TotalCommandInfo[i].count += FinalCmdInfo[i].count;
267             TotalCommandInfo[i].total_sec += FinalCmdInfo[i].total_sec; 
268             TotalCommandInfo[i].total_sum_of_squares += FinalCmdInfo[i].total_sum_of_squares; 
269             TotalCommandInfo[i].ErrorCount += FinalCmdInfo[i].ErrorCount; 
270             TotalCommandInfo[i].ErrorTime += FinalCmdInfo[i].ErrorTime; 
271             grand_total += FinalCmdInfo[j].total_sec;
272             if (!TotalCommandInfo[i].min_sec || (TotalCommandInfo[i].min_sec > FinalCmdInfo[i].min_sec))
273                 TotalCommandInfo[i].min_sec = FinalCmdInfo[i].min_sec;
274             if (TotalCommandInfo[i].max_sec < FinalCmdInfo[i].max_sec)
275                 TotalCommandInfo[i].max_sec = FinalCmdInfo[i].max_sec;
276         }
277         }
278
279
280     memset(WorkingDirectory, '\0', sizeof(WorkingDirectory));
281     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
282     sprintf(FileName, "%s\\log%05d\\%s\\ProcessStats.log", WorkingDirectory, LogID, HostName);
283
284     if (PrintStats)
285         LogStats(FileName, 0, CurrentLoop, 1, NumberOfThreads, HostName, -1, TotalCommandInfo, 
286                  CommandLine, TargetDirectory);
287     LogStats(FileName, 1, CurrentLoop, 1, NumberOfThreads, HostName, -1, TotalCommandInfo, 
288              CommandLine, TargetDirectory);
289
290     sprintf(FileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, "MasterStatLog.log");
291     UpdateMasterLog(FileName, TotalCommandInfo);
292     sprintf(FileName, "%s\\log%05d\\%s\\%s", WorkingDirectory, LogID, HostName, "MasterProcessStatLog.log");
293     UpdateMasterLog(FileName, TotalCommandInfo);
294 }
295
296
297 int main(int argc, char *argv[])
298 {
299     int         i;
300     int         LoopCount;
301     time_t      SecondsToRun;
302     int         NumberOfIterations;
303     int         StressTestUsed = 0;
304     int         rc;
305     int         opt;
306     int         ErrorState;
307     int         NumberOfProcesses;
308     int         NumberOfThreads;
309     int         IterationCount;
310     int         CurrentLoop = 0;
311     int         LogID;
312     time_t      TotalTime;
313     time_t      StartTime;
314     time_t      EndTime;
315     extern char *optarg;
316     extern int  optind;
317     FILE        *Mfp;
318     FILE        *fp;
319     char        *p;
320     char        tbuffer[10];
321     char        buffer[512];
322     char        FileName[256];
323     char        MoveFileName[256];
324     char        Locker[64];
325     char        command[512];
326     char        DateTime[512];
327     char        WorkingDirectory[512];
328     char        CommandLine[512];
329     char        TargetDirectory[512];
330     char        AfsLocker[256];
331     char        HostName[128];
332     char        JobName[128];
333     SYSTEMTIME  SystemTime;
334     SYSTEMTIME  LocalTime;
335     TIME_ZONE_INFORMATION TimeZoneInformation;
336     HANDLE      ExitMutexHandle;
337     HANDLE      JobHandle;
338
339
340     memset(HostName, '\0', sizeof(HostName));
341     memset(PathToSecondDir, '\0', sizeof(PathToSecondDir));
342     memset(Locker, '\0', sizeof(Locker));
343     SecondsToRun = 0;
344     NumberOfIterations = 0;
345     EndOnError = 0;
346     UseLocker = 0;
347     ChronLog = 0;
348     AfsTrace = 0;
349     verbose = 0;
350     NumberOfThreads = 1;
351     NumberOfProcesses = 1;
352     NumberOfThreads = 1;
353     LogID = 0;
354     PrintStats = FALSE;
355
356     while ((opt = getopt(argc, argv, "A:a:BbC:c:D:d:EeF:f:G:g:I:i:L:l:M:m:N:n:P:p:SsTtU:u:Vv")) != EOF)
357     {
358
359         switch (opt)
360         {
361             case 'a':
362             case 'A':
363                 break;
364             case 'b':
365             case 'B':
366                 ChronLog = 1;
367                 break;
368             case 'c':
369             case 'C':
370                 ClientText = optarg;
371                 break;
372             case 'd':
373             case 'D':
374                 StressTestUsed = 1;
375                 NumberOfProcesses = atoi(optarg);
376                 break;
377             case 'e':
378             case 'E':
379                 EndOnError = 1;
380                 break;
381             case 'f':
382             case 'F':
383                 strcpy(HostName, optarg);
384                 for (i = 0; i < (int)strlen(HostName); i++)
385                 {
386                     if ((HostName[i] == '\\') || (HostName[i] == '/'))
387                     {
388                         printf("\nInvalid -F usage...Subdirectories not allowed\n\n");
389                         usage();
390                         exit(1);
391                     }
392                 }
393                 break;
394             case 'g':
395             case 'G':
396                 StressTestUsed = 1;
397                 LogID = atoi(optarg);
398                 break;
399             case 'i':
400             case 'I':
401                 SecondsToRun = 0;
402                 NumberOfIterations = atoi(optarg);
403                 if (NumberOfIterations < 0)
404                     NumberOfIterations = 0;
405                 break;
406 #ifdef HAVE_HESOID
407             case 'l':
408             case 'L':
409                 strcpy(Locker, optarg);
410                 UseLocker = 1;
411                 break;
412 #endif /* HAVE_HESOID */
413             case 'm':
414             case 'M':
415                 NumberOfIterations = 0;
416                 SecondsToRun = atoi(optarg) * 60;
417                 if (SecondsToRun < 0)
418                     SecondsToRun = 0;
419                 break;
420             case 'n':
421             case 'N':
422                 NumberOfThreads = atoi(optarg);
423                 break;
424             case 'p':
425             case 'P':
426                 strcpy(PathToSecondDir, optarg);
427                 for(p = PathToSecondDir; *p; p++)
428                 {
429                     if(*p == '/')
430                         *p = '\\';
431                 }
432                 break;
433             case 's':
434             case 'S':
435                 PrintStats = TRUE;
436                 break;
437             case 't':
438             case 'T':
439                 AfsTrace = 1;
440                 break;
441             case 'u':
442             case 'U':
443                 UseLocker = 0;
444                 strcpy(Locker, optarg);
445                 break;
446             case 'v':
447             case 'V':
448                 verbose = 1;
449                 break;
450             default:
451                 usage();
452                 exit(1);
453                 break;
454         }
455     }
456
457     if (strlen(HostName) == 0)
458     {
459         printf("You must use the -f option to specify a target directory\n\n");
460         usage();
461         exit(1);
462     }
463     if (strlen(Locker) == 0)
464     {
465         printf("You must use either the -u or the -l option\n\n");
466         usage();
467         exit(1);
468     }
469
470     memset(CommandLine, '\0', sizeof(CommandLine));
471     for (i = 1; i < argc; i++)
472     {
473         if (StressTestUsed)
474         {
475             if (!stricmp(argv[i], "-f"))
476             {
477                 char temp[64];
478
479                 strcpy(temp, argv[i + 1]);
480                 temp[strlen(temp) - 5] = '\0';
481                 strcat(CommandLine, argv[i]);
482                 strcat(CommandLine, " ");
483                 strcat(CommandLine, temp);
484                 strcat(CommandLine, " ");
485                 ++i;
486                 continue;
487             }
488         }
489         strcat(CommandLine, argv[i]);
490         strcat(CommandLine, " ");
491     }
492
493     argc -= optind;
494     argv += optind;
495
496     if (strlen(Locker) == 0)
497     {
498         usage();
499         exit(1);
500     }
501
502
503     for(p = Locker; *p; p++)
504     {
505         if(*p == '/')
506             *p = '\\';
507     }
508  
509 #ifdef HAVE_HESOID
510     if (UseLocker)
511     {
512         char *sPtr;
513
514         sprintf(AfsLocker, "\\\\afs\\%s", Locker);
515         memset(buffer, '\0', sizeof(buffer));
516         GetLockerInfo(Locker, buffer);
517         if (strlen(buffer) != 0)
518         {
519             sPtr = strstr(buffer, "/afs/");
520             sPtr += strlen("/afs/");
521             strcpy(TargetDirectory, sPtr);
522             sPtr = strchr(TargetDirectory, ' ');
523             if (sPtr != NULL)
524                 (*sPtr) = '\0';
525             while ((sPtr = strchr(TargetDirectory, '/')) != NULL)
526                 (*sPtr) = '\\';
527         }
528         else
529         {
530             strcpy(TargetDirectory, Locker);
531         }
532     }
533     else
534     {
535 #endif /* HAVE_HESOID */
536         strcpy(AfsLocker, Locker);
537         if (!strnicmp(Locker, "\\\\afs\\", strlen("\\\\afs\\")))
538             strcpy(TargetDirectory, &Locker[strlen("\\\\afs\\")]);
539         else
540             strcpy(TargetDirectory, Locker);
541 #ifdef HAVE_HESOID
542     }
543 #endif /* HAVE_HESOID */
544
545
546     TotalTime = 0;
547     LoopCount = 0;
548     CurrentLoop = 0;
549
550     ExitMutexHandle = CreateMutex(NULL, FALSE, "AfsExitEvent");
551     ChronMutexHandle = CreateMutex(NULL, FALSE, "WinTortureChronMutex");
552     MutexHandle = CreateMutex(NULL, FALSE, "WinTortureMutex");
553     FileMutexHandle = CreateMutex(NULL, FALSE, "WinTortureFileMutex");
554     OSMutexHandle = CreateMutex(NULL, FALSE, "WinTortureOSMutex");
555
556     for (i = 0; i < MAX_THREADS; i++)
557     {
558         ThreadStatus[i] = 2;
559     }
560
561     sprintf(JobName, "%s%05d", "JOB", LogID);
562     JobHandle = CreateJobObject(NULL, JobName);
563     rc = AssignProcessToJobObject(JobHandle, GetCurrentProcess());
564
565     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
566     sprintf(FileName, "%s\\log%05d", WorkingDirectory, LogID);
567     CreateDirectory(FileName, NULL);
568     sprintf(FileName, "%s\\test", WorkingDirectory);
569     CreateDirectory(FileName, NULL);
570     if (!StressTestUsed)
571     {
572         sprintf(FileName, "%s\\log%05d\\Chron.log", WorkingDirectory, LogID);
573         DeleteFile(FileName);
574     }
575
576     sprintf(FileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, HostName);
577     sprintf(command, "rmdir /S /Q %s > %s\\test\\test", FileName, WorkingDirectory);
578     system(command);
579
580     ShutDownEventHandle = CreateEvent(NULL, TRUE, FALSE, "AfsShutdownEvent");
581     PauseEventHandle = CreateEvent(NULL, TRUE, FALSE, "AfsPauseEvent");
582     ContinueEventHandle = CreateEvent(NULL, TRUE, FALSE, "AfsContinueEvent");
583
584     IterationCount = 0;
585     time(&StartTime);
586     if ((NumberOfIterations == 0) && (SecondsToRun == 0))
587         NumberOfIterations = 1;
588     else if (SecondsToRun != 0)
589     {
590         SecondsToRun += StartTime;
591     }
592
593     while (1)
594     {
595         if (SecondsToRun != 0)
596         {
597             time(&StartTime);
598             if (StartTime > SecondsToRun)
599             {
600                 break;
601             }
602         }
603         if (NumberOfIterations != 0)
604         {
605             if (LoopCount >= NumberOfIterations)
606             {
607                 break;
608             }
609         }
610         if (rc = WaitForSingleObject(ShutDownEventHandle, 0) == WAIT_OBJECT_0)
611         {
612             break;
613         }
614         ++LoopCount;
615         CurrentLoop = LoopCount;
616
617         _strtime(tbuffer);
618         printf("\nIteration %d started at: %s\n", LoopCount, tbuffer);
619         create_procs(HostName, CommandLine, TargetDirectory, AfsLocker, Locker, 
620                      HostName, NumberOfThreads, CurrentLoop, LogID);
621         _strtime(tbuffer);
622         printf("Iteration %d ended at: %s\n", LoopCount, tbuffer);
623         time(&EndTime);
624         printf("Iteration %d lapse time: %ld seconds\n", LoopCount, EndTime - StartTime);
625         TotalTime += EndTime - StartTime;
626         sprintf(FileName, "%s\\log%05d\\IterationCount", WorkingDirectory, LogID);
627         WaitForSingleObject(MutexHandle, 20 * 1000);
628         if ((fp = fopen(FileName, "r")) != NULL)
629         {
630             fgets(buffer, sizeof(buffer), fp);
631             IterationCount = atoi(buffer);
632             fclose(fp);
633         }
634         ++IterationCount;
635         fp = fopen(FileName, "w");
636         fprintf(fp, "%d\n", IterationCount);
637         fclose(fp);
638         ReleaseMutex(MutexHandle);
639         if (EndOnError)
640         {
641             for (i = 0; i < MAX_THREADS; i++)
642             {
643                 if (ThreadStatus[i] == 1)
644                 {
645                     break;
646                 }
647             }
648             if (i >= MAX_THREADS)
649                 break;
650         }
651         ErrorState = 0;
652         for (i = 0; i < MAX_THREADS; i++)
653         {
654             if (ThreadStatus[i] == 0)
655             {
656                 ErrorState = 1;
657                 ThreadStatus[i] = 1;
658             }
659         }
660         if (ErrorState)
661         {
662             printf("\nSleeping for 3 minutes for error recovery\n\n");
663             Sleep(3 * 60 * 1000);
664         }
665     }
666     if (LoopCount == 0)
667         return(0);
668
669     sprintf(FileName, ".\\log%05d\\%s\\Master.log", LogID, HostName);
670     Mfp = fopen(FileName, "w+");
671     fprintf(Mfp, "Average Iteration Time = %.02f minutes\n\n", ((float)TotalTime/(float)(LoopCount))/60.0);
672     for (i = 0; i < NumberOfThreads; i++)
673     {
674         sprintf(FileName, ".\\log%05d\\%s\\Thread_%05d.log", LogID, HostName, i);
675         fp = fopen(FileName, "r");
676         if (fp != NULL)
677         {
678             fprintf(Mfp, "START OF THREAD %d\n\n", i);
679             while (fgets(buffer, 512, fp) != NULL)
680             {
681                 fprintf(Mfp, "%s", buffer);
682             }
683             fclose(fp);
684             fprintf(Mfp, "END OF THREAD %d\n\n", i);
685         }
686     }
687     fclose(Mfp);
688
689     memset(WorkingDirectory, '\0', sizeof(WorkingDirectory));
690     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
691
692     sprintf(FileName, "%s\\log%05d\\%s\\%s", WorkingDirectory, LogID, HostName, "MasterProcessStatLog.log");
693     sprintf(MoveFileName, "%s\\log%05d\\%s\\%s", WorkingDirectory, LogID, HostName, "MasterProcessStatLogRaw.log");
694     BuildMasterStatLog(FileName, MoveFileName, NumberOfProcesses, NumberOfThreads, CommandLine, LoopCount, 
695                        TargetDirectory, -1);
696
697 //    sprintf(DateTime, "%s-%04d%02d%02d-%02d%02d%02d", HostName,
698 //                                                      LocalTime.wYear, 
699 //                                                      LocalTime.wMonth, 
700 //                                                      LocalTime.wDay,
701 //                                                      LocalTime.wHour,
702 //                                                      LocalTime.wMinute,
703 //                                                      LocalTime.wSecond);
704 //    sprintf(command, "rename %s\\log%05d\\%s %s", WorkingDirectory, LogID, HostName, DateTime);
705 //    rc = system(command);
706
707     WaitForSingleObject(ExitMutexHandle, 20 * 1000);
708     Sleep(3 * 1000);
709
710     GetSystemTime(&SystemTime);
711     GetTimeZoneInformation(&TimeZoneInformation);
712     SystemTimeToTzSpecificLocalTime(&TimeZoneInformation, &SystemTime, &LocalTime);
713
714     NumberOfProcesses = 0;
715     sprintf(FileName, "%s\\log%05d\\ProcessCount", WorkingDirectory, LogID);
716     if ((fp = fopen(FileName, "r")) != NULL)
717     {
718         fgets(buffer, sizeof(buffer), fp);
719         NumberOfProcesses = atoi(buffer);
720         fclose(fp);
721     }
722     ++NumberOfProcesses;
723     fp = fopen(FileName, "w");
724     fprintf(fp, "%d\n", NumberOfProcesses);
725     fclose(fp);
726
727     if (FindProcessCount("wintorture.exe", JobHandle) == 1)
728     {
729         NumberOfProcesses = 0;
730         sprintf(FileName, "%s\\log%05d\\ProcessCount", WorkingDirectory, LogID);
731         sprintf(MoveFileName, "%s\\log%05d\\ProcessCountRaw", WorkingDirectory, LogID);
732         if ((fp = fopen(FileName, "r")) != NULL)
733         {
734             fgets(buffer, sizeof(buffer), fp);
735             NumberOfProcesses = atoi(buffer);
736             fclose(fp);
737             MoveFile(FileName, MoveFileName);
738         }
739
740         IterationCount = 0;
741         sprintf(FileName, "%s\\log%05d\\IterationCount", WorkingDirectory, LogID);
742         sprintf(MoveFileName, "%s\\log%05d\\IterationCountRaw", WorkingDirectory, LogID);
743         if ((fp = fopen(FileName, "r")) != NULL)
744         {
745             fgets(buffer, sizeof(buffer), fp);
746             IterationCount = atoi(buffer);
747             fclose(fp);
748             MoveFile(FileName, MoveFileName);
749         }
750
751         sprintf(FileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, "MasterStatLog.log");
752         sprintf(MoveFileName, "%s\\log%05d\\%s", WorkingDirectory, LogID, "MasterStatLogRaw.log");
753         BuildMasterStatLog(FileName, MoveFileName, NumberOfProcesses, NumberOfThreads, 
754                            CommandLine, IterationCount, TargetDirectory, -2);
755         sprintf(DateTime, "%s%05d-%04d%02d%02d-%02d%02d%02d", "log",
756                                                           LogID,
757                                                           LocalTime.wYear, 
758                                                           LocalTime.wMonth, 
759                                                           LocalTime.wDay,
760                                                           LocalTime.wHour,
761                                                           LocalTime.wMinute,
762                                                           LocalTime.wSecond);
763         sprintf(command, "rename %s\\log%05d %s", WorkingDirectory, LogID, DateTime);
764         rc = system(command);
765         ResetEvent(ShutDownEventHandle);
766         ResetEvent(PauseEventHandle);
767         ResetEvent(ContinueEventHandle);
768         CloseHandle(ShutDownEventHandle);
769         CloseHandle(PauseEventHandle);
770         CloseHandle(ContinueEventHandle);
771
772     }
773
774     ReleaseMutex(ExitMutexHandle);
775     CloseHandle(JobHandle);
776     return 0;
777 }
778
779 int ProcessNameAndID(DWORD processID, char *ProcessName, HANDLE JobHandle)
780 {
781     char    szProcessName[1024] = "unknown";
782     char    FileName[1024];
783     char    WorkingDirectory[512];
784     HANDLE  hProcess;
785     int     Count;
786
787     memset(WorkingDirectory, '\0', sizeof(WorkingDirectory));
788     GetCurrentDirectory(sizeof(WorkingDirectory), WorkingDirectory);
789     strcat(WorkingDirectory, "\\");
790     Count = 0;
791     hProcess = OpenProcess(PROCESS_ALL_ACCESS, FALSE, processID);
792     if (hProcess)
793     {
794         HMODULE hMod;
795         DWORD cbNeeded;
796
797         if (EnumProcessModules(hProcess, &hMod, sizeof(hMod), &cbNeeded))
798         {
799             memset(szProcessName, '\0', sizeof(szProcessName));
800             GetModuleBaseName(hProcess, hMod, szProcessName, sizeof(szProcessName));
801             if (!stricmp(szProcessName, ProcessName))
802             {
803                 memset(FileName, '\0', sizeof(FileName));
804                 if (GetModuleFileNameEx(hProcess, hMod, FileName, sizeof(FileName) - 1))
805                 {
806                     if (!strnicmp(WorkingDirectory, FileName, strlen(WorkingDirectory)))
807                     {
808                         ++Count;
809                     }
810                 }
811             }
812         }
813     }
814     CloseHandle(hProcess);
815     return(Count);
816 }
817
818 DWORD FindProcessCount(char *ProcessName, HANDLE JobHandle)
819 {
820     DWORD       aProcesses[8092];
821     DWORD       cbNeeded;
822     DWORD       cProcesses;
823     int         Count;
824     int         rc;
825     unsigned    int i;
826     JOBOBJECT_BASIC_PROCESS_ID_LIST IdList;
827
828     rc = QueryInformationJobObject(JobHandle, JobObjectBasicProcessIdList, &IdList, sizeof(IdList), NULL);
829     return(IdList.NumberOfAssignedProcesses);
830
831     if (!EnumProcesses(aProcesses, sizeof(aProcesses), &cbNeeded))
832         return(1);
833
834     Count = 0;
835     cProcesses = cbNeeded / sizeof(DWORD);
836
837     for (i = 0; i < cProcesses; i++)
838     {
839
840 //        Count += ProcessNameAndID(aProcesses[i], ProcessName, JobHandle);
841     }
842
843     return(Count);
844 }
845
846 char *_progname(char *nargv0)
847 {
848     char    *tmp;
849
850     tmp = strrchr(nargv0, '/');
851     if (tmp)
852         tmp++;
853     else
854         tmp = nargv0;
855     return(tmp);
856 }
857
858 #define BADCH   (int)'?'
859 #define BADARG  (int)':'
860 #define EMSG    ""
861
862 int getopt(int nargc, char *nargv[], char *ostr)
863 {
864     static char *__progname = 0;
865     static char *place = EMSG;          /* option letter processing */
866     char *oli;                          /* option letter list index */
867
868     __progname = __progname?__progname:_progname(*nargv);
869
870     if (optreset || !*place)
871     {
872         optreset = 0;
873         if (optind >= nargc || *(place = nargv[optind]) != '-')
874         {
875             place = EMSG;
876             return (-1);
877         }
878         if (place[1] && *++place == '-' && place[1] == '\0')
879         {
880             ++optind;
881             place = EMSG;
882             return (-1);
883         }
884     }
885     if ((optopt = (int)*place++) == (int)':' || !(oli = strchr(ostr, optopt))) 
886     {
887         if (optopt == (int)'-')
888             return (-1);
889         if (!*place)
890             ++optind;
891         if (opterr && *ostr != ':')
892             (void)fprintf(stderr, "%s: illegal option -- %c\n", __progname, optopt);
893         return (BADCH);
894     }
895     if (*++oli != ':') 
896     {
897         optarg = NULL;
898         if (!*place)
899             ++optind;
900     }
901     else
902     {
903         if (*place)
904             optarg = place;
905         else if (nargc <= ++optind) 
906         {
907             place = EMSG;
908             if (*ostr == ':')
909                 return (BADARG);
910             if (opterr)
911                 (void)fprintf(stderr, "%s: option requires an argument -- %c\n", __progname, optopt);
912             return (BADCH);
913         }
914         else
915             optarg = nargv[optind];
916         place = EMSG;
917         ++optind;
918     }
919     return (optopt);
920 }