2 * Run a set of tests, reporting results.
6 * runtests [-b <build-dir>] [-s <source-dir>] <test-list>
7 * runtests -o [-b <build-dir>] [-s <source-dir>] <test>
9 * In the first case, expects a list of executables located in the given file,
10 * one line per executable. For each one, runs it as part of a test suite,
11 * reporting results. Test output should start with a line containing the
12 * number of tests (numbered from 1 to this number), optionally preceded by
13 * "1..", although that line may be given anywhere in the output. Each
14 * additional line should be in the following format:
19 * not ok <number> # todo
21 * where <number> is the number of the test. An optional comment is permitted
22 * after the number if preceded by whitespace. ok indicates success, not ok
23 * indicates failure. "# skip" and "# todo" are a special cases of a comment,
24 * and must start with exactly that formatting. They indicate the test was
25 * skipped for some reason (maybe because it doesn't apply to this platform)
26 * or is testing something known to currently fail. The text following either
27 * "# skip" or "# todo" and whitespace is the reason.
29 * As a special case, the first line of the output may be in the form:
31 * 1..0 # skip some reason
33 * which indicates that this entire test case should be skipped and gives a
36 * Any other lines are ignored, although for compliance with the TAP protocol
37 * all lines other than the ones in the above format should be sent to
38 * standard error rather than standard output and start with #.
40 * This is a subset of TAP as documented in Test::Harness::TAP or
41 * TAP::Parser::Grammar, which comes with Perl.
43 * If the -o option is given, instead run a single test and display all of its
44 * output. This is intended for use with failing tests so that the person
45 * running the test suite can get more details about what failed.
47 * If built with the C preprocessor symbols SOURCE and BUILD defined, C TAP
48 * Harness will export those values in the environment so that tests can find
49 * the source and build directory and will look for tests under both
50 * directories. These paths can also be set with the -b and -s command-line
51 * options, which will override anything set at build time.
53 * Any bug reports, bug fixes, and improvements are very much welcome and
54 * should be sent to the e-mail address below. This program is part of C TAP
55 * Harness <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
57 * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009, 2010, 2011
58 * Russ Allbery <rra@stanford.edu>
60 * Permission is hereby granted, free of charge, to any person obtaining a
61 * copy of this software and associated documentation files (the "Software"),
62 * to deal in the Software without restriction, including without limitation
63 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
64 * and/or sell copies of the Software, and to permit persons to whom the
65 * Software is furnished to do so, subject to the following conditions:
67 * The above copyright notice and this permission notice shall be included in
68 * all copies or substantial portions of the Software.
70 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
71 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
72 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
73 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
74 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
75 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
76 * DEALINGS IN THE SOFTWARE.
79 /* Required for fdopen(), getopt(), and putenv(). */
81 # define _XOPEN_SOURCE 500
94 #include <sys/types.h>
99 /* sys/time.h must be included before sys/resource.h on some platforms. */
100 #include <sys/resource.h>
102 /* AIX doesn't have WCOREDUMP. */
104 # define WCOREDUMP(status) ((unsigned)(status) & 0x80)
108 * The source and build versions of the tests directory. This is used to set
109 * the SOURCE and BUILD environment variables and find test programs, if set.
110 * Normally, this should be set as part of the build process to the test
111 * subdirectories of $(abs_top_srcdir) and $(abs_top_builddir) respectively.
120 /* Test status codes. */
128 /* Indicates the state of our plan. */
130 PLAN_INIT, /* Nothing seen yet. */
131 PLAN_FIRST, /* Plan seen before any tests. */
132 PLAN_PENDING, /* Test seen and no plan yet. */
133 PLAN_FINAL /* Plan seen after some tests. */
136 /* Error exit statuses for test processes. */
137 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
138 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */
139 #define CHILDERR_STDERR 102 /* Couldn't open stderr file. */
141 /* Structure to hold data for a set of tests. */
143 char *file; /* The file name of the test. */
144 char *path; /* The path to the test program. */
145 enum plan_status plan; /* The status of our plan. */
146 unsigned long count; /* Expected count of tests. */
147 unsigned long current; /* The last seen test number. */
148 unsigned int length; /* The length of the last status message. */
149 unsigned long passed; /* Count of passing tests. */
150 unsigned long failed; /* Count of failing lists. */
151 unsigned long skipped; /* Count of skipped tests (passed). */
152 unsigned long allocated; /* The size of the results table. */
153 enum test_status *results; /* Table of results by test number. */
154 unsigned int aborted; /* Whether the set as aborted. */
155 int reported; /* Whether the results were reported. */
156 int status; /* The exit status of the test. */
157 unsigned int all_skipped; /* Whether all tests were skipped. */
158 char *reason; /* Why all tests were skipped. */
161 /* Structure to hold a linked list of test sets. */
164 struct testlist *next;
168 * Usage message. Should be used as a printf format with two arguments: the
169 * path to runtests, given twice.
171 static const char usage_message[] = "\
172 Usage: %s [-b <build-dir>] [-s <source-dir>] <test-list>\n\
173 %s -o [-b <build-dir>] [-s <source-dir>] <test>\n\
176 -b <build-dir> Set the build directory to <build-dir>\n\
177 -o Run a single test rather than a list of tests\n\
178 -s <source-dir> Set the source directory to <source-dir>\n\
180 runtests normally runs each test listed in a file whose path is given as\n\
181 its command-line argument. With the -o option, it instead runs a single\n\
182 test and shows its complete output.\n";
185 * Header used for test output. %s is replaced by the file name of the list
188 static const char banner[] = "\n\
189 Running all tests listed in %s. If any tests fail, run the failing\n\
190 test program with runtests -o to see more details.\n\n";
192 /* Header for reports of failed tests. */
193 static const char header[] = "\n\
194 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
195 -------------------------- -------------- ---- ---- ------------------------";
197 /* Include the file name and line number in malloc failures. */
198 #define xmalloc(size) x_malloc((size), __FILE__, __LINE__)
199 #define xrealloc(p, size) x_realloc((p), (size), __FILE__, __LINE__)
200 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
204 * Report a fatal error, including the results of strerror, and exit.
207 sysdie(const char *format, ...)
214 fprintf(stderr, "runtests: ");
215 va_start(args, format);
216 vfprintf(stderr, format, args);
218 fprintf(stderr, ": %s\n", strerror(oerrno));
224 * Allocate memory, reporting a fatal error and exiting on failure.
227 x_malloc(size_t size, const char *file, int line)
233 sysdie("failed to malloc %lu bytes at %s line %d",
234 (unsigned long) size, file, line);
240 * Reallocate memory, reporting a fatal error and exiting on failure.
243 x_realloc(void *p, size_t size, const char *file, int line)
245 p = realloc(p, size);
247 sysdie("failed to realloc %lu bytes at %s line %d",
248 (unsigned long) size, file, line);
254 * Copy a string, reporting a fatal error and exiting on failure.
257 x_strdup(const char *s, const char *file, int line)
265 sysdie("failed to strdup %lu bytes at %s line %d",
266 (unsigned long) len, file, line);
273 * Given a struct timeval, return the number of seconds it represents as a
274 * double. Use difftime() to convert a time_t to a double.
277 tv_seconds(const struct timeval *tv)
279 return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
284 * Given two struct timevals, return the difference in seconds.
287 tv_diff(const struct timeval *tv1, const struct timeval *tv0)
289 return tv_seconds(tv1) - tv_seconds(tv0);
294 * Given two struct timevals, return the sum in seconds as a double.
297 tv_sum(const struct timeval *tv1, const struct timeval *tv2)
299 return tv_seconds(tv1) + tv_seconds(tv2);
304 * Given a pointer to a string, skip any leading whitespace and return a
305 * pointer to the first non-whitespace character.
308 skip_whitespace(const char *p)
310 while (isspace((unsigned char)(*p)))
317 * Start a program, connecting its stdout to a pipe on our end and its stderr
318 * to /dev/null, and storing the file descriptor to read from in the two
319 * argument. Returns the PID of the new process. Errors are fatal.
322 test_start(const char *path, int *fd)
327 if (pipe(fds) == -1) {
330 sysdie("can't create pipe");
333 if (child == (pid_t) -1) {
336 sysdie("can't fork");
337 } else if (child == 0) {
338 /* In child. Set up our stdout and stderr. */
339 errfd = open("/dev/null", O_WRONLY);
341 _exit(CHILDERR_STDERR);
342 if (dup2(errfd, 2) == -1)
345 if (dup2(fds[1], 1) == -1)
348 /* Now, exec our process. */
349 if (execl(path, path, (char *) 0) == -1)
350 _exit(CHILDERR_EXEC);
352 /* In parent. Close the extra file descriptor. */
361 * Back up over the output saying what test we were executing.
364 test_backspace(struct testset *ts)
368 if (!isatty(STDOUT_FILENO))
370 for (i = 0; i < ts->length; i++)
372 for (i = 0; i < ts->length; i++)
374 for (i = 0; i < ts->length; i++)
381 * Read the plan line of test output, which should contain the range of test
382 * numbers. We may initialize the testset structure here if we haven't yet
383 * seen a test. Return true if initialization succeeded and the test should
384 * continue, false otherwise.
387 test_plan(const char *line, struct testset *ts)
393 * Accept a plan without the leading 1.. for compatibility with older
394 * versions of runtests. This will only be allowed if we've not yet seen
397 line = skip_whitespace(line);
398 if (strncmp(line, "1..", 3) == 0)
402 * Get the count, check it for validity, and initialize the struct. If we
403 * have something of the form "1..0 # skip foo", the whole file was
404 * skipped; record that. If we do skip the whole file, zero out all of
405 * our statistics, since they're no longer relevant. strtol is called
406 * with a second argument to advance the line pointer past the count to
407 * make it simpler to detect the # skip case.
409 n = strtol(line, (char **) &line, 10);
411 line = skip_whitespace(line);
413 line = skip_whitespace(line + 1);
414 if (strncasecmp(line, "skip", 4) == 0) {
415 line = skip_whitespace(line + 4);
417 ts->reason = xstrdup(line);
418 ts->reason[strlen(ts->reason) - 1] = '\0';
431 puts("ABORTED (invalid test count)");
436 if (ts->plan == PLAN_INIT && ts->allocated == 0) {
439 ts->plan = PLAN_FIRST;
440 ts->results = xmalloc(ts->count * sizeof(enum test_status));
441 for (i = 0; i < ts->count; i++)
442 ts->results[i] = TEST_INVALID;
443 } else if (ts->plan == PLAN_PENDING) {
444 if ((unsigned long) n < ts->count) {
445 printf("ABORTED (invalid test number %lu)\n", ts->count);
451 if ((unsigned long) n > ts->allocated) {
452 ts->results = xrealloc(ts->results, n * sizeof(enum test_status));
453 for (i = ts->allocated; i < ts->count; i++)
454 ts->results[i] = TEST_INVALID;
457 ts->plan = PLAN_FINAL;
464 * Given a single line of output from a test, parse it and return the success
465 * status of that test. Anything printed to stdout not matching the form
466 * /^(not )?ok \d+/ is ignored. Sets ts->current to the test number that just
470 test_checkline(const char *line, struct testset *ts)
472 enum test_status status = TEST_PASS;
476 unsigned long i, current;
479 /* Before anything, check for a test abort. */
480 bail = strstr(line, "Bail out!");
482 bail = skip_whitespace(bail + strlen("Bail out!"));
486 length = strlen(bail);
487 if (bail[length - 1] == '\n')
490 printf("ABORTED (%.*s)\n", (int) length, bail);
498 * If the given line isn't newline-terminated, it was too big for an
499 * fgets(), which means ignore it.
501 if (line[strlen(line) - 1] != '\n')
504 /* If the line begins with a hash mark, ignore it. */
508 /* If we haven't yet seen a plan, look for one. */
509 if (ts->plan == PLAN_INIT && isdigit((unsigned char)(*line))) {
510 if (!test_plan(line, ts))
512 } else if (strncmp(line, "1..", 3) == 0) {
513 if (ts->plan == PLAN_PENDING) {
514 if (!test_plan(line, ts))
517 puts("ABORTED (multiple plans)");
524 /* Parse the line, ignoring something we can't parse. */
525 if (strncmp(line, "not ", 4) == 0) {
529 if (strncmp(line, "ok", 2) != 0)
531 line = skip_whitespace(line + 2);
533 number = strtol(line, &end, 10);
534 if (errno != 0 || end == line)
535 number = ts->current + 1;
537 if (number <= 0 || (current > ts->count && ts->plan == PLAN_FIRST)) {
539 printf("ABORTED (invalid test number %lu)\n", current);
545 /* We have a valid test result. Tweak the results array if needed. */
546 if (ts->plan == PLAN_INIT || ts->plan == PLAN_PENDING) {
547 ts->plan = PLAN_PENDING;
548 if (current > ts->count)
550 if (current > ts->allocated) {
553 n = (ts->allocated == 0) ? 32 : ts->allocated * 2;
556 ts->results = xrealloc(ts->results, n * sizeof(enum test_status));
557 for (i = ts->allocated; i < n; i++)
558 ts->results[i] = TEST_INVALID;
564 * Handle directives. We should probably do something more interesting
565 * with unexpected passes of todo tests.
567 while (isdigit((unsigned char)(*line)))
569 line = skip_whitespace(line);
571 line = skip_whitespace(line + 1);
572 if (strncasecmp(line, "skip", 4) == 0)
574 if (strncasecmp(line, "todo", 4) == 0)
575 status = (status == TEST_FAIL) ? TEST_SKIP : TEST_FAIL;
578 /* Make sure that the test number is in range and not a duplicate. */
579 if (ts->results[current - 1] != TEST_INVALID) {
581 printf("ABORTED (duplicate test number %lu)\n", current);
587 /* Good results. Increment our various counters. */
589 case TEST_PASS: ts->passed++; break;
590 case TEST_FAIL: ts->failed++; break;
591 case TEST_SKIP: ts->skipped++; break;
592 case TEST_INVALID: break;
594 ts->current = current;
595 ts->results[current - 1] = status;
597 if (isatty(STDOUT_FILENO)) {
598 outlen = printf("%lu/%lu", current, ts->count);
599 ts->length = (outlen >= 0) ? outlen : 0;
606 * Print out a range of test numbers, returning the number of characters it
607 * took up. Takes the first number, the last number, the number of characters
608 * already printed on the line, and the limit of number of characters the line
609 * can hold. Add a comma and a space before the range if chars indicates that
610 * something has already been printed on the line, and print ... instead if
611 * chars plus the space needed would go over the limit (use a limit of 0 to
615 test_print_range(unsigned long first, unsigned long last, unsigned int chars,
618 unsigned int needed = 0;
621 for (n = first; n > 0; n /= 10)
624 for (n = last; n > 0; n /= 10)
630 if (limit > 0 && chars + needed > limit) {
632 if (chars <= limit) {
644 printf("%lu-", first);
652 * Summarize a single test set. The second argument is 0 if the set exited
653 * cleanly, a positive integer representing the exit status if it exited
654 * with a non-zero status, and a negative integer representing the signal
655 * that terminated it if it was killed by a signal.
658 test_summarize(struct testset *ts, int status)
661 unsigned long missing = 0;
662 unsigned long failed = 0;
663 unsigned long first = 0;
664 unsigned long last = 0;
667 fputs("ABORTED", stdout);
669 printf(" (passed %lu/%lu)", ts->passed, ts->count - ts->skipped);
671 for (i = 0; i < ts->count; i++) {
672 if (ts->results[i] == TEST_INVALID) {
674 fputs("MISSED ", stdout);
675 if (first && i == last)
679 test_print_range(first, last, missing - 1, 0);
687 test_print_range(first, last, missing - 1, 0);
690 for (i = 0; i < ts->count; i++) {
691 if (ts->results[i] == TEST_FAIL) {
692 if (missing && !failed)
695 fputs("FAILED ", stdout);
696 if (first && i == last)
700 test_print_range(first, last, failed - 1, 0);
708 test_print_range(first, last, failed - 1, 0);
709 if (!missing && !failed) {
710 fputs(!status ? "ok" : "dubious", stdout);
711 if (ts->skipped > 0) {
712 if (ts->skipped == 1)
713 printf(" (skipped %lu test)", ts->skipped);
715 printf(" (skipped %lu tests)", ts->skipped);
720 printf(" (exit status %d)", status);
722 printf(" (killed by signal %d%s)", -status,
723 WCOREDUMP(ts->status) ? ", core dumped" : "");
729 * Given a test set, analyze the results, classify the exit status, handle a
730 * few special error messages, and then pass it along to test_summarize() for
731 * the regular output. Returns true if the test set ran successfully and all
732 * tests passed or were skipped, false otherwise.
735 test_analyze(struct testset *ts)
739 if (ts->all_skipped) {
740 if (ts->reason == NULL)
743 printf("skipped (%s)\n", ts->reason);
745 } else if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
746 switch (WEXITSTATUS(ts->status)) {
749 puts("ABORTED (can't dup file descriptors)");
753 puts("ABORTED (execution failed -- not found?)");
755 case CHILDERR_STDERR:
757 puts("ABORTED (can't open /dev/null)");
760 test_summarize(ts, WEXITSTATUS(ts->status));
764 } else if (WIFSIGNALED(ts->status)) {
765 test_summarize(ts, -WTERMSIG(ts->status));
767 } else if (ts->plan != PLAN_FIRST && ts->plan != PLAN_FINAL) {
768 puts("ABORTED (no valid test plan)");
772 test_summarize(ts, 0);
773 return (ts->failed == 0);
779 * Runs a single test set, accumulating and then reporting the results.
780 * Returns true if the test set was successfully run and all tests passed,
784 test_run(struct testset *ts)
786 pid_t testpid, child;
792 /* Run the test program. */
793 testpid = test_start(ts->path, &outfd);
794 output = fdopen(outfd, "r");
798 sysdie("fdopen failed");
801 /* Pass each line of output to test_checkline(). */
802 while (!ts->aborted && fgets(buffer, sizeof(buffer), output))
803 test_checkline(buffer, ts);
804 if (ferror(output) || ts->plan == PLAN_INIT)
809 * Consume the rest of the test output, close the output descriptor,
810 * retrieve the exit status, and pass that information to test_analyze()
811 * for eventual output.
813 while (fgets(buffer, sizeof(buffer), output))
816 child = waitpid(testpid, &ts->status, 0);
817 if (child == (pid_t) -1) {
822 sysdie("waitpid for %u failed", (unsigned int) testpid);
826 status = test_analyze(ts);
828 /* Convert missing tests to failed tests. */
829 for (i = 0; i < ts->count; i++) {
830 if (ts->results[i] == TEST_INVALID) {
832 ts->results[i] = TEST_FAIL;
840 /* Summarize a list of test failures. */
842 test_fail_summary(const struct testlist *fails)
846 unsigned long i, first, last, total;
850 /* Failed Set Fail/Total (%) Skip Stat Failing (25)
851 -------------------------- -------------- ---- ---- -------------- */
852 for (; fails; fails = fails->next) {
854 total = ts->count - ts->skipped;
855 printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts->file, ts->failed,
856 total, total ? (ts->failed * 100.0) / total : 0,
858 if (WIFEXITED(ts->status))
859 printf("%4d ", WEXITSTATUS(ts->status));
869 for (i = 0; i < ts->count; i++) {
870 if (ts->results[i] == TEST_FAIL) {
871 if (first != 0 && i == last)
875 chars += test_print_range(first, last, chars, 19);
882 test_print_range(first, last, chars, 19);
887 if (ts->reason != NULL)
895 * Given the name of a test, a pointer to the testset struct, and the source
896 * and build directories, find the test. We try first relative to the current
897 * directory, then in the build directory (if not NULL), then in the source
898 * directory. In each of those directories, we first try a "-t" extension and
899 * then a ".t" extension. When we find an executable program, we fill in the
900 * path member of the testset struct. If none of those paths are executable,
901 * just fill in the name of the test with "-t" appended.
903 * The caller is responsible for freeing the path member of the testset
907 find_test(const char *name, struct testset *ts, const char *source,
911 const char *bases[4];
919 for (i = 0; bases[i] != NULL; i++) {
920 path = xmalloc(strlen(bases[i]) + strlen(name) + 4);
921 sprintf(path, "%s/%s-t", bases[i], name);
922 if (access(path, X_OK) != 0)
923 path[strlen(path) - 2] = '.';
924 if (access(path, X_OK) == 0)
930 path = xmalloc(strlen(name) + 3);
931 sprintf(path, "%s-t", name);
938 * Run a batch of tests from a given file listing each test on a line by
939 * itself. Takes two additional parameters: the root of the source directory
940 * and the root of the build directory. Test programs will be first searched
941 * for in the current directory, then the build directory, then the source
942 * directory. The file must be rewindable. Returns true iff all tests
946 test_batch(const char *testlist, const char *source, const char *build)
949 unsigned int length, i;
950 unsigned int longest = 0;
953 struct testset ts, *tmp;
954 struct timeval start, end;
956 struct testlist *failhead = NULL;
957 struct testlist *failtail = NULL;
958 struct testlist *next;
959 unsigned long total = 0;
960 unsigned long passed = 0;
961 unsigned long skipped = 0;
962 unsigned long failed = 0;
963 unsigned long aborted = 0;
966 * Open our file of tests to run and scan it, checking for lines that
967 * are too long and searching for the longest line.
969 tests = fopen(testlist, "r");
971 sysdie("can't open %s", testlist);
973 while (fgets(buffer, sizeof(buffer), tests)) {
975 length = strlen(buffer) - 1;
976 if (buffer[length] != '\n') {
977 fprintf(stderr, "%s:%u: line too long\n", testlist, line);
980 if (length > longest)
983 if (fseek(tests, 0, SEEK_SET) == -1)
984 sysdie("can't rewind %s", testlist);
987 * Add two to longest and round up to the nearest tab stop. This is how
988 * wide the column for printing the current test name will be.
992 longest += 8 - (longest % 8);
994 /* Start the wall clock timer. */
995 gettimeofday(&start, NULL);
998 * Now, plow through our tests again, running each one. Check line
999 * length again out of paranoia.
1002 while (fgets(buffer, sizeof(buffer), tests)) {
1004 length = strlen(buffer) - 1;
1005 if (buffer[length] != '\n') {
1006 fprintf(stderr, "%s:%u: line too long\n", testlist, line);
1009 buffer[length] = '\0';
1010 fputs(buffer, stdout);
1011 for (i = length; i < longest; i++)
1013 if (isatty(STDOUT_FILENO))
1015 memset(&ts, 0, sizeof(ts));
1016 ts.plan = PLAN_INIT;
1017 ts.file = xstrdup(buffer);
1018 find_test(buffer, &ts, source, build);
1020 if (test_run(&ts)) {
1024 if (ts.reason != NULL)
1027 tmp = xmalloc(sizeof(struct testset));
1028 memcpy(tmp, &ts, sizeof(struct testset));
1030 failhead = xmalloc(sizeof(struct testset));
1032 failhead->next = NULL;
1033 failtail = failhead;
1035 failtail->next = xmalloc(sizeof(struct testset));
1036 failtail = failtail->next;
1038 failtail->next = NULL;
1041 aborted += ts.aborted;
1042 total += ts.count + ts.all_skipped;
1043 passed += ts.passed;
1044 skipped += ts.skipped + ts.all_skipped;
1045 failed += ts.failed;
1049 /* Stop the timer and get our child resource statistics. */
1050 gettimeofday(&end, NULL);
1051 getrusage(RUSAGE_CHILDREN, &stats);
1053 /* Print out our final results. */
1054 if (failhead != NULL) {
1055 test_fail_summary(failhead);
1056 while (failhead != NULL) {
1057 next = failhead->next;
1065 printf("Aborted %lu test set", aborted);
1067 printf("Aborted %lu test sets", aborted);
1068 printf(", passed %lu/%lu tests", passed, total);
1070 else if (failed == 0)
1071 fputs("All tests successful", stdout);
1073 printf("Failed %lu/%lu tests, %.2f%% okay", failed, total,
1074 (total - failed) * 100.0 / total);
1077 printf(", %lu test skipped", skipped);
1079 printf(", %lu tests skipped", skipped);
1082 printf("Files=%u, Tests=%lu", line, total);
1083 printf(", %.2f seconds", tv_diff(&end, &start));
1084 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
1085 tv_seconds(&stats.ru_utime), tv_seconds(&stats.ru_stime),
1086 tv_sum(&stats.ru_utime, &stats.ru_stime));
1087 return (failed == 0 && aborted == 0);
1092 * Run a single test case. This involves just running the test program after
1093 * having done the environment setup and finding the test program.
1096 test_single(const char *program, const char *source, const char *build)
1100 memset(&ts, 0, sizeof(ts));
1101 find_test(program, &ts, source, build);
1102 if (execl(ts.path, ts.path, (char *) 0) == -1)
1103 sysdie("cannot exec %s", ts.path);
1108 * Main routine. Set the SOURCE and BUILD environment variables and then,
1109 * given a file listing tests, run each test listed.
1112 main(int argc, char *argv[])
1118 const char *source = SOURCE;
1119 const char *build = BUILD;
1121 while ((option = getopt(argc, argv, "b:hos:")) != EOF) {
1127 printf(usage_message, argv[0], argv[0]);
1140 if (argc - optind != 1) {
1141 fprintf(stderr, usage_message, argv[0], argv[0]);
1147 if (source != NULL) {
1148 setting = xmalloc(strlen("SOURCE=") + strlen(source) + 1);
1149 sprintf(setting, "SOURCE=%s", source);
1150 if (putenv(setting) != 0)
1151 sysdie("cannot set SOURCE in the environment");
1153 if (build != NULL) {
1154 setting = xmalloc(strlen("BUILD=") + strlen(build) + 1);
1155 sprintf(setting, "BUILD=%s", build);
1156 if (putenv(setting) != 0)
1157 sysdie("cannot set BUILD in the environment");
1161 test_single(argv[0], source, build);
1164 list = strrchr(argv[0], '/');
1169 printf(banner, list);
1170 exit(test_batch(argv[0], source, build) ? 0 : 1);