source: trunk/coreutils/tests/Coreutils.pm@ 2612

Last change on this file since 2612 was 2582, checked in by bird, 19 years ago

added _BINMODE variants for OUT, IN and ERR.

File size: 14.3 KB
Line 
1package Coreutils;
2# This is a testing framework.
3
4require 5.003;
5use strict;
6use vars qw($VERSION @ISA @EXPORT);
7
8use FileHandle;
9use File::Compare qw(compare);
10
11@ISA = qw(Exporter);
12($VERSION = '$Revision: 1.2 $ ') =~ tr/[0-9].//cd;
13@EXPORT = qw (run_tests);
14
15my $debug = $ENV{DEBUG};
16
17my @Types = qw (IN OUT ERR AUX CMP EXIT PRE POST OUT_SUBST ERR_SUBST ENV ENV_DEL
18 IN_BINMODE OUT_BINMODE ERR_BINMODE); # bird
19my %Types = map {$_ => 1} @Types;
20my %Zero_one_type = map {$_ => 1}
21 qw (OUT ERR EXIT PRE POST OUT_SUBST ERR_SUBST ENV
22 OUT_BINMODE ERR_BINMODE); # bird
23my $srcdir = $ENV{srcdir};
24my $Global_count = 1;
25
26# When running in a DJGPP environment, make $ENV{SHELL} point to bash.
27# Otherwise, a bad shell might be used (e.g. command.com) and many
28# tests would fail.
29defined $ENV{DJDIR}
30 and $ENV{SHELL} = "$ENV{DJDIR}/bin/bash.exe";
31
32# A file spec: a scalar or a reference to a single-keyed hash
33# ================
34# 'contents' contents only (file name is derived from test name)
35# {filename => 'contents'} filename and contents
36# {filename => undef} filename only -- $(srcdir)/filename must exist
37#
38# FIXME: If there is more than one input file, then you can't specify `REDIR'.
39# PIPE is still ok.
40#
41# I/O spec: a hash ref with the following properties
42# ================
43# - one key/value pair
44# - the key must be one of these strings: IN, OUT, ERR, AUX, CMP, EXIT
45# - the value must be a file spec
46# {OUT => 'data'} put data in a temp file and compare it to stdout from cmd
47# {OUT => {'filename'=>undef}} compare contents of existing filename to
48# stdout from cmd
49# {OUT => {'filename'=>[$CTOR, $DTOR]}} $CTOR and $DTOR are references to
50# functions, each which is passed the single argument `filename'.
51# $CTOR must create `filename'.
52# DTOR may be omitted in which case `sub{unlink @_[0]}' is used.
53# FIXME: implement this
54# {ERR => ...}
55# Same as for OUT, but compare with stderr, not stdout.
56# {OUT_SUBST => 's/variable_output/expected_output/'}
57# Transform actual standard output before comparing it against expected output.
58# This is useful e.g. for programs like du that produce output that
59# varies a lot from system. E.g., an empty file may consume zero file
60# blocks, or more, depending on the OS and on the file system type.
61# {ERR_SUBST => 's/variable_output/expected_output/'}
62# Transform actual stderr output before comparing it against expected.
63# This is useful when verifying that we get a meaningful diagnostic.
64# For example, in rm/fail-2eperm, we have to account for three different
65# diagnostics: Operation not permitted, Not owner, and Permission denied.
66# {EXIT => N} expect exit status of cmd to be N
67# {ENV => 'VAR=val ...'}
68# Prepend 'VAR=val ...' to the command that we execute via `system'.
69# {ENV_DEL => 'VAR'}
70# Remove VAR from the environment just before running the corresponding
71# command, and restore any value just afterwards.
72#
73# There may be many input file specs. File names from the input specs
74# are concatenated in order on the command line.
75# There may be at most one of the OUT-, ERR-, and EXIT-keyed specs.
76# If the OUT-(or ERR)-keyed hash ref is omitted, then expect no output
77# on stdout (or stderr).
78# If the EXIT-keyed one is omitted, then expect the exit status to be zero.
79
80# FIXME: Make sure that no junkfile is also listed as a
81# non-junkfile (i.e. with undef for contents)
82
83sub _shell_quote ($)
84{
85 my ($string) = @_;
86 $string =~ s/\'/\'\\\'\'/g;
87 return "'$string'";
88}
89
90sub _create_file ($$$$$) # bird
91{
92 my ($program_name, $test_name, $file_name, $data, $binary) = @_; # bird
93 my $file;
94 if (defined $file_name)
95 {
96 $file = $file_name;
97 }
98 else
99 {
100 $file = "$test_name.$Global_count";
101 ++$Global_count;
102 }
103
104 warn "creating file `$file' with contents `$data'\n" if $debug;
105
106 # The test spec gave a string.
107 # Write it to a temp file and return tempfile name.
108 my $fh = new FileHandle "> $file";
109 die "$program_name: $file: $!\n" if ! $fh;
110 if ($binary) { # bird
111 binmode $fh; # bird
112 } # bird
113 print $fh $data;
114 $fh->close || die "$program_name: $file: $!\n";
115
116 return $file;
117}
118
119sub _compare_files ($$$$$)
120{
121 my ($program_name, $test_name, $in_or_out, $actual, $expected) = @_;
122
123 my $differ = compare ($expected, $actual);
124 if ($differ)
125 {
126 my $info = (defined $in_or_out ? "std$in_or_out " : '');
127 warn "$program_name: test $test_name: ${info}mismatch, comparing "
128 . "$actual (actual) and $expected (expected)\n";
129 # Ignore any failure, discard stderr.
130 system "diff -c $actual $expected 2>/dev/null";
131 }
132
133 return $differ;
134}
135
136sub _process_file_spec ($$$$$$) # bird
137{
138 my ($program_name, $test_name, $file_spec, $type, $junk_files, $is_binmode) = @_; # bird
139
140 my ($file_name, $contents);
141 if (!ref $file_spec)
142 {
143 ($file_name, $contents) = (undef, $file_spec);
144 }
145 elsif (ref $file_spec eq 'HASH')
146 {
147 my $n = keys %$file_spec;
148 die "$program_name: $test_name: $type spec has $n elements --"
149 . " expected 1\n"
150 if $n != 1;
151 ($file_name, $contents) = each %$file_spec;
152
153 # This happens for the AUX hash in an io_spec like this:
154 # {CMP=> ['zy123utsrqponmlkji', {'@AUX@'=> undef}]},
155 defined $contents
156 or return $file_name;
157 }
158 else
159 {
160 die "$program_name: $test_name: invalid RHS in $type-spec\n"
161 }
162
163 my $is_junk_file = (! defined $file_name
164 || (($type eq 'IN' || $type eq 'AUX' || $type eq 'CMP')
165 && defined $contents));
166 my $file = _create_file ($program_name, $test_name,
167 $file_name, $contents,
168 $is_binmode); # bird
169
170 if ($is_junk_file)
171 {
172 push @$junk_files, $file
173 }
174 else
175 {
176 # FIXME: put $srcdir in here somewhere
177 warn "$program_name: $test_name: specified file `$file' does"
178 . " not exist\n"
179 if ! -f "$srcdir/$file";
180 }
181
182 return $file;
183}
184
185sub _at_replace ($$)
186{
187 my ($map, $s) = @_;
188 foreach my $eo (qw (AUX OUT ERR))
189 {
190 my $f = $map->{$eo};
191 $f
192 and $s =~ /\@$eo\@/
193 and $s =~ s/\@$eo\@/$f/g;
194 }
195 return $s;
196}
197
198# FIXME: cleanup on interrupt
199# FIXME: extract `do_1_test' function
200
201# FIXME: having to include $program_name here is an expedient kludge.
202# Library code doesn't `die'.
203sub run_tests ($$$$$)
204{
205 my ($program_name, $prog, $t_spec, $save_temps, $verbose) = @_;
206
207 # Warn about empty t_spec.
208 # FIXME
209
210 # Remove all temp files upon interrupt.
211 # FIXME
212
213 # Verify that test names are distinct.
214 my $bad_test_name = 0;
215 my %seen;
216 my %seen_8dot3;
217 my $t;
218 foreach $t (@$t_spec)
219 {
220 my $test_name = $t->[0];
221 if ($seen{$test_name})
222 {
223 warn "$program_name: $test_name: duplicate test name\n";
224 $bad_test_name = 1;
225 }
226 $seen{$test_name} = 1;
227
228 if (0)
229 {
230 my $t8 = lc substr $test_name, 0, 8;
231 if ($seen_8dot3{$t8})
232 {
233 warn "$program_name: 8.3 test name conflict: "
234 . "$test_name, $seen_8dot3{$t8}\n";
235 $bad_test_name = 1;
236 }
237 $seen_8dot3{$t8} = $test_name;
238 }
239
240 # The test name may be no longer than 12 bytes,
241 # so that we can add a two-byte suffix without exceeding
242 # the maximum of 14 imposed on some old file systems.
243 if (14 < (length $test_name) + 2)
244 {
245 warn "$program_name: $test_name: test name is too long (> 12)\n";
246 $bad_test_name = 1;
247 }
248 }
249 return 1 if $bad_test_name;
250
251 # FIXME check exit status
252 system ($prog, '--version') if $verbose;
253
254 my @junk_files;
255 my $fail = 0;
256 foreach $t (@$t_spec)
257 {
258 my @post_compare;
259 my $test_name = shift @$t;
260 my $expect = {};
261 my ($pre, $post);
262
263 # FIXME: maybe don't reset this.
264 $Global_count = 1;
265 my @args;
266 my $io_spec;
267 my %seen_type;
268 my @env_delete;
269 my $env_prefix = '';
270 foreach $io_spec (@$t)
271 {
272 if (!ref $io_spec)
273 {
274 push @args, $io_spec;
275 next;
276 }
277
278 die "$program_name: $test_name: invalid test spec\n"
279 if ref $io_spec ne 'HASH';
280
281 my $n = keys %$io_spec;
282 die "$program_name: $test_name: spec has $n elements --"
283 . " expected 1\n"
284 if $n != 1;
285 my ($type, $val) = each %$io_spec;
286 die "$program_name: $test_name: invalid key `$type' in test spec\n"
287 if ! $Types{$type};
288
289 # remove _BINMODE # bird
290 my $is_binmode = $type =~ s/_BINMODE$//; # bird
291 $type =~ s/_BINMODE//; # bird
292
293 # Make sure there's no more than one of OUT, ERR, EXIT, etc.
294 die "$program_name: $test_name: more than one $type spec\n"
295 if $Zero_one_type{$type} and $seen_type{$type}++;
296
297 if ($type eq 'PRE' or $type eq 'POST')
298 {
299 $expect->{$type} = $val;
300 next;
301 }
302
303 if ($type eq 'CMP')
304 {
305 my $t = ref $val;
306 $t && $t eq 'ARRAY'
307 or die "$program_name: $test_name: invalid CMP spec\n";
308 @$val == 2
309 or die "$program_name: $test_name: invalid CMP list; must have"
310 . " exactly 2 elements\n";
311 my @cmp_files;
312 foreach my $e (@$val)
313 {
314 my $r = ref $e;
315 $r && $r ne 'HASH'
316 and die "$program_name: $test_name: invalid element ($r)"
317 . " in CMP list; only scalars and hash references "
318 . "are allowed\n";
319 if ($r && $r eq 'HASH')
320 {
321 my $n = keys %$e;
322 $n == 1
323 or die "$program_name: $test_name: CMP spec has $n "
324 . "elements -- expected 1\n";
325
326 # Replace any `@AUX@' in the key of %$e.
327 my ($ff, $val) = each %$e;
328 my $new_ff = _at_replace $expect, $ff;
329 if ($new_ff ne $ff)
330 {
331 $e->{$new_ff} = $val;
332 delete $e->{$ff};
333 }
334 }
335 my $cmp_file = _process_file_spec ($program_name, $test_name,
336 $e, $type, \@junk_files,
337 $is_binmode); # bird
338 push @cmp_files, $cmp_file;
339 }
340 push @post_compare, [@cmp_files];
341
342 $expect->{$type} = $val;
343 next;
344 }
345
346 if ($type eq 'EXIT')
347 {
348 die "$program_name: $test_name: invalid EXIT code\n"
349 if $val !~ /^\d+$/;
350 # FIXME: make sure $data is numeric
351 $expect->{EXIT} = $val;
352 next;
353 }
354
355 if ($type =~ /^(OUT|ERR)_SUBST$/)
356 {
357 $expect->{RESULT_SUBST} ||= {};
358 $expect->{RESULT_SUBST}->{$1} = $val;
359 next;
360 }
361
362 if ($type eq 'ENV')
363 {
364 $env_prefix = "$val ";
365 next;
366 }
367
368 if ($type eq 'ENV_DEL')
369 {
370 push @env_delete, $val;
371 next;
372 }
373
374 my $file = _process_file_spec ($program_name, $test_name, $val,
375 $type, \@junk_files,
376 $is_binmode); # bird
377
378 if ($type eq 'IN')
379 {
380 push @args, _shell_quote $file;
381 }
382 elsif ($type eq 'AUX' || $type eq 'OUT' || $type eq 'ERR')
383 {
384 $expect->{$type} = $file;
385 }
386 else
387 {
388 die "$program_name: $test_name: invalid type: $type\n"
389 }
390 }
391
392 # Expect an exit status of zero if it's not specified.
393 $expect->{EXIT} ||= 0;
394
395 # Allow ERR to be omitted -- in that case, expect no error output.
396 foreach my $eo (qw (OUT ERR))
397 {
398 if (!exists $expect->{$eo})
399 {
400 $expect->{$eo} = _create_file ($program_name, $test_name,
401 undef, '', 0); # bird
402 push @junk_files, $expect->{$eo};
403 }
404 }
405
406 # FIXME: Does it ever make sense to specify a filename *and* contents
407 # in OUT or ERR spec?
408
409 # FIXME: this is really suboptimal...
410 my @new_args;
411 foreach my $a (@args)
412 {
413 $a = _at_replace $expect, $a;
414 push @new_args, $a;
415 }
416 @args = @new_args;
417
418 warn "$test_name...\n" if $verbose;
419 &{$expect->{PRE}} if $expect->{PRE};
420 my %actual;
421 $actual{OUT} = "$test_name.O";
422 $actual{ERR} = "$test_name.E";
423 push @junk_files, $actual{OUT}, $actual{ERR};
424 my @cmd = ($prog, @args, "> $actual{OUT}", "2> $actual{ERR}");
425 my $cmd_str = $env_prefix . join (' ', @cmd);
426
427 # Delete from the environment any symbols specified by syntax
428 # like this: {ENV_DEL => 'TZ'}.
429 my %pushed_env;
430 foreach my $env_sym (@env_delete)
431 {
432 my $val = delete $ENV{$env_sym};
433 defined $val
434 and $pushed_env{$env_sym} = $val;
435 }
436
437 warn "Running command: `$cmd_str'\n" if $debug;
438 my $rc = 0xffff & system $cmd_str;
439
440 # Restore any environment setting we changed via a deletion.
441 foreach my $env_sym (keys %pushed_env)
442 {
443 $ENV{$env_sym} = $pushed_env{$env_sym};
444 }
445
446 if ($rc == 0xff00)
447 {
448 warn "$program_name: test $test_name failed: command failed:\n"
449 . " `$cmd_str': $!\n";
450 $fail = 1;
451 goto cleanup;
452 }
453 $rc >>= 8 if $rc > 0x80;
454 if ($expect->{EXIT} != $rc)
455 {
456 warn "$program_name: test $test_name failed: exit status mismatch:"
457 . " expected $expect->{EXIT}, got $rc\n";
458 $fail = 1;
459 goto cleanup;
460 }
461
462 foreach my $eo (qw (OUT ERR))
463 {
464 my $subst_expr = $expect->{RESULT_SUBST}->{$eo};
465 if (defined $subst_expr)
466 {
467 my $out = $actual{$eo};
468 my $orig = "$out.orig";
469
470 # Move $out aside (to $orig), then then recreate $out
471 # by transforming each line of $orig via $subst_expr.
472 unlink $orig; # bird - emx hack
473 rename $out, $orig
474 or (warn "$program_name: cannot rename $out to $orig: $!\n"),
475 $fail = 1, next;
476 open IN, $orig
477 or (warn "$program_name: cannot open $orig for reading: $!\n"),
478 $fail = 1, (unlink $orig), next;
479 if (0) { # bird - emx hack
480 unlink $orig
481 or (warn "$program_name: cannot unlink $orig: $!\n"),
482 $fail = 1;
483 }
484 open OUT, ">$out"
485 or (warn "$program_name: cannot open $out for writing: $!\n"),
486 $fail = 1, next;
487 while (defined (my $line = <IN>))
488 {
489 eval "\$_ = \$line; $subst_expr; \$line = \$_";
490 print OUT $line;
491 }
492 close IN;
493 close OUT
494 or (warn "$program_name: failed to write $out: $!\n"),
495 $fail = 1, next;
496 if (1) { # bird - emx hack
497 unlink $orig;
498 }
499 }
500
501 my $eo_lower = lc $eo;
502 _compare_files ($program_name, $test_name, $eo_lower,
503 $actual{$eo}, $expect->{$eo})
504 and $fail = 1;
505 }
506
507 foreach my $pair (@post_compare)
508 {
509 my ($expected, $actual) = @$pair;
510 _compare_files $program_name, $test_name, undef, $actual, $expected
511 and $fail = 1;
512 }
513
514 cleanup:
515 &{$expect->{POST}} if $expect->{POST};
516
517 }
518
519 # FIXME: maybe unlink files inside the big foreach loop?
520 unlink @junk_files if ! $save_temps;
521
522 return $fail;
523}
524
525## package return
5261;
Note: See TracBrowser for help on using the repository browser.