of_unittest_expect 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408
  1. #!/usr/bin/perl
  2. # SPDX-License-Identifier: GPL-2.0
  3. #
  4. # Copyright 2020, 2022 Sony Corporation
  5. #
  6. # Author: Frank Rowand
  7. # This program is meant to be an aid to reading the verbose output of
  8. # on the console log that results from executing the Linux kernel
  9. # devicetree unittest (drivers/of/unitest.c).
  10. $VUFX = "220201a";
  11. use strict 'refs';
  12. use strict subs;
  13. use Getopt::Long;
  14. use Text::Wrap;
  15. # strip off everything before final "/"
  16. (undef, $script_name) = split(/^.*\//, $0);
  17. # following /usr/include/sysexits.h
  18. $EX_OK=0;
  19. $EX_USAGE=64;
  20. #______________________________________________________________________________
  21. sub compare {
  22. my ($expect, $got) = @_;
  23. my $expect_next;
  24. my $expect_next_lit;
  25. my $got_next;
  26. my $type;
  27. while ($expect) {
  28. ($expect_next, $type) = split(/<</, $expect);
  29. ($type) = split(/>>/, $type);
  30. $expect =~ s/^.*?>>//; # '?' is non-greedy, minimal match
  31. # literal, ignore all metacharacters when used in a regex
  32. $expect_next_lit = quotemeta($expect_next);
  33. $got_next = $got;
  34. $got_next =~ s/^($expect_next_lit).*/\1/;
  35. $got =~ s/^$expect_next_lit//;
  36. if ($expect_next ne $got_next) {
  37. return 0;
  38. }
  39. if ($type eq "int") {
  40. if ($got =~ /^[+-]*[0-9]+/) {
  41. $got =~ s/^[+-]*[0-9]+//;
  42. } else {
  43. return 0;
  44. }
  45. } elsif ($type eq "hex") {
  46. if ($got =~ /^(0x)*[0-9a-f]+/) {
  47. $got =~ s/^(0x)*[0-9a-f]+//;
  48. } else {
  49. return 0;
  50. }
  51. } elsif ($type eq "") {
  52. if ($expect_next ne $got_next) {
  53. return 0;
  54. } else {
  55. return 1;
  56. }
  57. } else {
  58. $internal_err++;
  59. print "** ERROR: special pattern not recognized: <<$type>>, CONSOLE_LOG line: $.\n";
  60. return 0;
  61. }
  62. }
  63. # should not get here
  64. $internal_err++;
  65. print "** ERROR: $script_name internal error, at end of compare(), CONSOLE_LOG line: $.\n";
  66. return 0;
  67. }
  68. #______________________________________________________________________________
  69. sub usage {
  70. # ***** when editing, be careful to not put tabs in the string printed:
  71. print STDERR
  72. "
  73. usage:
  74. $script_name CONSOLE_LOG
  75. -h print program usage
  76. --help print program usage
  77. --hide-expect suppress output of EXPECTed lines
  78. --line-num report line number of CONSOLE_LOG
  79. --no-expect-stats do not report EXPECT statistics
  80. --no-strip-ts do not strip leading console timestamps
  81. --verbose do not suppress EXPECT begin and end lines
  82. --version print program version and exit
  83. Process a console log for EXPECTed test related messages to either
  84. highlight expected devicetree unittest related messages or suppress
  85. the messages. Leading console timestamps will be stripped.
  86. Various unittests may trigger kernel messages from outside the
  87. unittest code. The unittest annotates that it expects the message
  88. to occur with an 'EXPECT \\ : text' (begin) before triggering the
  89. message, and an 'EXPECT / : text' (end) after triggering the message.
  90. If an expected message does not occur, that will be reported.
  91. For each expected message, the 'EXPECT \\ : text' (begin) and
  92. 'EXPECT / : text' (end), 'text' will contain the message text.
  93. If 'EXPECT \\' (begin) and 'EXPECT /' (end) lines do not contain
  94. matching 'text', that will be reported.
  95. If EXPECT lines are nested, 'EXPECT /' (end) lines must be in the
  96. reverse order of the corresponding 'EXPECT \\' (begin) lines.
  97. 'EXPECT \\ : text' (begin) and 'EXPECT / : text' (end) lines can
  98. contain special patterns in 'text':
  99. <<int>> matches: [+-]*[0-9]+
  100. <<hex>> matches: (0x)*[0-9a-f]+
  101. 'EXPECT \\' (begin) and 'EXPECT /' (end) lines are suppressed.
  102. A prefix is added to every line of output:
  103. 'ok ' Line matches an enclosing EXPECT begin/end pair
  104. '** ' Line reports $script_name warning or error
  105. '-> ' Line reports start or end of the unittests
  106. '>> ' Line reports a unittest test FAIL
  107. ' ' Lines that are not otherwise prefixed
  108. Issues detected in CONSOLE_LOG are reported to STDOUT, not to STDERR.
  109. Known Issues:
  110. --line-num causes the CONSOLE_LOG line number to be printed in 4 columns.
  111. If CONSOLE_LOG contains more than 9999 lines then more columns will be
  112. used to report the line number for lines greater than 9999 (eg for
  113. lines 10000 - 99999, 5 columns will be used).
  114. ";
  115. return {};
  116. }
  117. #______________________________________________________________________________
  118. #______________________________________________________________________________
  119. if (!GetOptions(
  120. "h" => \$help,
  121. "help" => \$help,
  122. "hide-expect" => \$hide_expect,
  123. "line-num" => \$print_line_num,
  124. "no-expect-stats" => \$no_expect_stats,
  125. "no-strip-ts" => \$no_strip_ts,
  126. "verbose" => \$verbose,
  127. "version" => \$version,
  128. )) {
  129. print STDERR "\n";
  130. print STDERR "ERROR processing command line options\n";
  131. print STDERR "\n";
  132. print STDERR "For help, type '$script_name --help'\n";
  133. print STDERR "\n";
  134. exit $EX_OK;
  135. }
  136. if ($no_strip_ts) {
  137. $strip_ts = 1;
  138. $no_strip_ts = 0;
  139. } else {
  140. $strip_ts = 0;
  141. $no_strip_ts = 1;
  142. }
  143. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  144. if ($help){
  145. &usage;
  146. exit $EX_OK;
  147. }
  148. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  149. if ($version) {
  150. print STDERR "\n$script_name $VUFX\n\n";
  151. print STDERR "\n";
  152. exit $EX_OK;
  153. }
  154. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  155. if ($#ARGV != 0) {
  156. # Limit input files to exactly one.
  157. #
  158. # 'while ($line = <ARGV>) {' in the code below supports multiple file
  159. # names on the command line, but the EXPECT statistics are reported
  160. # once for all input - it is not an expected use case to generate one
  161. # set of statistics for multiple input files.
  162. print STDERR "\n";
  163. print STDERR "Required arguments: CONSOLE_LOG\n";
  164. print STDERR "\n";
  165. exit $EX_USAGE;
  166. }
  167. #______________________________________________________________________________
  168. # Patterns to match 'EXPECT \ : ' (begin) and 'EXPECT / : ' (end)
  169. #
  170. # $exp_* are used as regex match patterns,
  171. # so '\\\\' in $exp_begin matches a single '\'
  172. # quotemeta() does not do the right thing in this case
  173. #
  174. # $pr_fmt is the prefix that unittest prints for every message
  175. $pr_fmt = "### dt-test ### ";
  176. $exp_begin = "${pr_fmt}EXPECT \\\\ : ";
  177. $exp_end = "${pr_fmt}EXPECT / : ";
  178. $line_num = "";
  179. $timestamp = "";
  180. LINE:
  181. while ($line = <ARGV>) {
  182. chomp $line;
  183. $prefix = " "; ## 2 characters
  184. if ($strip_ts) {
  185. $timestamp = $line;
  186. if ($timestamp =~ /^\[\s*[0-9]+\.[0-9]*\] /) {
  187. ($timestamp, $null) = split(/]/, $line);
  188. $timestamp = $timestamp . "] ";
  189. } else {
  190. $timestamp = "";
  191. }
  192. }
  193. $line =~ s/^\[\s*[0-9]+\.[0-9]*\] //;
  194. # ----- find EXPECT begin
  195. if ($line =~ /^\s*$exp_begin/) {
  196. $data = $line;
  197. $data =~ s/^\s*$exp_begin//;
  198. push @begin, $data;
  199. if ($verbose) {
  200. if ($print_line_num) {
  201. $line_num = sprintf("%4s ", $.);
  202. }
  203. printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line;
  204. }
  205. next LINE;
  206. }
  207. # ----- find EXPECT end
  208. if ($line =~ /^\s*$exp_end/) {
  209. $data = $line;
  210. $data =~ s/^\s*$exp_end//;
  211. if ($verbose) {
  212. if ($print_line_num) {
  213. $line_num = sprintf("%4s ", $.);
  214. }
  215. printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line;
  216. }
  217. $found = 0;
  218. $no_begin = 0;
  219. if (@found_or_begin > 0) {
  220. $begin = pop @found_or_begin;
  221. if (compare($data, $begin)) {
  222. $found = 1;
  223. }
  224. } elsif (@begin > 0) {
  225. $begin = pop @begin;
  226. } else {
  227. $no_begin = 1;
  228. }
  229. if ($no_begin) {
  230. $expect_missing_begin++;
  231. print "** ERROR: EXPECT end without any EXPECT begin:\n";
  232. print " end ---> $line\n";
  233. } elsif (! $found) {
  234. if ($print_line_num) {
  235. $line_num = sprintf("%4s ", $.);
  236. }
  237. $expect_not_found++;
  238. printf "** %s%s$script_name WARNING - not found ---> %s\n",
  239. $line_num, $timestamp, $data;
  240. } elsif (! compare($data, $begin)) {
  241. $expect_missing_end++;
  242. print "** ERROR: EXPECT end does not match EXPECT begin:\n";
  243. print " begin -> $begin\n";
  244. print " end ---> $line\n";
  245. } else {
  246. $expect_found++;
  247. }
  248. next LINE;
  249. }
  250. # ----- not an EXPECT line
  251. if (($line =~ /^${pr_fmt}start of unittest - you will see error messages$/) ||
  252. ($line =~ /^${pr_fmt}end of unittest - [0-9]+ passed, [0-9]+ failed$/ ) ) {
  253. $prefix = "->"; # 2 characters
  254. } elsif ($line =~ /^${pr_fmt}FAIL /) {
  255. $unittest_fail++;
  256. $prefix = ">>"; # 2 characters
  257. }
  258. $found = 0;
  259. foreach $begin (@begin) {
  260. if (compare($begin, $line)) {
  261. $found = 1;
  262. last;
  263. }
  264. }
  265. if ($found) {
  266. $begin = shift @begin;
  267. while (! compare($begin, $line)) {
  268. push @found_or_begin, $begin;
  269. $begin = shift @begin;
  270. }
  271. push @found_or_begin, $line;
  272. if ($hide_expect) {
  273. $suppress_line = 1;
  274. next LINE;
  275. }
  276. $prefix = "ok"; # 2 characters
  277. }
  278. if ($print_line_num) {
  279. $line_num = sprintf("%4s ", $.);
  280. }
  281. printf "%s %s%s%s\n", $prefix, $line_num, $timestamp, $line;
  282. }
  283. if (! $no_expect_stats) {
  284. print "\n";
  285. print "** EXPECT statistics:\n";
  286. print "**\n";
  287. printf "** EXPECT found : %4i\n", $expect_found;
  288. printf "** EXPECT not found : %4i\n", $expect_not_found;
  289. printf "** missing EXPECT begin : %4i\n", $expect_missing_begin;
  290. printf "** missing EXPECT end : %4i\n", $expect_missing_end;
  291. printf "** unittest FAIL : %4i\n", $unittest_fail;
  292. printf "** internal error : %4i\n", $internal_err;
  293. }
  294. if (@begin) {
  295. print "** ERROR: EXPECT begin without any EXPECT end:\n";
  296. print " This list may be misleading.\n";
  297. foreach $begin (@begin) {
  298. print " begin ---> $begin\n";
  299. }
  300. }