Skip to content

Commit d65dae2

Browse files
author
Deepa Dixit
committed
Bug#22818781: EXPLAIN_JSON_VALIDATE.TEST FAILS ON WINDOWS PRE-PUSH TESTING
Issue: ------ The explain_json_validate test calls a python script which accepts arguments. If the arguments are passed with wildcard characters, Windows cannot parse them and hence the test fails. Fix: ---- The fix involved adding an additional check if the test was running on Windows so that the arguments can be parsed correctly. Also, Python is not installed on Windows by default, and therefore the script is now ported to perl. Before the test used to be skipped if the machine did not have either Python or JSON support, but now it fails if it does not have JSON or perl support. Since the python script has been removed, all the tests which were using that script are now using the perl script. Reviewed-by: Horst Hunger <[email protected]> Reviewed-by: Guilhem Bichot <[email protected]> RB: 12537
1 parent 6123073 commit d65dae2

12 files changed

+171
-180
lines changed

‎mysql-test/include/explain_utils.inc

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -72,15 +72,11 @@ if ($innodb) {
7272
}
7373
}
7474
--eval EXPLAIN FORMAT=JSON $query;
75-
if ($validation) {
76-
--disable_query_log
77-
--replace_result $MASTER_MYSOCK MASTER_MYSOCK
75+
--disable_result_log
7876
--exec $MYSQL -S $MASTER_MYSOCK -u root -r test -e "EXPLAIN FORMAT=JSON $query;" > $MYSQLTEST_VARDIR/tmp/explain.json
79-
--replace_regex /[-]*// /FILE.[\/\\:_\.0-9A-Za-z]*/Validation:/
80-
--exec python $MYSQL_TEST_DIR/suite/opt_trace/validate_json.py $MYSQLTEST_VARDIR/tmp/explain.json
77+
--exec perl $MYSQL_TEST_DIR/suite/opt_trace/validate_json.pl $MYSQLTEST_VARDIR/tmp/explain.json
8178
--remove_file '$MYSQLTEST_VARDIR/tmp/explain.json'
82-
--enable_query_log
83-
}
79+
--enable_result_log
8480
}
8581

8682
if ($select) {
@@ -107,15 +103,11 @@ if ($innodb) {
107103
}
108104
}
109105
--eval EXPLAIN FORMAT=JSON $select;
110-
if ($validation) {
111-
--disable_query_log
112-
--replace_result $MASTER_MYSOCK MASTER_MYSOCK
106+
--disable_result_log
113107
--exec $MYSQL -S $MASTER_MYSOCK -u root -r test -e "EXPLAIN FORMAT=JSON $select;" > $MYSQLTEST_VARDIR/tmp/explain.json
114-
--replace_regex /[-]*// /FILE.[\/\\:_\.0-9A-Za-z]*/Validation:/
115-
--exec python $MYSQL_TEST_DIR/suite/opt_trace/validate_json.py $MYSQLTEST_VARDIR/tmp/explain.json
108+
--exec perl $MYSQL_TEST_DIR/suite/opt_trace/validate_json.pl $MYSQLTEST_VARDIR/tmp/explain.json
116109
--remove_file '$MYSQLTEST_VARDIR/tmp/explain.json'
117-
--enable_query_log
118-
}
110+
--enable_result_log
119111
}
120112
}
121113

‎mysql-test/include/python_with_json.inc

Lines changed: 0 additions & 25 deletions
This file was deleted.

‎mysql-test/suite/opt_trace/validate_json.pl

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
#!/usr/bin/perl
2+
use strict;
3+
use JSON;
4+
use File::Spec::Functions qw/ canonpath /;
5+
my $usage = "This is from WL#5257 \"first API for optimizer trace\".
6+
7+
Usage:
8+
%s [-q] <a_file> <another_file> <etc>
9+
10+
-q quiet mode: only display errors and warnings.
11+
12+
It will verify that all optimizer traces of files (usually a_file
13+
is a .result or .reject file which contains
14+
SELECT * FROM OPTIMIZER_TRACE; ) are JSON-compliant, and that
15+
they contain no duplicates keys.
16+
Exit code is 0 if all ok.";
17+
my $retcode = 0;
18+
my @ignored;
19+
my @input = @ARGV;
20+
21+
# Filter out "-q" options
22+
@input = grep {!/-q/} @input;
23+
24+
if (!@input)
25+
{
26+
print "$usage\n";
27+
exit 1;
28+
}
29+
30+
# If command line contains at least one "-q" option, it is quiet mode
31+
my $quiet= scalar(@input) <= scalar(@ARGV) -1;
32+
# On Windows, command line arguments specified using wildcards need to be evaluated.
33+
# On Unix too if the arguments are passed with single quotes.
34+
my $need_parse = grep(/\*/,@input);
35+
if ($need_parse)
36+
{
37+
my $platform_independent_dir;
38+
$platform_independent_dir= canonpath "@input";
39+
@input= glob "$platform_independent_dir";
40+
}
41+
42+
foreach my $input_file (@input)
43+
{
44+
handle_one_file($input_file);
45+
print "\n";
46+
}
47+
48+
if ( @ignored )
49+
{
50+
print STDERR "These files have been ignored:\n";
51+
foreach my $ig ( @ignored )
52+
{
53+
print "$ig\n";
54+
}
55+
print "\n";
56+
}
57+
if ( $retcode )
58+
{
59+
print STDERR "There are errors\n";
60+
}
61+
62+
else
63+
{
64+
print "\n";
65+
print "ALL OK\n";
66+
}
67+
68+
exit $retcode;
69+
70+
sub handle_one_file {
71+
72+
my ( $input_file ) = @_;
73+
if ( $input_file =~ /^.*(ctype_.*|mysqldump)\.result/ )
74+
{
75+
push @ignored ,$input_file;
76+
return;
77+
}
78+
print "FILE $input_file\n";
79+
print "\n";
80+
open(DATA,"<$input_file") or die "Can't open file";
81+
my @lines = <DATA>;
82+
close(DATA);
83+
my $first_trace_line = 0;
84+
my $trace_line = 0;
85+
my @trace = undef;
86+
label_to: foreach my $i ( @lines )
87+
{
88+
$trace_line = $trace_line + 1;
89+
if (( grep(/^.*(\t)?{\n/,$i) ) and ( $first_trace_line == 0 ))
90+
{
91+
@trace = undef;
92+
$first_trace_line = $trace_line;
93+
push @trace, "{\n";
94+
next label_to;
95+
}
96+
if (( $i =~ /^}/ ) and ( $first_trace_line != 0))
97+
{
98+
push @trace, "}";
99+
check($first_trace_line,@trace);
100+
$first_trace_line = 0;
101+
}
102+
if ( $first_trace_line != 0 )
103+
{
104+
# Eliminate /* */ from end_marker=on (not valid JSON)
105+
$i =~ s/\/\*.*\*\// /g;
106+
push @trace, $i;
107+
}
108+
109+
}
110+
}
111+
112+
113+
sub check {
114+
115+
my ( $first_trace_line, @trace ) = @_;
116+
my $string = join("", @trace);
117+
my $parsed;
118+
eval { $parsed = decode_json($string); };
119+
unless ( $parsed )
120+
{
121+
print "Parse error at line: $first_trace_line\n";
122+
my $error = $@;
123+
print "Error: $@\n";
124+
# If there is a character position specified, put a mark ('&') in front of this character
125+
if ($error =~ /invalid character.*at character offset (\d+)/)
126+
{
127+
substr($string,$1,0) = "&";
128+
print "$string\n";
129+
}
130+
else
131+
{
132+
print "$string\n";
133+
}
134+
$retcode = 1;
135+
print "\n";
136+
return;
137+
}
138+
# Detect non-unique keys in one object, by counting
139+
# number of quote symbols ("): the json module outputs only
140+
# one of the non-unique keys, making the number of "
141+
# smaller compared to the input string.
142+
143+
my $before = $string =~ tr/'"'//;
144+
my $re_json;
145+
$re_json= to_json($parsed);
146+
my $after = $re_json =~ tr/'"'//;
147+
if ( $before != $after )
148+
{
149+
print "Non-unique keys at line $first_trace_line ( $before vs $after )\n";
150+
print "$string\n";
151+
$retcode = 1;
152+
print "\n";
153+
return;
154+
}
155+
if ( !$quiet )
156+
{
157+
print "OK at line $first_trace_line\n";
158+
}
159+
}

‎mysql-test/suite/opt_trace/validate_json.py

Lines changed: 0 additions & 123 deletions
This file was deleted.

‎mysql-test/t/innodb_explain_json_non_select_all.test

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,10 @@ set @save_storage_engine= @@session.default_storage_engine;
2222
set session default_storage_engine = InnoDB;
2323
--let $innodb = 1
2424

25-
# Next 2 variables control the JSON format output and validation in explain_utils.
25+
# Next variable controls the JSON format output in explain_utils.
2626
# 1 = enable, 0 = disable
2727
--let $json = 1
28-
# Validation disabled due to not having Python with JSON on all PB machines.
29-
--let $validation = 0
30-
--file_exists $MYSQL_TEST_DIR/suite/opt_trace/validate_json.py
28+
--file_exists $MYSQL_TEST_DIR/suite/opt_trace/validate_json.pl
3129

3230
--source include/explain_non_select.inc
3331
set default_storage_engine= @save_storage_engine;

‎mysql-test/t/innodb_explain_json_non_select_none.test

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,9 @@ if (`select locate('mrr', @@optimizer_switch) > 0`)
3030
set @save_storage_engine= @@session.default_storage_engine;
3131
set session default_storage_engine = InnoDB;
3232
--let $innodb = 1
33-
# Next 2 variables control the JSON format output and validation in explain_utils.
33+
# Next variable controls the JSON format output in explain_utils.
3434
# 1 = enable, 0 = disable
3535
--let $json = 1
36-
# Validation disabled due to not having Python with JSON on all PB machines.
37-
--let $validation = 0
3836
--source include/explain_non_select.inc
3937
set default_storage_engine= @save_storage_engine;
4038

‎mysql-test/t/innodb_explain_non_select_all.test

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,8 @@ set @save_storage_engine= @@session.default_storage_engine;
2020
set session default_storage_engine = InnoDB;
2121
--let $innodb = 1
2222

23-
# json validation in explain_util.inc can be switched off by setting to zero.
23+
# json format in explain_util.inc can be switched off by setting to zero.
2424
--let $json = 0
25-
--let $validation = 0
2625

2726
--source include/explain_non_select.inc
2827
set default_storage_engine= @save_storage_engine;

‎mysql-test/t/innodb_explain_non_select_none.test

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ set @save_storage_engine= @@session.default_storage_engine;
2929
set session default_storage_engine = InnoDB;
3030
--let $innodb = 1
3131
--let $json = 0
32-
--let $validation = 0
3332
--source include/explain_non_select.inc
3433
set default_storage_engine= @save_storage_engine;
3534

0 commit comments

Comments
 (0)