Skip to content

Commit 0cd4c90

Browse files
tenzaptenzap
committed
more refinement to autotest classes
Properly manage when the CLI errors out (instead of test failure) Write also the outcome of CLI tests to result.html file Display more information in the console, esp. the errors.
1 parent 74ef256 commit 0cd4c90

File tree

2 files changed

+181
-17
lines changed

2 files changed

+181
-17
lines changed

‎autotest/AutoTest.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,23 +85,29 @@ public static void main(String[] args) {
8585
AutoTest parser = new AutoTest(instance);
8686
parser.parse(uri);
8787
if (parser.autoTestContentHandler.hasErrors()) {
88-
System.err.println("\tTest outcome:" +
88+
System.err.println("\tTests outcome:" +
8989
" " +
9090
parser.autoTestContentHandler.getTestSuccessCount() +
9191
" success(es)." +
9292
" " +
9393
parser.autoTestContentHandler.getTestFailCount() +
94-
" failure(s)."
94+
" failure(s)." +
95+
" " +
96+
parser.autoTestContentHandler.getTestErrorCount() +
97+
" error(s)."
9598
);
9699
System.exit(1);
97100
} else {
98-
System.out.println("\tTest outcome:" +
101+
System.out.println("\tTests outcome:" +
99102
" " +
100103
parser.autoTestContentHandler.getTestSuccessCount() +
101104
" success(es)." +
102105
" " +
103106
parser.autoTestContentHandler.getTestFailCount() +
104-
" failure(s)."
107+
" failure(s)." +
108+
" " +
109+
parser.autoTestContentHandler.getTestErrorCount() +
110+
" error(s)."
105111
);
106112
}
107113
} catch (Throwable t) {

‎autotest/AutoTestContentHandler.java‎

Lines changed: 171 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@
1616
import org.xml.sax.SAXException;
1717

1818
import java.io.BufferedWriter;
19+
import java.io.BufferedReader;
1920
import java.io.File;
2021
import java.io.FileWriter;
2122
import java.io.IOException;
2223
import java.io.InputStream;
24+
import java.io.InputStreamReader;
25+
import java.lang.StringBuilder;
2326
import java.net.MalformedURLException;
2427
import java.net.URL;
2528
import java.util.ArrayList;
@@ -69,6 +72,8 @@ public class AutoTestContentHandler implements ContentHandler {
6972
boolean inWarnings = false;
7073
int testFailCount = 0;
7174
int testSuccessCount = 0;
75+
int testErrorCount = 0; // Errors while trying to run the test
76+
boolean hasError = false;
7277
String urlString = "";
7378
String file = "";
7479
String desc = "";
@@ -78,6 +83,7 @@ public class AutoTestContentHandler implements ContentHandler {
7883
String warning;
7984
String medium;
8085
String testInstance = "servlet";
86+
StringBuilder errorSb;
8187

8288
/**
8389
* Default Constructor.
@@ -198,7 +204,13 @@ public void startElement(String nameSpaceURI, String localName,
198204
desc = "";
199205
result = new Result();
200206

201-
warning = null;
207+
// Set default value of warning to Zero, because
208+
// - if the GET request to the servlet doesn't define a warning, it will be 0 (as per the
209+
// ApplContext class default values).
210+
// - on the contrary, the default value for the warning of the CLI is 2.
211+
// So we have to set he default value to 0 here, so that when the warning is not defined
212+
// it means 0. This is required to harmonize test result between call to jar and call to servlet.
213+
warning = "0";
202214
profile = null;
203215
medium = null;
204216
for (int i = 0; i < attributs.getLength(); i++) {
@@ -238,6 +250,61 @@ public void startElement(String nameSpaceURI, String localName,
238250
}
239251
}
240252

253+
private void waitProcess(Process p, List<String> command) {
254+
boolean waitForValue = false;
255+
try {
256+
waitForValue = p.waitFor(20,java.util.concurrent.TimeUnit.SECONDS);
257+
} catch (InterruptedException e) {
258+
hasError = true;
259+
errorSb.append("Request: ");
260+
errorSb.append(command.toString());
261+
errorSb.append(System.getProperty("line.separator"));
262+
errorSb.append("Timeout reached. Subprocess stopped.");
263+
errorSb.append(System.getProperty("line.separator"));
264+
errorSb.append(e.getStackTrace().toString());
265+
printError(command, "Timeout reached. Subprocess stopped.");
266+
printErrorToConsole();
267+
return;
268+
}
269+
if (waitForValue == true && p.exitValue() == 1) {
270+
hasError = true;
271+
errorSb.append("Request: ");
272+
errorSb.append(command.toString());
273+
errorSb.append(System.getProperty("line.separator"));
274+
errorSb.append("Command failed with exit code: " + p.exitValue());
275+
errorSb.append(System.getProperty("line.separator"));
276+
277+
StringBuilder cmdOutput = new StringBuilder();
278+
try {
279+
String line;
280+
BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
281+
while ((line = input.readLine()) != null) {
282+
cmdOutput.append(line);
283+
cmdOutput.append(System.getProperty("line.separator"));
284+
}
285+
input.close();
286+
BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream()));
287+
while ((line = error.readLine()) != null) {
288+
cmdOutput.append(line);
289+
cmdOutput.append(System.getProperty("line.separator"));
290+
}
291+
error.close();
292+
errorSb.append(cmdOutput);
293+
}
294+
catch (IOException e) {
295+
errorSb.append(e.getMessage());
296+
errorSb.append(System.getProperty("line.separator"));
297+
errorSb.append(e.getStackTrace().toString());
298+
printError(command, e.getMessage());
299+
printErrorToConsole();
300+
return;
301+
}
302+
printError(command, "Command failed with exit code: " + p.exitValue() + "<pre>" + cmdOutput + "</pre>");
303+
printErrorToConsole();
304+
//System.exit(p.exitValue());
305+
}
306+
}
307+
241308
/**
242309
* @see org.xml.sax.ContentHandler#endElement(java.lang.String,
243310
* java.lang.String, java.lang.String)
@@ -269,6 +336,8 @@ public void endElement(String nameSpaceURI, String localName, String rawName)
269336
System.err.println(e.getMessage());
270337
}
271338
} else if (element == TEST) {
339+
hasError = false;
340+
errorSb = new StringBuilder("");
272341
System.out.print(urlString + "... ");
273342
String validURL = createValidURL(urlString);
274343
String val;
@@ -294,7 +363,11 @@ public void endElement(String nameSpaceURI, String localName, String rawName)
294363

295364
if (warning != null) {
296365
val += "&warning=" + warning;
297-
command.add("--warning=" + warning);
366+
if (warning.equals("no")) {
367+
command.add("--warning=-1");
368+
} else {
369+
command.add("--warning=" + warning);
370+
}
298371
}
299372
if (profile != null) {
300373
val += "&profile=" + profile;
@@ -332,12 +405,14 @@ public void endElement(String nameSpaceURI, String localName, String rawName)
332405
Map<String, String> env = pb.environment();
333406
env.put("CLASSPATH", env.get("CLASSPATH") + ":css-validator.jar");
334407
Process p = pb.start();
408+
waitProcess(p, command);
335409
res = p.getInputStream();
336410
} else if (testInstance.equals("cli")) {
337411
Runtime r = Runtime.getRuntime();
338412
command.add(0, "css-validator");
339413
ProcessBuilder pb = new ProcessBuilder(command);
340414
Process p = pb.start();
415+
waitProcess(p, command);
341416
res = p.getInputStream();
342417
} else {
343418
System.err.println("Unsupported operation. Invalid instance or instance not set: " + testInstance);
@@ -351,15 +426,26 @@ public void endElement(String nameSpaceURI, String localName, String rawName)
351426
}
352427

353428
if (testInstance.equals("servlet") && reply.getStatus() == 500) { // Internal Server Error
429+
hasError = true;
354430
if (buf.indexOf("env:Sender") != -1) {
355431
printError(val, "Reply status code: 500<br/>"
356432
+ "Invalid URL: Sender error");
433+
errorSb.append(val);
434+
errorSb.append(System.getProperty("line.separator"));
435+
errorSb.append("Reply status code: 500. Invalid URL: Sender error");
357436
} else if (buf.indexOf("env:Receiver") != -1) {
358437
printError(val, "Reply status code: 500<br/>"
359438
+ "Unreachable URL: Receiver error");
439+
errorSb.append(val);
440+
errorSb.append(System.getProperty("line.separator"));
441+
errorSb.append("Reply status code: 500. Unreachable URL: Receiver error");
360442
} else {
361443
printError(val, "Reply status code: 500");
444+
errorSb.append(val);
445+
errorSb.append(System.getProperty("line.separator"));
446+
errorSb.append("Reply status code: 500");
362447
}
448+
printErrorToConsole();
363449
} else {
364450
result = new Result();
365451
int begin = buf.indexOf("<m:validity>");
@@ -388,14 +474,42 @@ public void endElement(String nameSpaceURI, String localName, String rawName)
388474
}
389475

390476
} catch (MalformedURLException e) {
391-
printError(val, e.getMessage());
392-
printResultToConsole(urlString);
477+
if (hasError == false) {
478+
hasError = true;
479+
errorSb.append("Request: ");
480+
errorSb.append(testInstance.equals("servlet") ? truncateString(val) : command.toString());
481+
errorSb.append(System.getProperty("line.separator"));
482+
errorSb.append(truncateString(e.getMessage()));
483+
errorSb.append(System.getProperty("line.separator"));
484+
printError(val, e.getMessage());
485+
printErrorToConsole();
486+
}
393487
} catch (IOException e) {
394-
printError(val, e.getMessage());
395-
printResultToConsole(urlString);
488+
if (hasError == false) {
489+
hasError = true;
490+
errorSb.append("Request: ");
491+
errorSb.append(testInstance.equals("servlet") ? truncateString(val) : command.toString());
492+
errorSb.append(System.getProperty("line.separator"));
493+
errorSb.append(truncateString(e.getMessage()));
494+
errorSb.append(System.getProperty("line.separator"));
495+
printError(val, e.getMessage());
496+
printErrorToConsole();
497+
}
396498
} catch (HttpException e) {
397-
printError(val, e.getMessage());
398-
printResultToConsole(urlString);
499+
if (hasError == false) {
500+
hasError = true;
501+
errorSb.append("Request: ");
502+
errorSb.append(testInstance.equals("servlet") ? truncateString(val) : command.toString());
503+
errorSb.append(System.getProperty("line.separator"));
504+
errorSb.append(truncateString(e.getMessage()));
505+
errorSb.append(System.getProperty("line.separator"));
506+
printError(val, e.getMessage());
507+
printErrorToConsole();
508+
}
509+
}
510+
511+
if (hasError == true) {
512+
testErrorCount++;
399513
}
400514

401515
isFile = false;
@@ -412,6 +526,16 @@ public void endElement(String nameSpaceURI, String localName, String rawName)
412526
}
413527
}
414528

529+
private String truncateString(String str) {
530+
int maxLength = 512;
531+
int tailLength = 100;
532+
if (str.length() <= maxLength) {
533+
return str;
534+
} else {
535+
return str.substring(0, maxLength) + "...[TRUNCATED]..." + str.substring(str.length()-tailLength, str.length()) + "[TRUNCATED]";
536+
}
537+
}
538+
415539
/**
416540
* @see org.xml.sax.ContentHandler#characters(char[], int, int)
417541
*/
@@ -526,7 +650,6 @@ private boolean isErrorsEqual() {
526650
* the validator page result
527651
*/
528652
private void printResultToConsole(String urlString) {
529-
530653
if (isValidEqual() && isWarningsEqual() && isErrorsEqual()) {
531654
testSuccessCount++;
532655
System.out.println(" Success");
@@ -556,18 +679,49 @@ private void printResultToConsole(String urlString) {
556679
private void printError(String validatorPage, String message) {
557680

558681
validatorPage = validatorPage.replaceAll("&", "&amp;");
559-
urlString = urlString.replaceAll("&", "&amp;");
682+
String urlString2 = urlString.replaceAll("&", "&amp;");
560683

561684
print(" <div class=\"error\">");
562-
print(" <h3><a href=\"" + urlString + "\">"
563-
+ urlString + "</a></h3>");
685+
print(" <h3><a href=\"" + urlString2 + "\">"
686+
+ urlString2 + "</a></h3>");
564687
print(" <p><a href=\"" + validatorPage
565688
+ "\">Go to the Validator page</a></p>");
566689
print(" <p>" + desc + "</p>");
690+
print(" <p>" + truncateString(message) + "</p>");
691+
print(" </div>");
692+
}
693+
694+
/**
695+
* Used when an error occurs
696+
*
697+
* @param validatorPage
698+
* the validator page result
699+
* @param message
700+
* the message to be displayed
701+
*/
702+
private void printError(List<String> command, String message) {
703+
704+
String urlString2 = urlString.replaceAll("&", "&amp;");
705+
706+
print(" <div class=\"error\">");
707+
print(" <h3><a href=\"" + urlString2 + "\">"
708+
+ urlString2 + "</a></h3>");
709+
print(" <p>Command: " + command + "</p>");
710+
print(" <p>" + desc + "</p>");
567711
print(" <p>" + message + "</p>");
568712
print(" </div>");
569713
}
570714

715+
/**
716+
* Used when an error occurs. Prints to console.
717+
*
718+
*/
719+
private void printErrorToConsole() {
720+
System.out.println(" \u001B[31mError\u001B[0m");
721+
System.err.println(urlString.indent(4));
722+
System.err.println(errorSb.toString().indent(4)); // String.indent() requires java >= 12.
723+
}
724+
571725
/**
572726
* Replaces all URL special chars in a String with their matching URL
573727
* entities
@@ -611,7 +765,7 @@ public String createValidURL(String url) {
611765
}
612766

613767
public boolean hasErrors() {
614-
if (testFailCount > 0) {
768+
if (testFailCount > 0 || testErrorCount > 0) {
615769
return true;
616770
}
617771
return false;
@@ -625,4 +779,8 @@ public int getTestSuccessCount() {
625779
return testSuccessCount;
626780
}
627781

782+
public int getTestErrorCount() {
783+
return testErrorCount;
784+
}
785+
628786
}

0 commit comments

Comments
 (0)