Made test summary more descriptive and Updated Readme

This commit is contained in:
Sandeep Hegde 2017-11-13 20:40:16 -05:00
parent 2f900a0427
commit 61794f8bbd
4 changed files with 28 additions and 10 deletions

View file

@ -213,7 +213,7 @@ impl XMLHttpRequest {
}
fn sync_in_window(&self) -> bool {
self.sync.get() && self.global().is::<Window>()
self.sync.get() || self.global().is::<Window>()
}
fn initiate_async_xhr(context: Arc<Mutex<XHRContext>>,

View file

@ -61,3 +61,19 @@ The CI script for running mutation testing is present in /etc/ci folder. It can
4. The corresponding WPT tests are run for this mutant and the test results are logged.
5. Once all WPT are run for the first source file, the mutation continues for other source files mentioned in the json file and runs their corresponding WPT tests.
6. Once it has completed executing mutation testing for the entered path, it repeats the above procedure for sub-paths present inside the entered path.
### Test Summary
At the end of the test run the test summary displayed which looks like this:
```
Test Summary:
Mutant Killed (Success) 25
Mutant Survived (Failure) 10
Mutation Skipped 1
Unexpected error in mutation 0
```
* Mutant Killed (Success): The mutant was successfully killed by WPT test suite.
* Mutant Survived (Failure): The mutation has survived the WPT Test Suite, tests in WPT could not catch this mutation.
* Mutation Skipped: Files is skipped for mutation test due to the local changes in that file.
* Unexpected error in mutation: Mutation test could not run due to unexpected failures. (example: if no && preset in the file to replace)

View file

@ -13,8 +13,8 @@ import json
import sys
import test
test_summary = {
test.Status.PASSED: 0,
test.Status.FAILED: 0,
test.Status.KILLED: 0,
test.Status.SURVIVED: 0,
test.Status.SKIPPED: 0,
test.Status.UNEXPECTED: 0
}
@ -47,7 +47,9 @@ def mutation_test_for(mutation_path):
mutation_test_for(sys.argv[1])
print "\nTest Summary:"
for test_status in test_summary:
print "{0} : {1}".format(test_status.name, test_summary[test_status])
if test_summary[test.Status.FAILED] or test_summary[test.Status.UNEXPECTED]:
print "Mutant Killed (Success) \t{0}".format(test_summary[test.Status.KILLED])
print "Mutant Survived (Failure) \t{0}".format(test_summary[test.Status.SURVIVED])
print "Mutation Skipped \t\t{0}".format(test_summary[test.Status.SKIPPED])
print "Unexpected error in mutation \t{0}".format(test_summary[test.Status.UNEXPECTED])
if test_summary[test.Status.SURVIVED]:
sys.exit(1)

View file

@ -18,8 +18,8 @@ DEVNULL = open(os.devnull, 'wb')
class Status(Enum):
PASSED = 0
FAILED = 1
KILLED = 0
SURVIVED = 1
SKIPPED = 2
UNEXPECTED = 3
@ -63,10 +63,10 @@ def mutation_test(file_name, tests):
print "mutated file {0} diff".format(file_name)
sys.stdout.flush()
subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
status = Status.FAILED
status = Status.SURVIVED
else:
print("Success: Mutation killed by {0}".format(test.encode('utf-8')))
status = Status.PASSED
status = Status.KILLED
break
print "reverting mutant {0}:{1}".format(file_name, mutated_line)
sys.stdout.flush()