ghidra/gradle/root/test.gradle

734 lines
26 KiB
Groovy

/*********************************************************************************
* test.gradle
*
* Contains tasks for running unit/integration tests and generating associated
* reports.
*
* Testing tasks: unitTestReport
* integrationTestReport
*
*
*********************************************************************************/
/*********************************************************************************
* Define some vars. Note that the first few vars must be defined. If they
* aren't set by a calling script, we default them.
*
*********************************************************************************/
apply from: "gradle/support/testUtils.gradle"
configurations {
jmockitAgent
}
dependencies {
jmockitAgent('org.jmockit:jmockit:1.44') {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
}
}
if (!project.hasProperty('rootTestDir')) {
project.ext.rootTestDir = "build"
}
if (!project.hasProperty('machineName')) {
try {
project.ext.machineName = InetAddress.localHost.hostName
}
catch (UnknownHostException e) {
project.ext.machineName = "STANDALONE"
}
}
if (!project.hasProperty('xms')) {
project.ext.xms = "512M"
}
if (!project.hasProperty('xmx')) {
project.ext.xmx = "2048M"
}
loadApplicationProperties()
loadMachineSpecificProperties()
if (!project.hasProperty('srcTreeName')) {
project.ext.srcTreeName = System.properties.get("application.version")
}
project.ext.testRootDirName = "JunitTest_" + srcTreeName
project.ext.pcodeTestRootDirName = "PCodeTest_" + srcTreeName
project.ext.shareDir = System.properties.get('share.dir')
if (project.ext.shareDir != null) {
// add src tree name to machine specified share path
project.ext.testShareDir = shareDir + "/" + testRootDirName
project.ext.pcodeTestShareDir = shareDir + "/" + pcodeTestRootDirName
}
else {
project.ext.testShareDir = "$rootTestDir" + "/JUnit"
project.ext.pcodeTestShareDir = "$rootTestDir" + "/PCodeTest"
}
project.ext.upgradeProgramErrorMessage = "WARNING! Attempting data upgrade for "
project.ext.upgradeTimeErrorMessage = "Upgrade Time (ms): "
project.ext.logPropertiesUrl = getLogFileUrl()
// Specify final report directory via cmd line using "-PreportDir=<location>". This is useful for
// the 'parallelCombinedTestReport' task.
project.ext.reportDir = project.hasProperty('reportDir') ? project.getProperty('reportDir') + "/reports"
: testShareDir + "/reports"
project.ext.reportArchivesDir = testShareDir + "/reportsArchive"
project.ext.archivedReportDir = reportArchivesDir + "/reports"
project.ext.testOutputDir = file(rootTestDir).getAbsolutePath() + "/output"
project.ext.userHome = System.properties.get("user.home")
// 'parallelMode' will change configs to allow concurrent test execution.
// Enable this mode if 'parallelCombinedTestReport' invoked in the command line.
project.ext.parallelMode = project.gradle.startParameter.taskNames.contains('parallelCombinedTestReport')
project.ext.targetMode = project.gradle.startParameter.taskNames.contains('targetedTestReport')
project.ext.targetFiles = targetMode ? project.getProperty('targetFiles') : null
project.ext.targetProjects = targetMode ? getTestProjects(targetFiles) : null
// 'parallelCombinedTestReport' task will parse a JUnit test report for test duration.
// Specify the location of the report via cmd line using "-PtestTimeParserInputDir=<location>".
// If the location is not specified via cmd line, it will look for the latest test report
// in <reportDir>/../reportsArchive/reports_<date>
// If a reportsArchive path does not exist, it will assume all tests have the same duration and
// continue with test task creation.
project.ext.testTimeParserInputDir = project.hasProperty('testTimeParserInputDir') ?
project.getProperty('testTimeParserInputDir') + "/reports/classes" : getLastArchivedReport("$reportArchivesDir") + "/classes"
// A port of 0 will allow the kernel to give a free port number in the set of "User"
// or "Registered" Ports (usually 1024 - 49151). This will prevent port collisions among
// concurrent JUnit tests.
project.ext.debugPort = parallelMode ? 0 : generateRandomDebugPort()
/*********************************************************************************
* Create specified absolute directory if it does not exist
*********************************************************************************/
def createDir(dirPath) {
println "Creating directory: " + dirPath
File dir = new File(dirPath);
dir.mkdirs();
if (!dir.exists()) {
throw new RuntimeException("Failed to create required directory: " + dirPath);
}
if (!dir.canWrite()) {
throw new RuntimeException("Process does not have write permission for directory: " + dirPath);
}
}
/*********************************************************************************
* Retrieves log4j properties file and returns its path.
*********************************************************************************/
def getLogFileUrl() {
String logFilePath = "$rootDir/Ghidra/Framework/Generic/src/main/resources/generic.log4jtest.xml"
File logFile = new File(logFilePath)
if (logFile.exists()) {
return logFile.toURI().toString()
}
throw new GradleException("Cannot find log4j properties file using path '$logFilePath'")
}
/**
* Given a set of files (passed in via the -PtargetFiles command line param), this
* will search the project tree for each files' package to see if there is a test
* class defined that can be run.
*
* The param is a list of file paths created by 'git diff --names-only'
*/
def List<Project> getTestProjects(String files) {
List<Project> projects = new ArrayList<>();
List paths = files.tokenize('%')
for (String path : paths) {
File file = new File(path) // might blow up?
// Find the project for this file
def ps
while (file.parentFile != null) {
ps = rootProject.allprojects.findAll { it.projectDir == file }
if (ps) {
ps.each {
projects.add(it)
}
break;
}
file = file.parentFile
}
}
return projects
}
/*********************************************************************************
* Creates a new debug port randomly.
*********************************************************************************/
def generateRandomDebugPort() {
// keep port within narrow range (NOTE: must be greater than 1024 and less than 65535)
// Generate random byte value
Random random = new Random()
def ran = random.nextInt();
for (int i = 1; i < 4; i++) {
ran ^= (ran >> 8);
}
ran &= 0xff;
// Generate random port between 18300 and 18555
def port = 18300 + ran
return port
}
/*********************************************************************************
* Loads application specific property file that contains info we need.
* Properties will be immediately added to the global System.properties file so we
* can readily access them from just one place.
*********************************************************************************/
def loadApplicationProperties() {
Properties props = new Properties()
File appProperties = new File("Ghidra/application.properties");
if (!appProperties.exists()) {
return;
}
props.load(new FileInputStream(appProperties));
props.each {k, v ->
System.setProperty(k, v)
}
}
/*********************************************************************************
* Loads any machine specific property file that contains info we need.
* Only those properties with our machine name prefix will be immediately
* added to the global System.properties file so we can readily access them from
* just one place (machine name prefix will be omitted).
*********************************************************************************/
def loadMachineSpecificProperties() {
if (project.hasProperty('testPropertiesPath')) {
Properties props = new Properties()
def testPropertiesPath = project.getProperty('testPropertiesPath')
File testProperties = new File(testPropertiesPath);
if (!testProperties.exists()) {
return;
}
println "loadMachineSpecificProperties: Using machine specific properties file '$testProperties'"
props.load(new FileInputStream(testProperties));
// Note: Only load those properties that contain our machine name (set above). This means
// that local test runs will not use these values. Also, if the test machine name
// changes, then so too will have to change the properties file.
props.each { k, v ->
if (k.startsWith(machineName)) {
// strip off <machine-name>_ prefix from property key
def key = k.substring(machineName.length()+1)
println "loadMachineSpecificProperties: Setting system property $key:$v"
System.setProperty(key, v)
}
}
}
}
/*********************************************************************************
* Archive previously generated test report
*********************************************************************************/
task archiveTestReport {
File reports = file(reportDir)
File archivedReports = file(archivedReportDir)
File reportArchives = file(reportArchivesDir)
onlyIf {
containsIndexFile(reports)
}
doLast {
reportArchives.mkdirs()
delete archivedReports
reports.renameTo(archivedReports)
renameReportArchive(archivedReports) // renames archived reports directory
generateArchiveIndex(reportArchives)
}
}
/*********************************************************************************
* Remove remnants of previous tests.
*********************************************************************************/
task deleteTestTempAndReportDirs() {
doFirst {
delete file(rootTestDir).getAbsolutePath()
delete file(reportDir).getAbsolutePath()
}
}
/*********************************************************************************
* Initialize test task
*********************************************************************************/
def initTestJVM(Task task, String rootDirName) {
def testTempDir = file(rootTestDir).getAbsolutePath()
def testReportDir = file(reportDir).getAbsolutePath()
task.doFirst {
println "Test Machine Name: " + machineName
println "Root Test Dir: " + rootTestDir
println "Test Output Dir: " + testOutputDir
println "Test Temp Dir: " + testTempDir
println "Test Report Dir: " + testReportDir
println "Java Debug Port: " + debugPort
createDir(testTempDir)
createDir(testOutputDir)
}
// If false, testing will halt when an error is found.
task.ignoreFailures true
// If false, then tests are re-run every time, even if no code has changed.
task.outputs.upToDateWhen {false}
// Must set this to see System.out print statements.
task.testLogging.showStandardStreams = true
// Min/Max heap size. These are passed in.
task.minHeapSize xms
task.maxHeapSize xmx
// for jmockit; needs the javaagent option
// -javaagent:/path/to/jmockit.jar
task.doFirst {
def jmockitPath = configurations.jmockitAgent.singleFile
task.jvmArgs '-DupgradeProgramErrorMessage=' + upgradeProgramErrorMessage,
'-DupgradeTimeErrorMessage=' + upgradeTimeErrorMessage,
'-Dlog4j.configuration=' + logPropertiesUrl,
'-Dghidra.test.property.batch.mode=true',
'-Dghidra.test.property.parallel.mode=' + parallelMode,
'-Dghidra.test.property.output.dir=' + testOutputDir,
'-Dghidra.test.property.report.dir=' + testReportDir,
'-DSystemUtilities.isTesting=true',
'-Dmachine.name=' + machineName,
'-Djava.io.tmpdir=' + testTempDir,
'-Duser.data.dir=' + userHome + '/.ghidra/.ghidra-' + srcTreeName + '-Test',
'-Dcpu.core.override=8',
'-XX:ParallelGCThreads=8',
'-XX:+UseParallelGC',
'-Djava.awt.headless=false',
'-DDecompilerFunctionAnalyzer.enabled=false', // prevent long-winded analysis when testing (see DecompilerFunctionAnalyzer)
'-Dfile.encoding=UTF8',
'-Duser.country=US',
'-Duser.language=en',
'-Djdk.attach.allowAttachSelf',
'-javaagent:' + jmockitPath,
'-DLock.DEBUG=true',
'-Xdebug',
'-Xnoagent',
'-Djava.compiler=NONE',
'-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=' + debugPort
}
}
/*********************************************************************************
* Create this TestReport task so that individual test tasks may later assign themselves
* (using 'reportOn'). These tasks run tests concurrently.
*
* Invoking this task via command line enables 'parallelMode'.
*
* Example:
* ./gradlew parallelCombinedTestReport -PtestTimeParserInputDir=<path/to/report/classes> -PreportDir=<path/to/reports>
*
*********************************************************************************/
task parallelCombinedTestReport(type: TestReport) { t ->
group "test"
destinationDir = file("$reportDir")
logger.debug("parallelCombinedTestReport: Using destinationDir = $reportDir")
dependsOn ":deleteTestTempAndReportDirs"
mustRunAfter ":deleteTestTempAndReportDirs"
}
/*********************************************************************************
* UNIT TEST REPORT
*
* Summary: Runs all unit tests and generates a single report.
*
*********************************************************************************/
task unitTestReport(type: TestReport) { t ->
group "test"
description "Run unit tests and save HTML report."
destinationDir = file("$reportDir/unitTests")
outputs.upToDateWhen {false}
}
/*********************************************************************************
* INTEGRATION TEST REPORT
*
* Summary: Runs all integration tests and generates a single report.
*
*********************************************************************************/
task integrationTestReport(type: TestReport) { t ->
group "test"
description "Run integration tests and save HTML report."
destinationDir = file("$reportDir/integrationTests")
outputs.upToDateWhen {false}
}
/*********************************************************************************
* TARGETED TEST REPORT
*
* Summary: Runs unit tests against a set of projects
*
* The projects to be run against are stored in the project.ext.targetProjects
* var, which is populated at the top of this file. This is currently only
* used by the runAllTests.sh script which executes this task on each commit,
* so we can run all the tests for the files contained in that commit.
*
*********************************************************************************/
task targetedTestReport(type: TestReport) { t ->
group "test"
description "Run unit tests against a specific set of projects."
destinationDir = file("$reportDir/unitTests")
outputs.upToDateWhen {false}
dependsOn ":deleteTestTempAndReportDirs"
// This is a bit tricky but the tasks for each project are not yet defined
// at this point, so we have to put this in an afterEvaluate block before
// we can set up the reportOn dependency
project.ext.targetProjects.each {
it.afterEvaluate { p ->
if (p.tasks.findByName('test')) {
reportOn p.test
}
}
}
}
/*********************************************************************************
* Create a task of type: Test for a subproject.
* Each task will run a maximum of 'numMaxParallelForks' tests at a time.
* The task will include tests from classesList[classesListPosition, classesListPosition+numMaxParallelForks]
* and inherit excludes from its non-parallel counterpart.
*
* @param subproject
* @param testType - Either 'test' or 'integrationTest'. The sourceSets defined in setupJava.gradle
* @param bucketName - the test category (appConfig, dockingConfig, integrationConfig, etc...)
* @param taskNameCounter - Number to append to task name.
* @param classesList - List of classes for this subproject.testType. Sorted by duration.
* @param classesListPosition - Start position in 'classesList'
* @param numMaxParallelForks - Number of concurrent tests to run.
*********************************************************************************/
def createTestTask(Project subproject, String testType, String bucketName, int taskNameCounter, ArrayList classesList,
int classesListPosition, int numMaxParallelForks) {
subproject.task(testType+"_$taskNameCounter"+"_$bucketName", type: Test) { t ->
group "test"
testClassesDirs = files subproject.sourceSets["$testType"].output.classesDirs
classpath = subproject.sourceSets["$testType"].runtimeClasspath
maxParallelForks = numMaxParallelForks
initTestJVM(t, testRootDirName)
rootProject.tasks['parallelCombinedTestReport'].reportOn t
// include classes in task up to the maxParallelForks value
for (int i = 0; (i < maxParallelForks && classesListPosition < classesList.size); i++) {
// Convert my.package.MyClass to my/package/MyClass.class
include classesList[classesListPosition].replace(".","/") + ".class"
classesListPosition++
}
// Inherit excludes from subproject build.gradle.
// These excludes will override any includes.
excludes = subproject.tasks["$testType"].excludes
// Display granularity of 0 will log method-level events
// as "Task > Worker > org.SomeClass > org.someMethod".
testLogging {
displayGranularity 0
events "started", "passed", "skipped", "failed"
}
// Current task mustRunAfter previous task (test_1 before test_2,etc.)
if (taskNameCounter > 1) {
def prevTaskName = "$testType" + "_" + (taskNameCounter -1) + "_$bucketName"
mustRunAfter prevTaskName
}
logger.info("createTestTask: Created $subproject.name" + ":$testType" + "_$taskNameCounter" + "_$bucketName"
+ "\n\tincludes = \n" + t.getIncludes() + "\n\tinheriting excludes (overrides any 'includes') = \n"
+ t.excludes)
} // end task
}
/*********************************************************************************
* Split each subproject's integrationTest ("src/test.slow") and test ("src/test")
* task into multiple tasks, based on duration parsed from a previous JUnit report.
* Each task will run tests concurrently.
*
* Each task will run a maximum of the 'maxParallelForks' value at a time.
*********************************************************************************/
configure(subprojects.findAll {parallelMode == true}) { subproject ->
afterEvaluate {
if (!shouldSkipTestTaskCreation(subproject)) {
logger.info("parallelCombinedTestReport: Creating 'test' tasks for " + subproject.name + " subproject.")
Map<String,List> testMap = getTestsForSubProject(subproject.sourceSets.test.java)
for (Map.Entry<String,List> classMap : testMap.entrySet()) {
String bucketName = classMap.getKey();
int classesListPosition = 0 // current position in classesList
int taskNameCounter = 1 // task suffix
int numMaxParallelForks = 40 // unit tests are fast; 40 seems to be a reasonable number
List<String> tests = classMap.getValue();
while (classesListPosition < tests.size()) {
createTestTask(subproject, "test", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks)
classesListPosition+=numMaxParallelForks
taskNameCounter+=1; // "test_1_appConfig", "test_2_appConfig, etc.
}
}
}
if (!shouldSkipIntegrationTestTaskCreation(subproject)) {
logger.info("parallelCombinedTestReport: Creating 'integrationTest' tasks for " + subproject.name + " subproject.")
Map<String,List> testMap = getTestsForSubProject(subproject.sourceSets.integrationTest.java)
for (Map.Entry<String,List> classMap : testMap.entrySet()) {
String bucketName = classMap.getKey();
int classesListPosition = 0 // current position in classesList
int taskNameCounter = 1 // task suffix
// Through trial-and-error we found that 40 is too many
// concurrent integration tests (ghidratest server has 40 CPUs).
// 20 seems like a good balance of throughput vs resource usage for ghidratest server.
int numMaxParallelForks = 20
List<String> tests = classMap.getValue();
while (classesListPosition < tests.size()) {
createTestTask(subproject, "integrationTest", bucketName, taskNameCounter, tests, classesListPosition, numMaxParallelForks)
classesListPosition+=numMaxParallelForks
taskNameCounter+=1; // "integrationTest_1_appConfig", "integrationTest_2_appConfig, etc.
}
}
}
} // end afterEvaluate
}// end subprojects
/*********************************************************************************
* COMBINED TEST REPORT
*
* Summary: Runs all integration and unit tests, and creates a single report.
*
*********************************************************************************/
task combinedTestReport(type: TestReport) { t ->
group "test"
destinationDir = file("$reportDir")
dependsOn ":deleteTestTempAndReportDirs"
mustRunAfter ":deleteTestTempAndReportDirs"
}
/*********************************************************************************
* P-CODE TEST REPORT
*
* Summary: Runs all P-Code Tests
*
*********************************************************************************/
task pcodeTestReport(type: TestReport) { t ->
group "pcodeTest"
destinationDir = file("$pcodeTestShareDir" + "/reports")
}
/*********************************************************************************
* P-CODE TEST REPORT w/ MATRIX REPORT
*
* Summary: Runs all P-Code Tests and consolidates JUnit and Matrix report in
* results directory.
*
*********************************************************************************/
task pcodeConsolidatedTestReport(type: Copy) {
group "pcodeTest"
dependsOn ':pcodeTestReport'
into (pcodeTestShareDir + "/reports")
from (testOutputDir + "/test-output") {
exclude '**/*.xml', 'cache/**'
}
}
/*********************************************************************************
* Rename archived report directory to reflect lastModified date
*********************************************************************************/
def renameReportArchive(dir) {
long modifiedTime = dir.lastModified()
Calendar calendar = new GregorianCalendar()
calendar.setTimeInMillis( modifiedTime )
java.text.SimpleDateFormat dateFormat = new java.text.SimpleDateFormat("_MMM_d_yyyy")
String newFilename = dir.getName() + dateFormat.format(calendar.getTime())
File newFile = new File( dir.getParent(), newFilename )
newFile = getUniqueName( newFile )
dir.renameTo( newFile )
println "Archived reports: " + newFile.getCanonicalPath()
}
/*********************************************************************************
* Attempt to generate unique directory/file name to avoid duplication
*********************************************************************************/
def getUniqueName(file) {
if (!file.exists()) {
return file
}
for (int i = 2; i < 22; i++ ) {
String newName = file.getName() + "_" + i
File newFile = new File( file.getParentFile(), newName )
if (!newFile.exists()) {
return newFile
}
}
return file
}
/*********************************************************************************
* Generate report archive index.html file within the archivesDir directory
*********************************************************************************/
def generateArchiveIndex(archivesDir) {
File file = new File( archivesDir, "index.html" );
println "Generating Archived Reports Index: " + file.getCanonicalPath()
PrintWriter p = new PrintWriter(file);
try {
p.println("<HTML>");
p.println("<HEAD>");
printStyleSheet(p);
p.println("</HEAD>");
p.println("<BODY>");
p.println("<CENTER>");
p.println("<H2>Past Test Reports</H2>");
p.println(" <TABLE BORDER=1 ALIGN=CENTER WIDTH=\"90%\">");
// get a dir listing of all the files from the report dir and
// create a hyperlink for the filename
File[] reportDirFiles = archivesDir.listFiles();
Arrays.sort(reportDirFiles, new Comparator<File>() {
public int compare(File file1, File file2) {
if ( file1 == file2 ) {
return 0;
}
long timestamp1 = file1.lastModified();
long timestamp2 = file2.lastModified();
if (timestamp1 == timestamp2) {
return file1.getName().compareTo(file2.getName());
}
return (timestamp1 < timestamp2) ? 1 : -1;
}
});
// archived reports older than one month will be removed
long now = System.currentTimeMillis();
println "Current time: " + now
long oldestTime = now - (1000L * 60 * 60 * 24 * 30);
println "Delete date: " + oldestTime
for (File f : reportDirFiles) {
if (!f.isDirectory()) {
continue;
}
println "File: " + f.toString()
println "\tlast modified: " + f.lastModified()
if (f.lastModified() < oldestTime) {
println "\t\tFile older than the oldest time--deleting!..."
f.deleteDir(); // delete old archive results
}
else {
println "\t\tFile is a spring chicken--creating link..."
createLinkForDir(p, f);
}
}
p.println(" </TABLE>");
p.println("</CENTER>");
p.println("</BODY>");
p.println("</HTML>");
}
finally {
p.close();
}
}
/*********************************************************************************
* Add HTML code to PrintWriter p which provides link to specified file which
* is assumed to reside within the same directory as the index.html file being written
*********************************************************************************/
def createLinkForDir(p, file) {
// skip files - dirs only
if ( !file.isDirectory() ) {
return;
}
String columnPadding = " "
p.println( columnPadding + "<TR><TD>" )
String dirName = file.getName()
if (containsIndexFile(file)) {
String linkText = "<a href=\"./" + dirName + "/index.html\">" + dirName + "</a>"
p.println( columnPadding + linkText);
}
else {
p.println( columnPadding + dirName);
}
p.println( columnPadding + "</TD></TR>" )
}
/*********************************************************************************
* Returns true if the specified directory File contains an index.html file
*********************************************************************************/
def containsIndexFile(dir) {
File indexFile = new File(dir, "index.html");
return indexFile.exists();
}
/*********************************************************************************
* Add HTML code to PrintWriter p which provides the STYLE tag
*********************************************************************************/
def printStyleSheet(p) {
p.println( "<style>" );
p.println( "<!--" );
p.println( "" );
p.println( "-->" );
p.println( "</style>" );
}