diff --git a/cli/src/main/kotlin/com/malinskiy/marathon/cli/config/ConfigFactory.kt b/cli/src/main/kotlin/com/malinskiy/marathon/cli/config/ConfigFactory.kt index 9d15acc7f..36e4be671 100644 --- a/cli/src/main/kotlin/com/malinskiy/marathon/cli/config/ConfigFactory.kt +++ b/cli/src/main/kotlin/com/malinskiy/marathon/cli/config/ConfigFactory.kt @@ -60,7 +60,7 @@ class ConfigFactory(val mapper: ObjectMapper) { config.debug, vendorConfiguration, config.analyticsTracking - ) + ).also { println(it) } } private fun readConfigFile(configFile: File): FileConfiguration? { diff --git a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt index 652c03787..381364fe1 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/Marathon.kt @@ -8,6 +8,7 @@ import com.malinskiy.marathon.device.DeviceProvider import com.malinskiy.marathon.exceptions.NoDevicesException import com.malinskiy.marathon.execution.Configuration import com.malinskiy.marathon.execution.Scheduler +import com.malinskiy.marathon.execution.TestPackageFilter import com.malinskiy.marathon.execution.TestParser import com.malinskiy.marathon.execution.TestShard import com.malinskiy.marathon.execution.progress.ProgressReporter diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/TrackerFactory.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/TrackerFactory.kt index 819ed62cd..aaf71cfa1 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/TrackerFactory.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/TrackerFactory.kt @@ -4,6 +4,7 @@ import com.google.gson.Gson import com.malinskiy.marathon.analytics.tracker.local.DeviceTracker import com.malinskiy.marathon.analytics.tracker.local.JUnitTracker import com.malinskiy.marathon.analytics.tracker.local.RawTestResultTracker +import com.malinskiy.marathon.analytics.tracker.local.TestCoverageTracker import com.malinskiy.marathon.analytics.tracker.local.TestResultsTracker import com.malinskiy.marathon.analytics.tracker.remote.influx.InfluxDbProvider import com.malinskiy.marathon.analytics.tracker.remote.influx.InfluxDbTracker @@ -12,6 +13,7 @@ import com.malinskiy.marathon.execution.Configuration import com.malinskiy.marathon.io.FileManager import com.malinskiy.marathon.report.allure.AllureTestListener import com.malinskiy.marathon.report.internal.DeviceInfoReporter +import com.malinskiy.marathon.report.internal.TestCoverageReporter import com.malinskiy.marathon.report.internal.TestResultReporter import com.malinskiy.marathon.report.junit.JUnitReporter import java.io.File @@ -26,12 +28,13 @@ internal class TrackerFactory(private val configuration: Configuration, val allureTracker = AllureTestListener(configuration, File(configuration.outputDir, "allure-results")) fun create(): Tracker { - val defaultTrackers = listOf( - JUnitTracker(JUnitReporter(fileManager)), - DeviceTracker(deviceInfoReporter), - TestResultsTracker(testResultReporter), - rawTestResultTracker, - allureTracker + val defaultTrackers = listOfNotNull( + JUnitTracker(JUnitReporter(fileManager)), + DeviceTracker(deviceInfoReporter), + TestResultsTracker(testResultReporter), + rawTestResultTracker, + allureTracker, + if (configuration.isCodeCoverageEnabled) TestCoverageTracker(TestCoverageReporter(fileManager)) else null ) return when { configuration.analyticsConfiguration is InfluxDbConfiguration -> { diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/local/TestCoverageTracker.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/local/TestCoverageTracker.kt new file mode 100644 index 000000000..717832224 --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/local/TestCoverageTracker.kt @@ -0,0 +1,19 @@ +package com.malinskiy.marathon.analytics.tracker.local + +import com.malinskiy.marathon.analytics.tracker.NoOpTracker +import com.malinskiy.marathon.device.DeviceInfo +import com.malinskiy.marathon.device.DevicePoolId +import com.malinskiy.marathon.execution.TestResult +import com.malinskiy.marathon.report.internal.TestCoverageReporter + +internal class TestCoverageTracker(private val testCoverageReporter: TestCoverageReporter) : + NoOpTracker() { + + override fun trackTestFinished( + poolId: DevicePoolId, + device: DeviceInfo, + testResult: TestResult + ) { + testCoverageReporter.testFinished(poolId, device, testResult) + } +} diff --git a/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/remote/influx/InfluxDbTracker.kt b/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/remote/influx/InfluxDbTracker.kt index 03b1d284a..89ad213c7 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/remote/influx/InfluxDbTracker.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/analytics/tracker/remote/influx/InfluxDbTracker.kt @@ -5,6 +5,7 @@ import com.malinskiy.marathon.device.DeviceInfo import com.malinskiy.marathon.device.DevicePoolId import com.malinskiy.marathon.execution.TestResult import com.malinskiy.marathon.execution.TestStatus +import com.malinskiy.marathon.log.MarathonLogging import com.malinskiy.marathon.test.toSafeTestName import org.influxdb.InfluxDB import org.influxdb.dto.Point @@ -12,18 +13,27 @@ import java.util.concurrent.TimeUnit internal class InfluxDbTracker(private val influxDb: InfluxDB) : NoOpTracker() { + private val logger = MarathonLogging.logger("InfluxDB") + override fun trackRawTestRun(poolId: DevicePoolId, device: DeviceInfo, testResult: TestResult) { - influxDb.write(Point.measurement("tests") - .time(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - .tag("testname", testResult.test.toSafeTestName()) - .tag("package", testResult.test.pkg) - .tag("class", testResult.test.clazz) - .tag("method", testResult.test.method) - .tag("deviceSerial", device.serialNumber) - .addField("ignored", if (testResult.isIgnored) 1.0 else 0.0) - .addField("success", if (testResult.status == TestStatus.PASSED) 1.0 else 0.0) - .addField("duration", testResult.durationMillis()) - .build()) + val point = Point.measurement("tests") + .time(System.currentTimeMillis(), TimeUnit.MILLISECONDS) + .tag("testname", testResult.test.toSafeTestName()) + .tag("package", testResult.test.pkg) + .tag("class", testResult.test.clazz) + .tag("method", testResult.test.method) + .tag("deviceSerial", device.serialNumber) + .addField("ignored", if (testResult.isIgnored) 1.0 else 0.0) + .addField("success", if (testResult.status == TestStatus.PASSED) 1.0 else 0.0) + .addField("duration", testResult.durationMillis()) + .build() + runCatching { + influxDb.write(point) + }.onSuccess { + logger.trace { "Tracked in influxDB $point" } + }.onFailure { + logger.error(it) { "Error writing to influxDB with $point" } + } } override fun terminate() { diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/Attachment.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/Attachment.kt index 4208e8999..be556fe05 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/execution/Attachment.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/Attachment.kt @@ -7,11 +7,13 @@ data class Attachment(val file: File, val type: AttachmentType) enum class AttachmentType { SCREENSHOT, VIDEO, + COVERAGE, LOG; fun toMimeType() = when(this) { SCREENSHOT -> "image/gif" VIDEO -> "video/mp4" + COVERAGE -> "application/ec" LOG -> "text/txt" } } \ No newline at end of file diff --git a/core/src/main/kotlin/com/malinskiy/marathon/io/FileType.kt b/core/src/main/kotlin/com/malinskiy/marathon/io/FileType.kt index 03e0bb82a..ab3803b2e 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/io/FileType.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/io/FileType.kt @@ -7,6 +7,6 @@ enum class FileType(val dir: String, val suffix: String) { LOG("logs", "log"), DEVICE_INFO("devices", "json"), VIDEO("video", "mp4"), - SCREENSHOT("screenshot", "gif") - + SCREENSHOT("screenshot", "gif"), + COVERAGE("coverage", "ec") } diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/internal/TestCoverageReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/internal/TestCoverageReporter.kt new file mode 100644 index 000000000..06355080d --- /dev/null +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/internal/TestCoverageReporter.kt @@ -0,0 +1,34 @@ +package com.malinskiy.marathon.report.internal + +import com.malinskiy.marathon.device.DeviceInfo +import com.malinskiy.marathon.device.DevicePoolId +import com.malinskiy.marathon.execution.TestResult +import com.malinskiy.marathon.io.FileManager +import com.malinskiy.marathon.io.FileType +import com.malinskiy.marathon.log.MarathonLogging +import com.malinskiy.marathon.test.toTestName + +class TestCoverageReporter(private val fileManager: FileManager) { + + private val logger = MarathonLogging.logger("TestCoverageReporter") + + fun testFinished(poolId: DevicePoolId, device: DeviceInfo, testResult: TestResult) { + testResult.attachments + .onEach { logger.debug { "Attachments on test ${testResult.test.toTestName()} ${it.file.name}" } } + .filter { it.file.name.endsWith(".ec") } + .forEach { + val file = + fileManager.createFile(FileType.COVERAGE, poolId, device, testResult.test) + file.setWritable(true) + logger.debug { "Coverage file is ${it.file.name}" } + fileCounter++ + val bytes = it.file.readBytes() + file.appendBytes(bytes) + } + logger.debug { "Total coverage file generated so far $fileCounter" } + } + + companion object { + private var fileCounter = 0 + } +} \ No newline at end of file diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDevice.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDevice.kt index c03326326..e1af66739 100644 --- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDevice.kt +++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDevice.kt @@ -11,6 +11,7 @@ import com.malinskiy.marathon.android.executor.listeners.DebugTestRunListener import com.malinskiy.marathon.android.executor.listeners.LogCatListener import com.malinskiy.marathon.android.executor.listeners.NoOpTestRunListener import com.malinskiy.marathon.android.executor.listeners.ProgressTestRunListener +import com.malinskiy.marathon.android.executor.listeners.TestCoverageResultListener import com.malinskiy.marathon.android.executor.listeners.TestRunResultsListener import com.malinskiy.marathon.android.executor.listeners.screenshot.ScreenCapturerTestRunListener import com.malinskiy.marathon.android.executor.listeners.video.ScreenRecorderTestRunListener @@ -36,10 +37,13 @@ import kotlinx.coroutines.newFixedThreadPoolContext import java.util.* import kotlin.coroutines.CoroutineContext -class AndroidDevice(val ddmsDevice: IDevice, - private val serialStrategy: SerialStrategy = SerialStrategy.AUTOMATIC) : Device, CoroutineScope { +class AndroidDevice( + val ddmsDevice: IDevice, + private val serialStrategy: SerialStrategy = SerialStrategy.AUTOMATIC, + packageName: String = "" +) : Device, CoroutineScope { - val fileManager = RemoteFileManager(ddmsDevice) + val fileManager = RemoteFileManager(ddmsDevice, packageName) private val dispatcher by lazy { newFixedThreadPoolContext(1, "AndroidDevice - execution - ${ddmsDevice.serialNumber}") @@ -164,14 +168,17 @@ class AndroidDevice(val ddmsDevice: IDevice, val timer = SystemTimer() + val coverageListener = TestCoverageResultListener(fileManager, devicePoolId, this) + return CompositeTestRunListener( - listOf( - recorderListener, - logCatListener, - TestRunResultsListener(testBatch, this, deferred, timer, attachmentProviders), - DebugTestRunListener(this), - ProgressTestRunListener(this, devicePoolId, progressReporter) - ) + listOfNotNull( + recorderListener, + logCatListener, + TestRunResultsListener(testBatch, this, deferred, timer, attachmentProviders), + DebugTestRunListener(this), + ProgressTestRunListener(this, devicePoolId, progressReporter), + if (configuration.isCodeCoverageEnabled) coverageListener else null + ) ) } diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDeviceProvider.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDeviceProvider.kt index 3d3ff1833..dfeb4202c 100644 --- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDeviceProvider.kt +++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/AndroidDeviceProvider.kt @@ -42,6 +42,12 @@ class AndroidDeviceProvider : DeviceProvider, CoroutineScope { if (vendorConfiguration !is AndroidConfiguration) { throw IllegalStateException("Invalid configuration $vendorConfiguration passed") } + val packageName = ApkParser().parseInstrumentationInfo( + vendorConfiguration.testApplicationOutput + ).applicationPackage + + logger.debug { "applicationPackage name is $packageName" } + DdmPreferences.setTimeOut(DEFAULT_DDM_LIB_TIMEOUT) AndroidDebugBridge.initIfNeeded(false) @@ -51,7 +57,11 @@ class AndroidDeviceProvider : DeviceProvider, CoroutineScope { override fun deviceChanged(device: IDevice?, changeMask: Int) { device?.let { launch(context = bootWaitContext) { - val maybeNewAndroidDevice = AndroidDevice(it, vendorConfiguration.serialStrategy) + val maybeNewAndroidDevice = AndroidDevice( + it, + vendorConfiguration.serialStrategy, + packageName + ) val healthy = maybeNewAndroidDevice.healthy logger.debug { "Device ${device.serialNumber} changed state. Healthy = $healthy" } @@ -70,7 +80,7 @@ class AndroidDeviceProvider : DeviceProvider, CoroutineScope { override fun deviceConnected(device: IDevice?) { device?.let { launch { - val maybeNewAndroidDevice = AndroidDevice(it, vendorConfiguration.serialStrategy) + val maybeNewAndroidDevice = AndroidDevice(it, vendorConfiguration.serialStrategy, packageName) val healthy = maybeNewAndroidDevice.healthy logger.debug { "Device ${maybeNewAndroidDevice.serialNumber} connected. Healthy = $healthy" } diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/RemoteFileManager.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/RemoteFileManager.kt index f8aa26c9e..b4b11f077 100644 --- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/RemoteFileManager.kt +++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/RemoteFileManager.kt @@ -8,15 +8,20 @@ import com.android.ddmlib.ShellCommandUnresponsiveException import com.android.ddmlib.TimeoutException import com.android.ddmlib.testrunner.TestIdentifier import com.malinskiy.marathon.log.MarathonLogging - +import java.io.File import java.io.IOException -class RemoteFileManager(private val device: IDevice) { +class RemoteFileManager( + private val device: IDevice, + packageName: String = "com.agoda.mobile.consumer.debug" +) { private val logger = MarathonLogging.logger("RemoteFileManager") private val outputDir = device.getMountPoint(MNT_EXTERNAL_STORAGE) + private val filesDir = "/data/data/$packageName/files" + private val nullOutputReceiver = NullOutputReceiver() fun removeRemotePath(remotePath: String) { @@ -53,6 +58,32 @@ class RemoteFileManager(private val device: IDevice) { return remoteFileForTest(videoFileName(test)) } + fun remoteCoverageForTest(): String { + logger.debug { "Directories accessible in Devices ---------------------->" } + logger.debug { "$outputDir" } + File(outputDir).listAllFilesInTheDirectory() + logger.debug { "$filesDir" } + File(filesDir).listAllFilesInTheDirectory() + logger.debug { "data folder" } + File("/data").listAllFilesInTheDirectory() + logger.debug { "storage folder" } + File("/storage").listAllFilesInTheDirectory() + logger.debug { "All files listed ------------------------------>" } + return "/sdcard/coverage.ec" + } + + private fun File.listAllFilesInTheDirectory() { + if (isDirectory) { + listFiles().forEach { + it.listAllFilesInTheDirectory() + } + } else { + if (name.contains("coverage")) { + logger.debug { "$absolutePath" } + } + } + } + private fun remoteFileForTest(filename: String): String { return "$outputDir/$filename" } diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/AndroidDeviceTestRunner.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/AndroidDeviceTestRunner.kt index 7c8dc2d1d..c900f258e 100644 --- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/AndroidDeviceTestRunner.kt +++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/AndroidDeviceTestRunner.kt @@ -50,8 +50,6 @@ class AndroidDeviceTestRunner(private val device: AndroidDevice) { val androidConfiguration = configuration.vendorConfiguration as AndroidConfiguration val info = ApkParser().parseInstrumentationInfo(androidConfiguration.testApplicationOutput) val runner = prepareTestRunner(configuration, androidConfiguration, info, testBatch) - - try { clearData(androidConfiguration, info) notifyIgnoredTest(ignoredTests, listener) @@ -126,6 +124,9 @@ class AndroidDeviceTestRunner(private val device: AndroidDevice) { runner.setMaxTimeout(batchTimeout, TimeUnit.MILLISECONDS) runner.setClassNames(tests) + androidConfiguration.instrumentationArgs.toMutableMap().apply { + put("coverage", "true") + } androidConfiguration.instrumentationArgs.forEach { (key, value) -> runner.addInstrumentationArg(key, value) } diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestCoverageResultListener.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestCoverageResultListener.kt new file mode 100644 index 000000000..53b434d00 --- /dev/null +++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestCoverageResultListener.kt @@ -0,0 +1,73 @@ +package com.malinskiy.marathon.android.executor.listeners + +import com.android.ddmlib.testrunner.TestIdentifier +import com.malinskiy.marathon.android.AndroidDevice +import com.malinskiy.marathon.android.toTest +import com.malinskiy.marathon.device.DevicePoolId +import com.malinskiy.marathon.device.toDeviceInfo +import com.malinskiy.marathon.execution.Attachment +import com.malinskiy.marathon.execution.AttachmentType +import com.malinskiy.marathon.io.FileManager +import com.malinskiy.marathon.io.FileType +import com.malinskiy.marathon.log.MarathonLogging +import com.malinskiy.marathon.report.attachment.AttachmentListener +import com.malinskiy.marathon.report.attachment.AttachmentProvider +import kotlin.system.measureTimeMillis + +class TestCoverageResultListener( + private val fileManager: FileManager, + private val pool: DevicePoolId, + private val device: AndroidDevice +) : NoOpTestRunListener(), AttachmentProvider { + + private val logger = MarathonLogging.logger("CoverageListener") + + private val attachmentListeners = mutableListOf() + + private lateinit var lastTestIdentifier: TestIdentifier + + override fun testEnded(test: TestIdentifier, testMetrics: Map) { + super.testEnded(test, testMetrics) + lastTestIdentifier = test + } + + override fun testRunEnded(elapsedTime: Long, runMetrics: Map) { + super.testRunEnded(elapsedTime, runMetrics) + logger.debug { "checking coverage files on device" } + runCatching { + pullCoverageFile(lastTestIdentifier) + }.onSuccess { + removeCoverageFile() + } + } + + private fun pullCoverageFile(test: TestIdentifier) { + val localCoverageFile = + fileManager.createFile(FileType.COVERAGE, pool, device.toDeviceInfo(), test.toTest()) + logger.debug { "local coverage file ${localCoverageFile.name}" } + val remoteFilePath = device.fileManager.remoteCoverageForTest() + logger.debug { "Pulling from $remoteFilePath" } + val millis = measureTimeMillis { + device.fileManager.pullFile(remoteFilePath, localCoverageFile.toString()) + } + logger.debug { "Pulling file finished in ${millis}ms $remoteFilePath " } + attachmentListeners.forEach { + it.onAttachment( + test.toTest(), + Attachment(localCoverageFile, AttachmentType.COVERAGE) + ) + } + } + + private fun removeCoverageFile() { + val remoteFilePath = device.fileManager.remoteCoverageForTest() + val millis = measureTimeMillis { + device.fileManager.removeRemotePath(remoteFilePath) + } + logger.debug { "Removed file in ${millis}ms $remoteFilePath" } + } + + override fun registerListener(listener: AttachmentListener) { + attachmentListeners.add(listener) + } +} \ No newline at end of file diff --git a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsListener.kt b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsListener.kt index 535d59107..f37b91a10 100644 --- a/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsListener.kt +++ b/vendor/vendor-android/src/main/kotlin/com/malinskiy/marathon/android/executor/listeners/TestRunResultsListener.kt @@ -46,6 +46,7 @@ class TestRunResultsListener( } attachments[test]!!.add(attachment) + logger.debug { "attachment added ${attachment.file.name} ${attachment.type}" } } private val logger = MarathonLogging.logger("TestRunResultsListener")