Skip to content

Commit

Permalink
feat: Allow including binary files when auto build is enabled (#5299)
Browse files Browse the repository at this point in the history 
* feat: Allow including binary files when auto build is enabled

* attempt fixing test failures on windows

* fix windows tests
  • Loading branch information
@osdemah
osdemah authored Jan 27, 2025
1 parent e3f7adb commit ec81010
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 62 deletions.
67 changes: 51 additions & 16 deletions ...oftware/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import com.intellij.openapi.vfs.VirtualFile
import com.intellij.testFramework.RuleChain
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Before
Expand Down Expand Up @@ -72,7 +73,51 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
}

@Test
fun testZipProject() {
fun testZipProjectWithoutAutoDev() {
checkZipProject(
false,
setOf(
"src/MyClass.java",
"gradlew",
"gradlew.bat",
"README.md",
"gradle/wrapper/gradle-wrapper.properties",
"builder/GetTestBuilder.java",
"settings.gradle",
"build.gradle",
".gitignore",
)
)
}

@Test
fun testZipProjectWithAutoDev() {
checkZipProject(
true,
setOf(
"src/MyClass.java",
"icons/menu.svg",
"assets/header.jpg",
"gradle/wrapper/gradle-wrapper.jar",
"gradle/wrapper/gradle-wrapper.properties",
"images/logo.png",
"builder/GetTestBuilder.java",
"gradlew",
"README.md",
".gitignore",
"License.md",
"output.bin",
"archive.zip",
"gradlew.bat",
"license.txt",
"build.gradle",
"devfile.yaml",
"settings.gradle"
)
)
}

fun checkZipProject(autoBuildEnabled: Boolean, expectedFiles: Set<String>) {
addFilesToProjectModule(
".gitignore",
".gradle/cached.jar",
Expand All @@ -97,9 +142,12 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
"build/outputs",
"dist/bundle.js",
"gradle/wrapper/gradle-wrapper.jar",
"devfile.yaml",
)

val zipResult = featureDevSessionContext.getProjectZip(false)
projectRule.fixture.addFileToModule(module, "large-file.txt", "loblob".repeat(1024 * 1024))

val zipResult = featureDevSessionContext.getProjectZip(autoBuildEnabled)
val zipPath = zipResult.payload.path

val zippedFiles = mutableSetOf<String>()
Expand All @@ -111,19 +159,6 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
}
}

val expectedFiles = setOf(
"src/MyClass.java",
"gradlew",
"gradlew.bat",
"README.md",
"gradle/wrapper/gradle-wrapper.properties",
"builder/GetTestBuilder.java",
"settings.gradle",
"build.gradle",
"gradle/wrapper/gradle-wrapper.jar",
".gitignore",
)

assertTrue(zippedFiles == expectedFiles)
assertEquals(zippedFiles, expectedFiles)
}
}
97 changes: 52 additions & 45 deletions ...ommunity/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.VirtualFileVisitor
import com.intellij.openapi.vfs.isFile
import com.intellij.platform.ide.progress.withBackgroundProgress
import kotlinx.coroutines.async
import kotlinx.coroutines.flow.channelFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext
import software.aws.toolkits.jetbrains.services.amazonq.QConstants.MAX_FILE_SIZE_BYTES
import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS
import software.aws.toolkits.jetbrains.utils.isDevFile
import software.aws.toolkits.resources.AwsCoreBundle
Expand All @@ -33,7 +33,6 @@ import java.nio.file.Paths
import java.nio.file.StandardCopyOption
import java.util.Base64
import java.util.UUID
import kotlin.coroutines.coroutineContext
import kotlin.io.path.Path
import kotlin.io.path.createParentDirectories
import kotlin.io.path.getPosixFilePermissions
Expand All @@ -47,8 +46,7 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep
class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) {
// TODO: Need to correct this class location in the modules going further to support both amazonq and codescan.

private val requiredFilesForExecution = setOf("gradle/wrapper/gradle-wrapper.jar")
private val additionalGitIgnoreRules = setOf(
private val additionalGitIgnoreFolderRules = setOf(
".aws-sam",
".gem",
".git",
Expand All @@ -58,6 +56,12 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
".project",
".rvm",
".svn",
"node_modules",
"build",
"dist",
)

private val additionalGitIgnoreBinaryFilesRules = setOf(
"*.zip",
"*.bin",
"*.png",
Expand All @@ -70,9 +74,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
"license.md",
"License.md",
"LICENSE.md",
"node_modules",
"build",
"dist"
)

// well known source files that do not have extensions
Expand All @@ -90,12 +91,17 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
// selectedSourceFolder: is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading.
private var _selectedSourceFolder = projectRoot
private var ignorePatternsWithGitIgnore = emptyList<Regex>()
private var ignorePatternsForBinaryFiles = additionalGitIgnoreBinaryFilesRules
.map { convertGitIgnorePatternToRegex(it) }
.mapNotNull { pattern ->
runCatching { Regex(pattern) }.getOrNull()
}
private val gitIgnoreFile = File(selectedSourceFolder.path, ".gitignore")

init {
ignorePatternsWithGitIgnore = try {
buildList {
addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) })
addAll(additionalGitIgnoreFolderRules.map { convertGitIgnorePatternToRegex(it) })
addAll(parseGitIgnore())
}.mapNotNull { pattern ->
runCatching { Regex(pattern) }.getOrNull()
Expand Down Expand Up @@ -130,35 +136,49 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
return ALLOWED_CODE_EXTENSIONS.contains(extension)
}

private fun ignoreFileByExtension(file: VirtualFile) =
!isFileExtensionAllowed(file)
fun ignoreFile(file: VirtualFile, applyExtraBinaryFilesRules: Boolean = true): Boolean = ignoreFile(file.presentableUrl, applyExtraBinaryFilesRules)

fun ignoreFile(path: String, applyExtraBinaryFilesRules: Boolean = true): Boolean {
val allIgnoreRules = if (applyExtraBinaryFilesRules) ignorePatternsWithGitIgnore + ignorePatternsForBinaryFiles else ignorePatternsWithGitIgnore
val matchedRules = allIgnoreRules.map { pattern ->
// avoid partial match (pattern.containsMatchIn) since it causes us matching files
// against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
// we convert the glob rules to regex, add a trailing /* to all rules and then match
// entries against them by adding a trailing /.
// TODO: Add unit tests for gitignore matching
val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
pattern.matches("$relative/")
}
return matchedRules.any { it }
}

suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl)
private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)

suspend fun ignoreFile(path: String): Boolean {
if (requiredFilesForExecution.any { path.endsWith(it) }) {
private fun shouldIncludeInZipFile(file: VirtualFile, isAutoBuildFeatureEnabled: Boolean): Boolean {
// large files always ignored
if (file.length > MAX_FILE_SIZE_BYTES) {
return false
}
// this method reads like something a JS dev would write and doesn't do what the author thinks
val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
withContext(coroutineContext) {
// avoid partial match (pattern.containsMatchIn) since it causes us matching files
// against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
// we convert the glob rules to regex, add a trailing /* to all rules and then match
// entries against them by adding a trailing /.
// TODO: Add unit tests for gitignore matching
val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
async { pattern.matches("$relative/") }
}

// always respect gitignore rules and remove binary files if auto build is disabled
val isFileIgnoredByPattern = ignoreFile(file, !isAutoBuildFeatureEnabled)
if (isFileIgnoredByPattern) {
return false
}

// this will serially iterate over and block
// ideally we race the results https://github.com/Kotlin/kotlinx.coroutines/issues/2867
// i.e. Promise.any(...)
return deferredResults.any { it.await() }
}
// all other files are included when auto build enabled
if (isAutoBuildFeatureEnabled) {
return true
}

// when auto build is disabled, only include files with well known extensions and names except "devfile.yam"
if (!isDevFile(file) && (wellKnown(file) || isFileExtensionAllowed(file))) {
return true
}

fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
// Any other files should not be included
return false
}

suspend fun zipFiles(projectRoot: VirtualFile, isAutoBuildFeatureEnabled: Boolean?): File = withContext(getCoroutineBgContext()) {
val files = mutableListOf<VirtualFile>()
Expand All @@ -169,26 +189,13 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
projectRoot,
object : VirtualFileVisitor<Unit>() {
override fun visitFile(file: VirtualFile): Boolean {
val isWellKnown = runBlocking { wellKnown(file) }
val isFileIgnoredByExtension = runBlocking { ignoreFileByExtension(file) }
// if `isAutoBuildFeatureEnabled` is false, then filter devfile
val isFilterDevFile = if (isAutoBuildFeatureEnabled == true) false else isDevFile(file)

if (!isWellKnown && isFileIgnoredByExtension) {
val isIncluded = shouldIncludeInZipFile(file, isAutoBuildFeatureEnabled == true)
if (!isIncluded) {
val extension = file.extension.orEmpty()
ignoredExtensionMap[extension] = (ignoredExtensionMap[extension] ?: 0) + 1
return false
}

if (isFilterDevFile) {
return false
}

val isFileIgnoredByPattern = runBlocking { ignoreFile(file.name) }
if (isFileIgnoredByPattern) {
return false
}

if (file.isFile) {
totalSize += file.length
files.add(file)
Expand Down
1 change: 1 addition & 0 deletions ...re/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/QConstants.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ package software.aws.toolkits.jetbrains.services.amazonq
object QConstants {
const val Q_MARKETPLACE_URI = "https://aws.amazon.com/q/developer/"
const val CODEWHISPERER_LOGIN_HELP_URI = "https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/auth-access.html"
const val MAX_FILE_SIZE_BYTES = 1024 * 1024
}
1 change: 0 additions & 1 deletion ...brains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ val ALLOWED_CODE_EXTENSIONS = setOf(
"idl",
"ini",
"io",
"jar",
"java",
"jl",
"js",
Expand Down

0 comments on commit ec81010

Please sign in to comment.