Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CXP-3109: Remove shadow plugin from kafka-connect-s3 #46

Merged
merged 1 commit into from
Apr 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,6 @@ We made the decision to hard fork when it became clear that we would be responsi

## Usage

***NOTE***: You want to use the shadow jar produced by this project.
As a gradle dependency, it is `com.spredfast.kafka.connect.s3:kafka-connect-s3:0.4.0:shadow`.
The shadow jar ensures there are no conflicts with other libraries.

Use just like any other Connector: add it to the Connect classpath and configure a task. Read the rest of this document for configuration details.

## Important Configuration
Expand Down Expand Up @@ -98,7 +94,7 @@ See the [wiki](https://github.com/spredfast/kafka-connect-s3/wiki) for further d

## Build and Run

You should be able to build this with `./gradlew shadowJar`. Once the jar is generated in build/libs, include it in `CLASSPATH` (e.g., export `CLASSPATH=.:$CLASSPATH:/fullpath/to/kafka-connect-s3-jar` )
You should be able to build this with `./gradlew build`. Once the jar is generated in build/libs, include it in `CLASSPATH` (e.g., export `CLASSPATH=.:$CLASSPATH:/fullpath/to/kafka-connect-s3-jar` )

Run: `bin/connect-standalone.sh example-connect-worker.properties example-connect-s3-sink.properties`(from the root directory of project, make sure you have kafka on the path, if not then give full path of kafka before `bin`)

Expand Down
26 changes: 3 additions & 23 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
plugins {
id "com.diffplug.spotless" version "6.15.0"
id "com.github.johnrengelman.shadow" version "7.1.2"
id 'com.palantir.git-version' version '0.15.0'
id 'idea'
id 'java-library'
Expand Down Expand Up @@ -53,25 +52,6 @@ allprojects {
}
}

apply plugin: "com.github.johnrengelman.shadow"
shadowJar {
archiveClassifier = null
dependencies {
// provided in the connect classpath
exclude(dependency('org.apache.kafka:connect-api'))
exclude(dependency('org.apache.kafka:kafka-clients'))
exclude(dependency('net.jpountz.lz4:.*:.*'))
exclude(dependency('org.xerial.snappy:.*:.*'))
exclude(dependency('org.slf4j:.*:.*'))
}

// for things we directly depend on, repackage so we don't conflict with other connectors
relocate 'com.fasterxml', 'com.spredfast.shade.fasterxml'
relocate 'org.apache.commons', 'com.spredfast.shade.apache.commons'
relocate 'org.apache.http', 'com.spredfast.shade.apache.http'
relocate 'org.joda', 'com.spredfast.shade.joda'
}

dependencies {
implementation project(':sink')
implementation project(':source')
Expand Down Expand Up @@ -103,9 +83,9 @@ ext {

publishing {
publications {
shadow(MavenPublication) { publication ->
project.shadow.component(publication)
groupId = 'io.sugarcrm'
maven(MavenPublication) {
groupId 'io.sugarcrm'
artifact jar
}
}

Expand Down
26 changes: 2 additions & 24 deletions sink/build.gradle
Original file line number Diff line number Diff line change
@@ -1,24 +1,5 @@
description = "Kafka Connect Sink that writes to S3"

apply plugin: "com.github.johnrengelman.shadow"

shadowJar {
dependencies {
// provided in the connect classpath
exclude(dependency('org.apache.kafka:connect-api'))
exclude(dependency('org.apache.kafka:kafka-clients'))
exclude(dependency('net.jpountz.lz4:.*:.*'))
exclude(dependency('org.xerial.snappy:.*:.*'))
exclude(dependency('org.slf4j:.*:.*'))
}

// for things we directly depend on, repackage so we don't conflict with other connectors
relocate 'com.fasterxml', 'com.spredfast.shade.fasterxml'
relocate 'org.apache.commons', 'com.spredfast.shade.apache.commons'
relocate 'org.apache.http', 'com.spredfast.shade.apache.http'
relocate 'org.joda', 'com.spredfast.shade.joda'
}

configurations {
extLibs
}
Expand All @@ -35,7 +16,7 @@ dependencies {
testImplementation("ch.qos.logback:logback-core:1.4.5")
testImplementation("ch.qos.logback:logback-classic:1.4.5")

extLibs project(path: ':source', configuration: 'shadow' )
extLibs project(path: ':source')
}

tasks.register('copyLibs', Copy) {
Expand All @@ -44,14 +25,11 @@ tasks.register('copyLibs', Copy) {
}
}

tasks.build.dependsOn tasks.shadowJar

test {
testLogging {
outputs.upToDateWhen {false}
showStandardStreams = true
}
}

tasks.test.dependsOn tasks.shadowJar, tasks.copyLibs

tasks.test.dependsOn tasks.copyLibs
20 changes: 0 additions & 20 deletions source/build.gradle
Original file line number Diff line number Diff line change
@@ -1,25 +1,5 @@
description = "Kafka Connect Source that reads from S3"

apply plugin: "com.github.johnrengelman.shadow"

shadowJar {
dependencies {
// provided in the connect classpath
exclude(dependency('org.apache.kafka:connect-api'))
exclude(dependency('org.apache.kafka:kafka-clients'))
exclude(dependency('net.jpountz.lz4:.*:.*'))
exclude(dependency('org.xerial.snappy:.*:.*'))
exclude(dependency('org.slf4j:.*:.*'))
}

// for things we directly depend on, repackage so we don't conflict with other connectors
relocate 'com.fasterxml', 'com.spredfast.shade.fasterxml'
relocate 'org.apache.commons', 'com.spredfast.shade.apache.commons'
relocate 'org.apache.http', 'com.spredfast.shade.apache.http'
relocate 'org.joda', 'com.spredfast.shade.joda'
}

tasks.build.dependsOn tasks.shadowJar
tasks.test.dependsOn(":sink:copyLibs")

dependencies {
Expand Down
Loading