[temp] Changes from 2.7.2.Final

.github/workflows/yb-pg-parity.yml

@@ -0,0 +1,26 @@
+name: YugabyteDB PG parity sanity
+
+on:
+  push:
+    branches:
+      - ybdb-debezium-2.5.2
+  pull_request:
+    branches:
+      - ybdb-debezium-2.5.2
+
+jobs:
+  build:
+    name: "Build PG connector"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Action
+        uses: actions/checkout@v4
+
+      - name: Set up Java 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17
+
+      - name: Download dependencies
+        run: ./mvnw clean install -Dquick -pl debezium-connector-postgres -am

.github/workflows/yb-release-quay.yml

@@ -0,0 +1,77 @@
+name: Publish YugabyteDBConnector
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version of the connector to be released"
+        required: true
+        type: string
+      isSnapshotBuild:
+        description: "Snapshot build?"
+        required: true
+        type: boolean
+        default: true
+
+permissions: write-all
+
+jobs:
+  build:
+    name: "Publish YugabyteDBConnector"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Action
+        uses: actions/checkout@v4
+
+      - name: Set up Java 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17
+
+      - name: Set version for release
+        run: ./mvnw versions:set -DnewVersion=${{ inputs.version }}
+
+      - name: Compile jar file
+        run: ./mvnw clean install -Dquick -pl debezium-connector-postgres -am
+      - name: Login to Quay.io
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_BOT_USERNAME }}
+          password: ${{ secrets.QUAY_BOT_TOKEN }}
+      - name: Build docker image
+        run: docker build . -t quay.io/yugabyte/ybdb-debezium:${{ inputs.version }} --platform linux/amd64
+      - name: Push image to Quay
+        run: docker push quay.io/yugabyte/ybdb-debezium:${{ inputs.version }}
+      - name: Create GitHub release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ inputs.version }}
+          release_name: Release ${{ inputs.version }}
+          draft: true
+          prerelease: ${{ inputs.isSnapshotBuild }}
+      - name: Upload fat jar to GitHub release
+        id: upload-release-asset-fat-jar
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./debezium-connector-postgres/target/debezium-connector-yugabytedb-${{ inputs.version }}-jar-with-dependencies.jar
+          asset_name: debezium-connector-yugabytedb-${{ inputs.version }}-jar-with-dependencies.jar
+          asset_content_type: application/java-archive
+    #   Commenting the code to upload a light jar for the time being.
+    #   - name: Upload fat jar to GitHub release
+    #     id: upload-release-asset-jar
+    #     uses: actions/upload-release-asset@v1
+    #     env:
+    #       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    #     with:
+    #       upload_url: ${{ steps.create_release.outputs.upload_url }}
+    #       asset_path: ./debezium-connector-postgresql/target/debezium-connector-postgres-*.jar
+    #       asset_name: debezium-connector-postgresql-${{ inputs.version }}.jar
+    #       asset_content_type: application/java-archive

CHANGELOG.md

@@ -692,6 +692,11 @@ January 18th 2024 [Detailed release notes](https://issues.redhat.com/secure/Rele
 
 ### Fixes since 2.5.0.Final
 
+* Connector is getting stopped while processing bulk update(50k) records in debezium server 2.0.1.Final [DBZ-6955](https://issues.redhat.com/browse/DBZ-6955)
+* Empty object sent to GCP Pub/Sub after DELETE event [DBZ-7098](https://issues.redhat.com/browse/DBZ-7098)
+* Enhance Oracle's CREATE TABLE for Multiple Table Specifications [DBZ-7286](https://issues.redhat.com/browse/DBZ-7286)
+* PostgreSQL ad-hoc blocking snapshots fail when snapshot mode is "never" [DBZ-7311](https://issues.redhat.com/browse/DBZ-7311)
+* Ad-hoc blocking snapshot dies with "invalid snapshot identifier" immediately after connector creation [DBZ-7312](https://issues.redhat.com/browse/DBZ-7312)
 * Empty object sent to GCP Pub/Sub after DELETE event [DBZ-7098](https://issues.redhat.com/browse/DBZ-7098)
 * Debezium-ddl-parser crashes on parsing MySQL DDL statement (subquery with UNION) [DBZ-7259](https://issues.redhat.com/browse/DBZ-7259)
 * Oracle DDL parsing error in PARTITION REFERENCE [DBZ-7266](https://issues.redhat.com/browse/DBZ-7266)

COPYRIGHT.txt

@@ -494,6 +494,7 @@ Shuran Zhang
 SiuFay
 Siddhant Agnihotry
 Stanley Shyiko
+Stavros Champilomatis
 Stathis Souris
 Stavros Champilomatis
 Stefan Miklosovic

Dockerfile

@@ -0,0 +1,71 @@
+# On your terminal, run the following to build the image:
+# mvn clean package -Dquick
+
+FROM debezium/connect:2.5.2.Final
+
+WORKDIR $KAFKA_CONNECT_PLUGINS_DIR
+RUN rm -f debezium-connector-postgres/debezium-connector-postgres-*.jar
+RUN rm -rf debezium-connector-db2
+RUN rm -rf debezium-connector-informix
+RUN rm -rf debezium-connector-mongodb
+RUN rm -rf debezium-connector-jdbc
+RUN rm -rf debezium-connector-mysql
+RUN rm -rf debezium-connector-oracle
+RUN rm -rf debezium-connector-spanner
+RUN rm -rf debezium-connector-sqlserver
+RUN rm -rf debezium-connector-vitess
+WORKDIR /
+
+# Copy the Debezium Connector for Postgres adapted for YugabyteDB
+COPY debezium-connector-postgres/target/debezium-connector-yugabytedb-*.jar $KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres
+
+# Set the TLS version to be used by Kafka processes
+ENV KAFKA_OPTS="-Djdk.tls.client.protocols=TLSv1.2 -javaagent:/kafka/etc/jmx_prometheus_javaagent-0.17.2.jar=8080:/kafka/etc/jmx-exporter/metrics.yml"
+
+# Package all the Avro related jar files
+RUN mkdir -p $KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres/avro-supporting-jars/
+WORKDIR $KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres/avro-supporting-jars
+RUN curl -sLo kafka-connect-avro-converter-7.5.3.jar https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-converter/7.5.3/kafka-connect-avro-converter-7.5.3.jar 
+RUN curl -sLo kafka-connect-avro-data-7.5.3.jar https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-data/7.5.3/kafka-connect-avro-data-7.5.3.jar
+RUN curl -sLo kafka-avro-serializer-7.5.3.jar https://packages.confluent.io/maven/io/confluent/kafka-avro-serializer/7.5.3/kafka-avro-serializer-7.5.3.jar
+RUN curl -sLo kafka-schema-serializer-7.5.3.jar https://packages.confluent.io/maven/io/confluent/kafka-schema-serializer/7.5.3/kafka-schema-serializer-7.5.3.jar 
+RUN curl -sLo kafka-schema-converter-7.5.3.jar https://packages.confluent.io/maven/io/confluent/kafka-schema-converter/7.5.3/kafka-schema-converter-7.5.3.jar
+RUN curl -sLo kafka-schema-registry-client-7.5.3.jar https://packages.confluent.io/maven/io/confluent/kafka-schema-registry-client/7.5.3/kafka-schema-registry-client-7.5.3.jar 
+RUN curl -sLo common-config-7.5.3.jar https://packages.confluent.io/maven/io/confluent/common-config/7.5.3/common-config-7.5.3.jar
+RUN curl -sLo common-utils-7.5.3.jar https://packages.confluent.io/maven/io/confluent/common-utils/7.5.3/common-utils-7.5.3.jar 
+
+RUN curl -sLo avro-1.11.3.jar https://repo1.maven.org/maven2/org/apache/avro/avro/1.11.3/avro-1.11.3.jar
+RUN curl -sLo commons-compress-1.21.jar https://repo1.maven.org/maven2/org/apache/commons/commons-compress/1.21/commons-compress-1.21.jar
+RUN curl -sLo failureaccess-1.0.jar https://repo1.maven.org/maven2/com/google/guava/failureaccess/1.0/failureaccess-1.0.jar
+RUN curl -sLo guava-33.0.0-jre.jar https://repo1.maven.org/maven2/com/google/guava/guava/33.0.0-jre/guava-33.0.0-jre.jar 
+RUN curl -sLo minimal-json-0.9.5.jar https://repo1.maven.org/maven2/com/eclipsesource/minimal-json/minimal-json/0.9.5/minimal-json-0.9.5.jar
+RUN curl -sLo re2j-1.6.jar https://repo1.maven.org/maven2/com/google/re2j/re2j/1.6/re2j-1.6.jar 
+RUN curl -sLo slf4j-api-1.7.36.jar https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar
+RUN curl -sLo snakeyaml-2.0.jar https://repo1.maven.org/maven2/org/yaml/snakeyaml/2.0/snakeyaml-2.0.jar 
+RUN curl -sLo swagger-annotations-2.1.10.jar https://repo1.maven.org/maven2/io/swagger/core/v3/swagger-annotations/2.1.10/swagger-annotations-2.1.10.jar 
+RUN curl -sLo jackson-databind-2.14.2.jar https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.14.2/jackson-databind-2.14.2.jar 
+RUN curl -sLo jackson-core-2.14.2.jar https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.14.2/jackson-core-2.14.2.jar 
+RUN curl -sLo jackson-annotations-2.14.2.jar https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.14.2/jackson-annotations-2.14.2.jar 
+RUN curl -sLo jackson-dataformat-csv-2.14.2.jar https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-csv/2.14.2/jackson-dataformat-csv-2.14.2.jar 
+RUN curl -sLo logredactor-1.0.12.jar https://repo1.maven.org/maven2/io/confluent/logredactor/1.0.12/logredactor-1.0.12.jar
+RUN curl -sLo logredactor-metrics-1.0.12.jar https://repo1.maven.org/maven2/io/confluent/logredactor-metrics/1.0.12/logredactor-metrics-1.0.12.jar
+WORKDIR /
+
+# Add the required jar files to be packaged with the base connector
+RUN cd $KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres && curl -sLo kafka-connect-jdbc-10.6.5-CUSTOM.7.jar https://github.com/yugabyte/kafka-connect-jdbc/releases/download/10.6.5-CUSTOM.7/kafka-connect-jdbc-10.6.5-CUSTOM.7.jar
+RUN cd $KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres && curl -sLo jdbc-yugabytedb-42.3.5-yb-1.jar https://repo1.maven.org/maven2/com/yugabyte/jdbc-yugabytedb/42.3.5-yb-1/jdbc-yugabytedb-42.3.5-yb-1.jar
+RUN cd $KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres && curl -sLo transforms-for-apache-kafka-connect-1.5.0.zip https://github.com/Aiven-Open/transforms-for-apache-kafka-connect/releases/download/v1.5.0/transforms-for-apache-kafka-connect-1.5.0.zip
+RUN cd $KAFKA_CONNECT_PLUGINS_DIR/debezium-connector-postgres && unzip transforms-for-apache-kafka-connect-1.5.0.zip
+
+# Add Jmx agent and metrics pattern file to expose the metrics info
+RUN mkdir /kafka/etc && cd /kafka/etc && curl -so jmx_prometheus_javaagent-0.17.2.jar https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.17.2/jmx_prometheus_javaagent-0.17.2.jar
+
+COPY metrics.yml /kafka/etc/jmx-exporter/
+
+ENV CLASSPATH=$KAFKA_HOME
+ENV JMXHOST=localhost
+ENV JMXPORT=1976
+
+# properties file having instructions to roll over log files in case the size exceeds a given limit
+COPY log4j.properties $KAFKA_HOME/config/log4j.properties
+

debezium-connector-postgres/YB_DEV_NOTES.md

@@ -0,0 +1,18 @@
+# YugabyteDB Development Instructions
+
+## Compiling code
+
+Since the smart driver changes require us to build the debezium core as well, build can be completed using:
+
+```bash
+./mvnw clean install -Passembly -DskipITs -DskipTests -pl debezium-connector-postgres -am
+```
+
+## Running tests
+
+1. Compile PG connector code from the root directory with the above command.
+2. Start YugabyteDB instance using `yugabyted`:
+   ```bash
+   ./bin/yugabyted start --ui=false --advertise_address=127.0.0.1 --master_flags="ysql_cdc_active_replication_slot_window_ms=0" --tserver_flags="allowed_preview_flags_csv={cdcsdk_enable_dynamic_table_support},cdcsdk_enable_dynamic_table_support=true,cdcsdk_publication_list_refresh_interval_secs=3,ysql_cdc_active_replication_slot_window_ms=0,ysql_sequence_cache_method=server"
+   ```
+3. Run tests 

debezium-connector-postgres/pom.xml

@@ -7,25 +7,25 @@
         <relativePath>../debezium-parent/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    <artifactId>debezium-connector-postgres</artifactId>
-    <name>Debezium Connector for PostgreSQL</name>
+    <artifactId>debezium-connector-yugabytedb</artifactId>
+    <name>YugabyteDB Source Connector</name>
     <packaging>jar</packaging>
     <properties>
         <!-- 
           Specify the properties that will be used for setting up the integration tests' Docker container.
           Note that the `dockerhost.ip` property is computed from the IP address of DOCKER_HOST, which will
           work on all platforms. We'll set some of these as system properties during integration testing.
       -->
-        <version.postgres.server>12</version.postgres.server>
-        <postgres.host>${docker.host.address}</postgres.host>
-        <postgres.port>5432</postgres.port>
-        <postgres.user>postgres</postgres.user>
-        <postgres.password>postgres</postgres.password>
-        <postgres.db.name>postgres</postgres.db.name>
+        <version.postgres.server>10</version.postgres.server>
+        <postgres.host>127.0.0.1</postgres.host>
+        <postgres.port>5433</postgres.port>
+        <postgres.user>yugabyte</postgres.user>
+        <postgres.password>yugabyte</postgres.password>
+        <postgres.db.name>yugabyte</postgres.db.name>
         <postgres.encoding>UTF8</postgres.encoding>
         <postgres.system.lang>en_US.utf8</postgres.system.lang>
-        <postgres.image>quay.io/debezium/postgres:${version.postgres.server}</postgres.image>
-        <postgres.config.file>/usr/share/postgresql/postgresql.conf.sample</postgres.config.file>
+<!--        <postgres.image>quay.io/debezium/postgres:${version.postgres.server}</postgres.image>-->
+<!--        <postgres.config.file>/usr/share/postgresql/postgresql.conf.sample</postgres.config.file>-->
         <docker.skip>false</docker.skip>
         <docker.showLogs>true</docker.showLogs>
         <docker.initimage>ln -fs /usr/share/zoneinfo/US/Samoa /etc/localtime &amp;&amp; echo timezone=US/Samoa &gt;&gt; ${postgres.config.file}</docker.initimage>
@@ -51,6 +51,11 @@
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.yugabyte</groupId>
+            <artifactId>jdbc-yugabytedb</artifactId>
+            <version>42.3.5-yb-4</version>
+        </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java</artifactId>
@@ -161,6 +166,25 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.6.0</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>assemble-all</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>com.github.os72</groupId>
                 <artifactId>protoc-jar-maven-plugin</artifactId>
@@ -201,7 +225,7 @@
                                     <LANG>${postgres.system.lang}</LANG>
                                 </env>
                                 <ports>
-                                    <port>${postgres.port}:5432</port>
+                                    <port>${postgres.port}:5433</port>
                                 </ports>
                                 <log>
                                     <prefix>postgres</prefix>