Skip to content

Commit a26f41d

Browse files
MrartChengJie1053
andauthored
[FLINK-35670][pipeline-connector][postgres] Add PostgreSQL source. (#3968)
PostgreSQL Pipeline Source does not support synchronizing table structure changes currently. --------- Co-authored-by: ChengJie1053 <18033291053@163.com>
1 parent 332ec71 commit a26f41d

File tree

36 files changed

+4200
-21
lines changed

36 files changed

+4200
-21
lines changed

.github/labeler.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,5 @@ maxcompute-pipeline-connector:
9595
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute/**/*
9696
iceberg-pipeline-connector:
9797
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/**/*
98+
postgres-pipeline-connector:
99+
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres/**/*

.github/workflows/flink_cdc_base.yml

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ env:
5959
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
6060
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
6161
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\
62+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
63+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres,\
6264
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
6365
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
6466
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss,\
@@ -71,10 +73,13 @@ env:
7173
MODULES_MYSQL_PIPELINE: "\
7274
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql"
7375

74-
MODULES_POSTGRES: "\
76+
MODULES_POSTGRES_SOURCE: "\
7577
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
7678
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
7779

80+
MODULES_POSTGRES_PIPELINE: "\
81+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-postgres"
82+
7883
MODULES_ORACLE: "\
7984
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
8085
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
@@ -200,8 +205,11 @@ jobs:
200205
("mysql-pipeline")
201206
modules=${{ env.MODULES_MYSQL_PIPELINE }}
202207
;;
203-
("postgres")
204-
modules=${{ env.MODULES_POSTGRES }}
208+
("postgres-source")
209+
modules=${{ env.MODULES_POSTGRES_SOURCE }}
210+
;;
211+
("postgres-pipeline")
212+
modules=${{ env.MODULES_POSTGRES_PIPELINE }}
205213
;;
206214
("oracle")
207215
modules=${{ env.MODULES_ORACLE }}
@@ -255,11 +263,11 @@ jobs:
255263
modules=${{ env.MODULES_FLUSS }}
256264
;;
257265
("pipeline_e2e")
258-
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
266+
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
259267
modules=${{ env.MODULES_PIPELINE_E2E }}
260268
;;
261269
("source_e2e")
262-
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
270+
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES_SOURCE }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
263271
modules=${{ env.MODULES_SOURCE_E2E }}
264272
;;
265273
esac

.github/workflows/flink_cdc_ci.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ jobs:
6868
uses: ./.github/workflows/flink_cdc_base.yml
6969
with:
7070
java-versions: "[8]"
71-
modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss']"
71+
modules: "['mysql-pipeline', 'postgres-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss']"
7272
source-ut:
7373
name: Source Unit Tests
7474
uses: ./.github/workflows/flink_cdc_base.yml
7575
with:
7676
java-versions: "[8]"
77-
modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
77+
modules: "['mysql-source', 'postgres-source', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
7878
pipeline_e2e:
7979
strategy:
8080
fail-fast: false

.github/workflows/flink_cdc_ci_nightly.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,14 @@ jobs:
6060
uses: ./.github/workflows/flink_cdc_base.yml
6161
with:
6262
java-versions: "[11]"
63-
modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss']"
63+
modules: "['mysql-pipeline', 'postgres-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss']"
6464
source-ut:
6565
if: github.repository == 'apache/flink-cdc'
6666
name: Source Unit Tests
6767
uses: ./.github/workflows/flink_cdc_base.yml
6868
with:
6969
java-versions: "[11]"
70-
modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
70+
modules: "['mysql-source', 'postgres-source', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
7171
pipeline_e2e:
7272
if: github.repository == 'apache/flink-cdc'
7373
strategy:

0 commit comments

Comments
 (0)