Skip to content

Commit 13c6586

Browse files
linjianchanglinjianchang
authored
Merge branch 'apache:master' into master-36796
2 parents 3479f43 + 8437bc9 commit 13c6586

File tree

318 files changed

+20558
-7232
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

318 files changed

+20558
-7232
lines changed

‎.github/labeler.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,3 +93,5 @@ oceanbase-pipeline-connector:
9393
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase/**/*
9494
maxcompute-pipeline-connector:
9595
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute/**/*
96+
iceberg-pipeline-connector:
97+
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg/**/*

‎.github/workflows/build_docs.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ jobs:
5454
matrix:
5555
branch:
5656
- master
57+
- release-3.4
5758
- release-3.3
58-
- release-3.2
5959

6060
steps:
6161
- uses: actions/checkout@v3
@@ -69,8 +69,8 @@ jobs:
6969
echo "flink_branch=${currentBranch}" >> ${GITHUB_ENV}
7070
7171
if [ "${currentBranch}" = "master" ]; then
72-
echo "flink_alias=release-3.4" >> ${GITHUB_ENV}
73-
elif [ "${currentBranch}" = "release-3.3" ]; then
72+
echo "flink_alias=release-3.5" >> ${GITHUB_ENV}
73+
elif [ "${currentBranch}" = "release-3.4" ]; then
7474
echo "flink_alias=stable" >> ${GITHUB_ENV}
7575
fi
7676

‎.github/workflows/flink_cdc_base.yml

Lines changed: 90 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,18 @@ name: Flink CDC Base Workflow
1818
on:
1919
workflow_call:
2020
inputs:
21-
java-version:
21+
java-versions:
2222
description: "Jdk versions to test against."
2323
required: false
2424
type: string
2525
default: "['8']"
26-
flink-version:
26+
flink-versions:
2727
description: "Flink versions to test against."
2828
required: false
2929
type: string
3030
default: "['generic']"
31-
module:
32-
description: "Flink CDC module to test against."
31+
modules:
32+
description: "Flink CDC modules to test against."
3333
required: true
3434
type: string
3535
parallelism:
@@ -48,23 +48,29 @@ env:
4848
flink-cdc-common,\
4949
flink-cdc-composer,\
5050
flink-cdc-runtime,\
51-
flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base"
51+
flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base,\
52+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
5253

5354
MODULES_PIPELINE_CONNECTORS: "\
54-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
55-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
5655
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
57-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
58-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
59-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
6056
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\
57+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg,\
58+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
59+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
60+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
6161
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\
62-
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
62+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
63+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
64+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss,\
65+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
6366

64-
MODULES_MYSQL: "\
67+
MODULES_MYSQL_SOURCE: "\
6568
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
6669
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
6770

71+
MODULES_MYSQL_PIPELINE: "\
72+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql"
73+
6874
MODULES_POSTGRES: "\
6975
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
7076
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
@@ -85,10 +91,13 @@ env:
8591
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
8692
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
8793

88-
MODULES_OCEANBASE: "\
94+
MODULES_OCEANBASE_SOURCE: "\
8995
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
9096
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
9197

98+
MODULES_OCEANBASE_PIPELINE: "\
99+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase"
100+
92101
MODULES_DB2: "\
93102
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
94103
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
@@ -97,21 +106,46 @@ env:
97106
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
98107
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
99108

109+
MODULES_DORIS: "\
110+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris"
111+
112+
MODULES_STARROCKS: "\
113+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks"
114+
115+
MODULES_ICEBERG: "\
116+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg"
117+
118+
MODULES_KAFKA: "\
119+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka"
120+
121+
MODULES_PAIMON: "\
122+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
123+
124+
MODULES_ELASTICSEARCH: "\
125+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
126+
127+
MODULES_MAXCOMPUTE: "\
128+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
129+
130+
MODULES_FLUSS: "\
131+
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-fluss"
132+
100133
MODULES_PIPELINE_E2E: "\
101134
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
102135

103136
MODULES_SOURCE_E2E: "\
104137
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
105138

106139
jobs:
107-
compile_and_test:
140+
test:
108141
runs-on: ubuntu-latest
109142
timeout-minutes: 120
110143
strategy:
111144
fail-fast: false
112145
matrix:
113-
java-version: ${{ fromJSON(inputs.java-version) }}
114-
flink-version: ${{ fromJSON(inputs.flink-version) }}
146+
java-version: ${{ fromJSON(inputs.java-versions) }}
147+
flink-version: ${{ fromJSON(inputs.flink-versions) }}
148+
module: ${{ fromJSON(inputs.modules) }}
115149
steps:
116150
- run: echo "Running CI pipeline for JDK version ${{ matrix.java-version }}"
117151
- name: Clean up disk space
@@ -149,22 +183,22 @@ jobs:
149183
maven-version: 3.8.6
150184

151185
- name: Compile and test
152-
timeout-minutes: 60
186+
timeout-minutes: 90
153187
run: |
154188
. .github/workflows/utils.sh
155189
jvm_timezone=$(random_timezone)
156190
echo "JVM timezone is set to $jvm_timezone"
157191
set -o pipefail
158192
159-
case ${{ inputs.module }} in
193+
case ${{ matrix.module }} in
160194
("core")
161195
modules=${{ env.MODULES_CORE }}
162196
;;
163-
("pipeline_connectors")
164-
modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
197+
("mysql-source")
198+
modules=${{ env.MODULES_MYSQL_SOURCE }}
165199
;;
166-
("mysql")
167-
modules=${{ env.MODULES_MYSQL }}
200+
("mysql-pipeline")
201+
modules=${{ env.MODULES_MYSQL_PIPELINE }}
168202
;;
169203
("postgres")
170204
modules=${{ env.MODULES_POSTGRES }}
@@ -184,34 +218,61 @@ jobs:
184218
("tidb")
185219
modules=${{ env.MODULES_TIDB }}
186220
;;
187-
("oceanbase")
188-
modules=${{ env.MODULES_OCEANBASE }}
221+
("oceanbase-source")
222+
modules=${{ env.MODULES_OCEANBASE_SOURCE }}
223+
;;
224+
("oceanbase-pipeline")
225+
modules=${{ env.MODULES_OCEANBASE_PIPELINE }}
189226
;;
190227
("db2")
191228
modules=${{ env.MODULES_DB2 }}
192229
;;
193230
("vitess")
194231
modules=${{ env.MODULES_VITESS }}
195232
;;
233+
("doris")
234+
modules=${{ env.MODULES_DORIS }}
235+
;;
236+
("elasticsearch")
237+
modules=${{ env.MODULES_ELASTICSEARCH }}
238+
;;
239+
("iceberg")
240+
modules=${{ env.MODULES_ICEBERG }}
241+
;;
242+
("kafka")
243+
modules=${{ env.MODULES_KAFKA }}
244+
;;
245+
("maxcompute")
246+
modules=${{ env.MODULES_MAXCOMPUTE }}
247+
;;
248+
("paimon")
249+
modules=${{ env.MODULES_PAIMON }}
250+
;;
251+
("starrocks")
252+
modules=${{ env.MODULES_STARROCKS }}
253+
;;
254+
("fluss")
255+
modules=${{ env.MODULES_FLUSS }}
256+
;;
196257
("pipeline_e2e")
197-
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
258+
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
198259
modules=${{ env.MODULES_PIPELINE_E2E }}
199260
;;
200261
("source_e2e")
201-
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
262+
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
202263
modules=${{ env.MODULES_SOURCE_E2E }}
203264
;;
204265
esac
205266
206-
if [ ${{ inputs.module }} != "pipeline_e2e" ] && [ ${{ inputs.module }} != "source_e2e" ]; then
267+
if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
207268
compile_modules=$modules
208269
fi
209270
210271
build_maven_parameter=""
211272
212-
if [ ${{ inputs.module }} == "mongodb6" ]; then
273+
if [ ${{ matrix.module }} == "mongodb6" ]; then
213274
build_maven_parameter="-DspecifiedMongoVersion=6.0.16"
214-
elif [ ${{ inputs.module }} == "mongodb7" ]; then
275+
elif [ ${{ matrix.module }} == "mongodb7" ]; then
215276
build_maven_parameter="-DspecifiedMongoVersion=7.0.12"
216277
fi
217278

‎.github/workflows/flink_cdc_ci.yml

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,24 @@ jobs:
5757
run: mvn --no-snapshot-updates -B package -DskipTests
5858
- name: Run license check
5959
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
60-
ut:
61-
strategy:
62-
fail-fast: false
63-
matrix:
64-
module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
65-
name: Unit Tests
60+
common:
61+
name: Common Unit Tests
62+
uses: ./.github/workflows/flink_cdc_base.yml
63+
with:
64+
java-versions: "[8]"
65+
modules: "['core']"
66+
pipeline-ut:
67+
name: Pipeline Unit Tests
68+
uses: ./.github/workflows/flink_cdc_base.yml
69+
with:
70+
java-versions: "[8]"
71+
modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss']"
72+
source-ut:
73+
name: Source Unit Tests
6674
uses: ./.github/workflows/flink_cdc_base.yml
6775
with:
68-
java-version: "[8]"
69-
module: ${{ matrix.module }}
76+
java-versions: "[8]"
77+
modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
7078
pipeline_e2e:
7179
strategy:
7280
fail-fast: false
@@ -75,20 +83,14 @@ jobs:
7583
name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
7684
uses: ./.github/workflows/flink_cdc_base.yml
7785
with:
78-
java-version: "[8]"
79-
flink-version: "['1.19.2', '1.20.1']"
80-
module: pipeline_e2e
86+
java-versions: "[8]"
87+
flink-versions: "['1.19.2', '1.20.1']"
88+
modules: "['pipeline_e2e']"
8189
parallelism: ${{ matrix.parallelism }}
8290
source_e2e:
8391
name: Source E2E Tests
8492
uses: ./.github/workflows/flink_cdc_base.yml
8593
with:
86-
java-version: "[8]"
87-
flink-version: "['1.19.2', '1.20.1']"
88-
module: source_e2e
89-
migration_test:
90-
name: Migration Tests
91-
uses: ./.github/workflows/flink_cdc_migration_test_base.yml
92-
with:
93-
java-version: "[8]"
94-
flink-version: "['1.19.2', '1.20.1']"
94+
java-versions: "[8]"
95+
flink-versions: "['1.19.2', '1.20.1']"
96+
modules: "['source_e2e']"

‎.github/workflows/flink_cdc_ci_nightly.yml

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -47,42 +47,45 @@ jobs:
4747
run: mvn --no-snapshot-updates -B package -DskipTests
4848
- name: Run license check
4949
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
50-
ut:
50+
common:
5151
if: github.repository == 'apache/flink-cdc'
52-
strategy:
53-
fail-fast: false
54-
matrix:
55-
module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
56-
name: Unit Tests
52+
name: Common Unit Tests
53+
uses: ./.github/workflows/flink_cdc_base.yml
54+
with:
55+
java-versions: "[11]"
56+
modules: "['core']"
57+
pipeline-ut:
58+
if: github.repository == 'apache/flink-cdc'
59+
name: Pipeline Unit Tests
5760
uses: ./.github/workflows/flink_cdc_base.yml
5861
with:
59-
java-version: "[11]"
60-
module: ${{ matrix.module }}
62+
java-versions: "[11]"
63+
modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks', 'fluss']"
64+
source-ut:
65+
if: github.repository == 'apache/flink-cdc'
66+
name: Source Unit Tests
67+
uses: ./.github/workflows/flink_cdc_base.yml
68+
with:
69+
java-versions: "[11]"
70+
modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
6171
pipeline_e2e:
6272
if: github.repository == 'apache/flink-cdc'
6373
strategy:
6474
fail-fast: false
6575
matrix:
6676
parallelism: [ 1, 4 ]
67-
name: Pipeline E2E Tests (${{ matrix.parallelism }} Parallelism)
77+
name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
6878
uses: ./.github/workflows/flink_cdc_base.yml
6979
with:
70-
java-version: "[11]"
71-
flink-version: "['1.19.2', '1.20.1']"
72-
module: pipeline_e2e
80+
java-versions: "[11]"
81+
flink-versions: "['1.19.2', '1.20.1']"
82+
modules: "['pipeline_e2e']"
7383
parallelism: ${{ matrix.parallelism }}
7484
source_e2e:
7585
if: github.repository == 'apache/flink-cdc'
7686
name: Source E2E Tests
7787
uses: ./.github/workflows/flink_cdc_base.yml
7888
with:
79-
java-version: "[11]"
80-
flink-version: "['1.19.2', '1.20.1']"
81-
module: source_e2e
82-
migration_test:
83-
if: github.repository == 'apache/flink-cdc'
84-
name: Migration Tests
85-
uses: ./.github/workflows/flink_cdc_migration_test_base.yml
86-
with:
87-
java-version: "[11]"
88-
flink-version: "['1.19.2', '1.20.1']"
89+
java-versions: "[11]"
90+
flink-versions: "['1.19.2', '1.20.1']"
91+
modules: "['source_e2e']"

0 commit comments

Comments
 (0)