diff --git a/cloud-etl/connectors/s3_avro.json b/cloud-etl/connectors/s3_avro.json index 5ceb4d18b..30a25083f 100644 --- a/cloud-etl/connectors/s3_avro.json +++ b/cloud-etl/connectors/s3_avro.json @@ -6,6 +6,7 @@ "kafka.api.secret": "$CLOUD_SECRET", "aws.access.key.id": "$AWS_ACCESS_KEY_ID", "aws.secret.access.key": "$AWS_SECRET_ACCESS_KEY", + "store.url": "$S3_ENDPOINT_URL", "s3.bucket.name": "$S3_BUCKET", "data.format": "AVRO", "time.interval" : "HOURLY", diff --git a/cloud-etl/connectors/s3_no_avro.json b/cloud-etl/connectors/s3_no_avro.json index 0a181e1a3..4c9e1d4a5 100644 --- a/cloud-etl/connectors/s3_no_avro.json +++ b/cloud-etl/connectors/s3_no_avro.json @@ -6,6 +6,7 @@ "kafka.api.secret": "$CLOUD_SECRET", "aws.access.key.id": "$AWS_ACCESS_KEY_ID", "aws.secret.access.key": "$AWS_SECRET_ACCESS_KEY", + "store.url": "$S3_ENDPOINT_URL", "s3.bucket.name": "$S3_BUCKET", "time.interval" : "HOURLY", "data.format": "BYTES", diff --git a/cloud-etl/read-data.sh b/cloud-etl/read-data.sh index 3d3361580..c2b2c0601 100755 --- a/cloud-etl/read-data.sh +++ b/cloud-etl/read-data.sh @@ -81,7 +81,7 @@ else --schema-registry-endpoint $SCHEMA_REGISTRY_URL\ --from-beginning\ --print-key\ - --value-format avro + --value-format avro fi @@ -92,7 +92,7 @@ AVRO_VERSION=1.9.1 #fi if [[ "$DESTINATION_STORAGE" == "s3" ]]; then - for key in $(aws s3api list-objects --bucket $S3_BUCKET | jq -r '.Contents[].Key'); do + for key in $(aws s3api list-objects --bucket $S3_BUCKET --profile $S3_PROFILE | jq -r '.Contents[].Key'); do echo "S3 key: $key" #aws s3 cp s3://$S3_BUCKET/$key data.avro #echo "java -Dlog4j.configuration="file:log4j.properties" -jar avro-tools-${AVRO_VERSION}.jar tojson data.avro" diff --git a/cloud-etl/setup_storage_s3.sh b/cloud-etl/setup_storage_s3.sh index d8101ca59..3822a8e77 100755 --- a/cloud-etl/setup_storage_s3.sh +++ b/cloud-etl/setup_storage_s3.sh @@ -1,5 +1,5 @@ #!/bin/bash - + # Source library source ../utils/helper.sh source ../utils/ccloud_library.sh @@ -25,6 +25,9 @@ fi export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id --profile $S3_PROFILE) export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key --profile $S3_PROFILE) +export S3_ENDPOINT_URL=$(aws configure get endpoint_url --profile $S3_PROFILE) +export S3_ENDPOINT_URL="${S3_ENDPOINT_URL:-https://s3.$STORAGE_REGION.amazonaws.com}" + ccloud::create_connector connectors/s3_no_avro.json || exit 1 ccloud::wait_for_connector_up connectors/s3_no_avro.json 300 || exit 1 ccloud::create_connector connectors/s3_avro.json || exit 1