Build Media-Dev for Apphub #407
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This action will be trigger by docker.yml action | |
| # docker.yml action download the artifact for build | |
| name: Build Media-Dev for Apphub | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: [media_dev_event] | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| env: | |
| ACCESS_TOKEN: ${{ secrets.CONTENTFUL_GRAPHQLTOKEN }} | |
| LOCALES: "en-US,zh-CN" | |
| DATA_DIR: "json" | |
| APPSTORE_LOGO_DIR: "logos" | |
| SCREENSHOT_DIR: "screenshots" | |
| LIMIT: 100 | |
| CI: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install GNU Parallel | |
| run: sudo apt-get install -y parallel | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.x" | |
| - name: Install AWS CLI via pip | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install awscli | |
| - name: Fetch catalog data from contentful graphql | |
| run: | | |
| mkdir -p $DATA_DIR | |
| IFS=',' read -ra LOCALE_ARRAY <<< "$LOCALES" | |
| for LOCALE in "${LOCALE_ARRAY[@]}"; do | |
| curl -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer $ACCESS_TOKEN" \ | |
| -d '{"query":"query($locale: String){catalog(id: \"2Yp0TY3kBHgG6VDjsHZNpK\",locale:$locale) {linkedFrom(allowedLocales:[\"en-US\"]) {catalogCollection(limit:20) {items {key position title linkedFrom(allowedLocales:[\"en-US\"]) {catalogCollection(limit:20) {items {key title position}}}}}}}}","variables":{"locale":"'$LOCALE'"}}' \ | |
| https://graphql.contentful.com/content/v1/spaces/ffrhttfighww | \ | |
| jq '.data.catalog.linkedFrom.catalogCollection.items' > $DATA_DIR/catalog_${LOCALE:0:2}.json | |
| done | |
| - name: Fetch product data json and download screenshots | |
| run: | | |
| mkdir -p $SCREENSHOT_DIR && IFS=',' read -ra LOCALE_ARRAY <<< "$LOCALES" && for LOCALE in "${LOCALE_ARRAY[@]}"; do | |
| SKIP=0 | |
| TOTAL=0 | |
| ITEMS=() | |
| while [ $SKIP -le $TOTAL ]; do | |
| RESPONSE=$(curl -X POST \ | |
| -H "Content-Type: application/json" \ | |
| -H "Authorization: Bearer $ACCESS_TOKEN" \ | |
| -d '{"query":"query($locale: String,$skip: Int){productCollection(locale:$locale,where:{appStore:true},limit: '$LIMIT', skip: $skip) {total items {sys {id} key hot trademark summary overview websiteurl description screenshots distribution vcpu memory production storage logo {imageurl} catalogCollection(limit:15) {items {key title catalogCollection(limit:5){items{key title position}}}}}}}", "variables":{"locale":"'$LOCALE'", "skip":'$SKIP'}}' \ | |
| https://graphql.contentful.com/content/v1/spaces/ffrhttfighww) | |
| ITEMS+=($(echo $RESPONSE | jq '.data.productCollection.items[]')) | |
| TOTAL=$(echo $RESPONSE | jq '.data.productCollection.total') | |
| SKIP=$((SKIP + LIMIT)) | |
| done | |
| echo ${ITEMS[@]} | jq -r '.screenshots[]?.value' | sed 's/"//g' | parallel -j 10 "wget -t 0 -N {} -P $SCREENSHOT_DIR/${LOCALE:0:2} || echo {} >> $SCREENSHOT_DIR/Error.log" | |
| echo ${ITEMS[@]} | jq -s '.' > $DATA_DIR/product_${LOCALE:0:2}.json | |
| done | |
| - name: Download library-dev.zip from Websoft9 artifact | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_SECRET_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_KEY }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| run: | | |
| aws s3 cp s3://artifact/dev/websoft9/plugin/library/library-dev.zip ./ --endpoint-url="https://$CLOUDFLARE_ACCOUNT_ID.r2.cloudflarestorage.com" | |
| unzip -o library-dev.zip -d ./library-dev/ | |
| - name: Merge product version data | |
| run: | | |
| IFS=',' read -ra LOCALE_ARRAY <<< "$LOCALES" | |
| for LOCALE in "${LOCALE_ARRAY[@]}"; do | |
| PRODUCT_FILE="$DATA_DIR/product_${LOCALE:0:2}.json" | |
| LIBRARY_APPS_DIR="./library-dev/library/apps" | |
| if [ -f "$PRODUCT_FILE" ]; then | |
| for APP_DIR in "$LIBRARY_APPS_DIR"/*; do | |
| if [ -d "$APP_DIR" ]; then | |
| APP_NAME=$(basename "$APP_DIR") | |
| VARIABLES_FILE="$APP_DIR/variables.json" | |
| if [ -f "$VARIABLES_FILE" ]; then | |
| EDITIONS=$(jq -c '.edition' "$VARIABLES_FILE") | |
| jq --argjson editions "$EDITIONS" --arg app_name "$APP_NAME" ' | |
| map( | |
| if .key == $app_name then | |
| .distribution = ($editions | map({key: .dist, value: .version})) | |
| else | |
| . | |
| end | |
| ) | |
| ' "$PRODUCT_FILE" > "${PRODUCT_FILE}.tmp" && mv "${PRODUCT_FILE}.tmp" "$PRODUCT_FILE" | |
| fi | |
| fi | |
| done | |
| else | |
| echo "Product file $PRODUCT_FILE not found." | |
| fi | |
| done | |
| - name: Download directory from Cloudflare R2 | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_SECRET_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_KEY }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| run: | | |
| aws s3 sync s3://libs/Websoft9/logo/product $APPSTORE_LOGO_DIR --endpoint-url="https://$CLOUDFLARE_ACCOUNT_ID.r2.cloudflarestorage.com" | |
| - name: Create Zip Archive | |
| run: | | |
| rsync -av json logos screenshots media | |
| zip -r media-dev.zip media/ | |
| mkdir media-dev | |
| mv media-dev.zip media-dev/ | |
| # - name: Upload library-dev.zip to artifacts | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: media | |
| # path: media/ | |
| - name: Upload To cloudflare r2 | |
| uses: ryand56/r2-upload-action@latest | |
| with: | |
| r2-account-id: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| r2-access-key-id: ${{ secrets.CLOUDFLARE_R2_SECRET_ID }} | |
| r2-secret-access-key: ${{ secrets.CLOUDFLARE_R2_SECRET_KEY }} | |
| r2-bucket: artifact | |
| source-dir: media-dev | |
| destination-dir: ./dev/websoft9/plugin/media | |
| - name: Purge Cloudflare Cache | |
| uses: jakejarvis/cloudflare-purge-action@master | |
| env: | |
| CLOUDFLARE_ZONE: ${{ secrets.CLOUDFLARE_ZONE_ID }} | |
| CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| PURGE_URLS: '["https://artifact.websoft9.com/dev/websoft9/plugin/media/media-dev.zip"]' |