diff --git a/.circleci/config.yml b/.circleci/config.yml index a461abd9454..072010cada8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,13 +14,13 @@ references: working_directory: ~/addons-frontend docker: # This is the NodeJS version we run in production. - - image: cimg/node:18.20 + - image: cimg/node:20.18 defaults-next: &defaults-next working_directory: ~/addons-frontend docker: # This is the next NodeJS version we will support. - - image: cimg/node:20.12 + - image: cimg/node:22.14 defaults-release: &defaults-release machine: @@ -211,9 +211,15 @@ jobs: <<: *defaults-release steps: - better_checkout - - run: pip install tox - - run: TOXENV=dennis-lint tox - + - run: + name: Install Dennis + command: pip install dennis==1.1.0 + - run: + name: Lint locales + command: | + bash -c 'dennis-cmd lint locale/*/LC_MESSAGES/*.po --rules W202' + bash -c 'dennis-cmd lint --errorsonly locale/*/LC_MESSAGES/*.po' + bash -c 'dennis-cmd lint --errorsonly locale/templates/LC_MESSAGES/*.pot' release-blog-utils: <<: *defaults @@ -235,9 +241,6 @@ jobs: - run: name: Install dependencies command: sudo apt-get update; sudo apt-get install pigz - - run: - name: "Install Tox" - command: pip install tox - run: docker info # Build the container, using Circle's Docker cache. Only use 1 image per # day to keep the cache size down. @@ -271,6 +274,12 @@ jobs: command: | IMG="image-$(date +%j).gz"; docker save addons-frontend | pigz --fast -c > ~/addons-frontend/docker/$IMG; ls -l ~/addons-frontend/docker ls -l ~/addons-frontend/docker + - run: + name: Check image works + command: | + docker run --rm -p 4000:4000 -d -e NODE_ENV=production -e NODE_CONFIG_ENV=prod addons-frontend sh -c "yarn build && yarn start" + sleep 60 + curl --retry 3 --retry-delay 2 http://127.0.0.1:4000/__frontend_lbheartbeat__ - run: name: Push to repo command: | @@ -279,17 +288,6 @@ jobs: docker tag addons-frontend $DOCKERHUB_REPO:$CIRCLE_TAG docker images docker push $DOCKERHUB_REPO:$CIRCLE_TAG - - run: - name: Set hosts - command: | - echo 127.0.0.1 olympia.test | sudo tee -a /etc/hosts - cat /etc/hosts - - run: - name: Test Image - command: | - sudo sysctl -w vm.max_map_count=262144 - ./tests/smoke/setup_docker.sh - tox -e smoke-tests release-master: # build for the master branch <<: *defaults-release @@ -298,9 +296,6 @@ jobs: - run: name: Install dependencies command: sudo apt-get update; sudo apt-get install pigz - - run: - name: "Install Tox" - command: pip install tox - run: docker info # Build the container, using Circle's Docker cache. Only use 1 image per # day to keep the cache size down. @@ -334,23 +329,18 @@ jobs: command: | IMG="image-$(date +%j).gz"; docker save addons-frontend | pigz --fast -c > ~/addons-frontend/docker/$IMG; ls -l ~/addons-frontend/docker ls -l ~/addons-frontend/docker + - run: + name: Check image works + command: | + docker run --rm -p 4000:4000 -d -e NODE_ENV=production -e NODE_CONFIG_ENV=prod addons-frontend sh -c "yarn build && yarn start" + sleep 60 + curl --retry 3 --retry-delay 2 http://127.0.0.1:4000/__frontend_lbheartbeat__ - run: name: Push to repo command: | [ ! -z $DOCKERHUB_REPO ] docker tag addons-frontend $DOCKERHUB_REPO:latest docker push $DOCKERHUB_REPO:latest - - run: - name: Set hosts - command: | - echo 127.0.0.1 olympia.test | sudo tee -a /etc/hosts - cat /etc/hosts - - run: - name: Test Image - command: | - sudo sysctl -w vm.max_map_count=262144 - ./tests/smoke/setup_docker.sh - tox -e smoke-tests workflows: version: 2 diff --git a/.github/actions/context/action.yml b/.github/actions/context/action.yml new file mode 100644 index 00000000000..7604b6ecb3e --- /dev/null +++ b/.github/actions/context/action.yml @@ -0,0 +1,114 @@ +name: 'Dump Context' +description: 'Display context for action run' + +outputs: + # All github action outputs are strings, even if set to "true" + # so when using these values always assert against strings or convert from json + # \$\{{ needs.context.outputs.is_fork == 'true' }} // true + # \$\{{ fromJson(needs.context.outputs.is_fork) == false }} // true + # \$\{{ needs.context.outputs.is_fork == true }} // false + # \$\{{ needs.context.outputs.is_fork }} // false + is_fork: + description: "" + value: ${{ steps.context.outputs.is_fork }} + is_default_branch: + description: "" + value: ${{ steps.context.outputs.is_default_branch }} + is_release_master: + description: "" + value: ${{ steps.context.outputs.is_release_master }} + is_release_tag: + description: "" + value: ${{ steps.context.outputs.is_release_tag }} + +runs: + using: 'composite' + steps: + - name: Dump GitHub context + shell: bash + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Dump job context + shell: bash + env: + JOB_CONTEXT: ${{ toJson(job) }} + run: echo "$JOB_CONTEXT" + - name: Dump steps context + shell: bash + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + run: echo "$STEPS_CONTEXT" + - name: Dump runner context + shell: bash + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + run: echo "$RUNNER_CONTEXT" + - name: Dump env context + shell: bash + env: + ENV_CONTEXT: ${{ toJson(env) }} + run: | + echo "$ENV_CONTEXT" + - name: Dump inputs context + shell: bash + env: + INPUTS_CONTEXT: ${{ toJson(inputs) }} + run: | + echo "$INPUTS_CONTEXT" + + - name: Set context + id: context + env: + # The default branch of the repository, in this case "master" + default_branch: ${{ github.event.repository.default_branch }} + shell: bash + run: | + event_name="${{ github.event_name }}" + event_action="${{ github.event.action }}" + + # Stable check for if the workflow is running on the default branch + # https://stackoverflow.com/questions/64781462/github-actions-default-branch-variable + is_default_branch="${{ format('refs/heads/{0}', env.default_branch) == github.ref }}" + + # In most events, the epository refers to the head which would be the fork + is_fork="${{ github.event.repository.fork }}" + + # This is different in a pull_request where we need to check the head explicitly + if [[ "${{ github.event_name }}" == 'pull_request' ]]; then + # repository on a pull request refers to the base which is always mozilla/addons-server + is_head_fork="${{ github.event.pull_request.head.repo.fork }}" + # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions + is_dependabot="${{ github.actor == 'dependabot[bot]' }}" + + # If the head repository is a fork or if the PR is opened by dependabot + # we consider the run to be a fork. Dependabot and proper forks are treated + # the same in terms of limited read only github token scope + if [[ "$is_head_fork" == 'true' || "$is_dependabot" == 'true' ]]; then + is_fork="true" + fi + fi + + is_release_master="false" + is_release_tag="false" + + # Releases can only happen if we are NOT on a fork + if [[ "$is_fork" == 'false' ]]; then + # A master release occurs on a push to the default branch of the origin repository + if [[ "$event_name" == 'push' && "$is_default_branch" == 'true' ]]; then + is_release_master="true" + fi + + # A tag release occurs when a release is published + if [[ "$event_name" == 'release' && "$event_action" == 'publish' ]]; then + is_release_tag="true" + fi + fi + + echo "is_default_branch=$is_default_branch" >> $GITHUB_OUTPUT + echo "is_fork=$is_fork" >> $GITHUB_OUTPUT + echo "is_release_master=$is_release_master" >> $GITHUB_OUTPUT + echo "is_release_tag=$is_release_tag" >> $GITHUB_OUTPUT + + echo "event_name: $event_name" + cat $GITHUB_OUTPUT diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 34b19b515ba..56128e086e0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,10 @@ updates: schedule: interval: "daily" open-pull-requests-limit: 99 + ignore: + - dependency-name: "ua-parser-js" + versions: ["2.x"] + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: daily diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000000..ae0d7421d83 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,68 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + +jobs: + context: + runs-on: ubuntu-latest + + outputs: + is_fork: ${{ steps.context.outputs.is_fork }} + + steps: + - uses: actions/checkout@v4 + - id: context + uses: ./.github/actions/context + + locales: + needs: context + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + - uses: actions/setup-node@v4 + with: + node-version: 18 + cache: 'yarn' + + - name: Install gettext + run: sudo apt-get install gettext + + - name: Yarn install + run: yarn install --frozen-lockfile --prefer-offline + + - name: Extract locales + run: yarn extract-locales + + - name: Push Locales + run: | + event_name="${{ github.event_name }}" + is_fork="${{ needs.context.outputs.is_fork }}" + + if [[ "$is_fork" == 'true' ]]; then + cat <<'EOF' + Github actions are not authorized to push from workflows triggered by forks. + We cannot verify if the l10n extraction push will work or not. + Please submit a PR from the base repository if you are modifying l10n extraction scripts. + EOF + exit 0 + fi + + ARGS="" + + if [[ "$event_name" == 'pull_request' ]]; then + ARGS="--dry-run" + fi + + ./bin/push-locales $ARGS + + + diff --git a/.prettierignore b/.prettierignore index b7923be159e..a65d5d6fe25 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,6 +2,7 @@ *.* # exclude these files Dockerfile +src/fonts/LICENSE # exclude these directories /assets/ /bin/ diff --git a/Dockerfile b/Dockerfile index 3d5a44dabc7..1c7d285c58d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # # Build # -FROM node:18.20-slim AS builder +FROM node:20.18-slim AS builder WORKDIR /srv/node COPY package.json yarn.lock /srv/node/ @@ -12,7 +12,7 @@ RUN yarn install --pure-lockfile # # Install # -FROM node:18.20-slim +FROM node:20.18-slim ARG app_uid=9500 ARG app_dir=/app diff --git a/babel.config.locales.js b/babel.config.locales.js new file mode 100644 index 00000000000..c05c53235ce --- /dev/null +++ b/babel.config.locales.js @@ -0,0 +1,41 @@ +// Create UTC creation date in the correct format. +const potCreationDate = new Date() + .toISOString() + .replace('T', ' ') + .replace(/:\d{2}.\d{3}Z/, '+0000'); + +module.exports = { + extends: './babel.config.js', + plugins: [ + [ + 'module:babel-gettext-extractor', + { + headers: { + 'Project-Id-Version': 'amo', + 'Report-Msgid-Bugs-To': 'EMAIL@ADDRESS', + 'POT-Creation-Date': potCreationDate, + 'PO-Revision-Date': 'YEAR-MO-DA HO:MI+ZONE', + 'Last-Translator': 'FULL NAME ', + 'Language-Team': 'LANGUAGE ', + 'MIME-Version': '1.0', + 'Content-Type': 'text/plain; charset=utf-8', + 'Content-Transfer-Encoding': '8bit', + 'plural-forms': 'nplurals=2; plural=(n!=1);', + }, + functionNames: { + gettext: ['msgid'], + dgettext: ['domain', 'msgid'], + ngettext: ['msgid', 'msgid_plural', 'count'], + dngettext: ['domain', 'msgid', 'msgid_plural', 'count'], + pgettext: ['msgctxt', 'msgid'], + dpgettext: ['domain', 'msgctxt', 'msgid'], + npgettext: ['msgctxt', 'msgid', 'msgid_plural', 'count'], + dnpgettext: ['domain', 'msgctxt', 'msgid', 'msgid_plural', 'count'], + }, + fileName: './locale/templates/LC_MESSAGES/amo.pot', + baseDirectory: process.cwd(), + stripTemplateLiteralIndent: true, + }, + ], + ], +}; diff --git a/bin/extract-locales b/bin/extract-locales index 4d00c81a105..282d7b10d08 100755 --- a/bin/extract-locales +++ b/bin/extract-locales @@ -1,3 +1,59 @@ -#!/usr/bin/env sh +#!/usr/bin/env zx + +import {$, path, echo, within, glob} from 'zx'; + +const root = path.join(__dirname, '..'); +const localeDir = path.join(root, 'locale'); +const templateFile = path.join(localeDir, '/templates/LC_MESSAGES/amo.pot'); + +within(async () => { + echo('Extracting locales...'); + + const sourceDir = path.join(root, 'src', 'amo'); + const outputDir = path.join(root, 'dist', 'locales'); + const localesConfig = path.join(root, 'babel.config.locales.js'); + + await $`babel ${sourceDir} \ + --out-dir ${outputDir} \ + --config-file ${localesConfig} \ + --verbose \ + `; + + const {stdout: output} = await $`git diff --numstat -- ${templateFile}`; + + // git diff --numstat returns the number of insertions and deletions for each file + // this regex extracts the numbers from the output + const regex = /([0-9]+).*([0-9]+)/; + + const [, insertions = 0, deletions = 0] = output.match(regex) || []; + + const isLocaleClean = insertions < 2 && deletions < 2; + + if (isLocaleClean) { + return echo('No locale changes, nothing to update, ending process'); + } + + echo(`Found ${insertions} insertions and ${deletions} deletions in ${templateFile}.`); + + const poFiles = await glob(`${localeDir}/**/amo.po`); + + echo(`Merging ${poFiles.length} translation files.`); + + for await (const poFile of poFiles) { + const dir = path.dirname(poFile); + const stem = path.basename(poFile, '.po'); + const tempFile = path.join(dir, `${stem}.po.tmp`); + echo(`merging: ${poFile}`); + + try { + await $`msgmerge --no-fuzzy-matching -q -o ${tempFile} ${poFile} ${templateFile}` + await $`mv ${tempFile} ${poFile}` + } catch (error) { + await $`rm ${tempFile}`; + throw new Error(`Error merging ${poFile}`); + } + } + + return true; +}); -yarn extract-locales diff --git a/bin/loading-page.html b/bin/loading-page.html index 66564863d0a..93b5d5daed9 100644 --- a/bin/loading-page.html +++ b/bin/loading-page.html @@ -6,7 +6,7 @@ Add-ons for Firefox: Webpack Loading @@ -99,7 +99,7 @@