Merge branch 'main' into issue-2762-shareable-workflow

Author: emma-sg

.github/workflows/deploy-dev.yaml

@@ -63,6 +63,15 @@ jobs:
           cache-from: type=gha,scope=backend
           cache-to: type=gha,scope=backend,mode=max
 
+      - name: Build Emails
+        uses: docker/build-push-action@v3
+        with:
+          context: emails
+          push: true
+          tags: ${{ secrets.DO_REGISTRY_PATH }}/webrecorder/browsertrix-emails:latest
+          cache-from: type=gha,scope=emails
+          cache-to: type=gha,scope=emails,mode=max
+
       - name: Build Frontend
         uses: docker/build-push-action@v3
         env:

.github/workflows/emails-build.yaml

@@ -0,0 +1,77 @@
+name: Emails Build
+on: pull_request
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # In order to be able to have required checks, a workflow can't be entirely
+  # skipped: see https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks
+  paths-filter:
+    name: "Changed files?"
+    runs-on: ubuntu-latest
+    outputs:
+      matches: ${{ steps.filter.outputs.matches }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2 # important, to fetch previous commit
+
+      # workaround for https://github.com/dorny/paths-filter/issues/240
+      - id: previous-sha
+        run: 'echo "sha=$(git rev-parse HEAD^1)" >> $GITHUB_OUTPUT'
+
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          base: "${{ steps.previous-sha.outputs.sha }}"
+          filters: |
+            matches:
+              - 'emails/**'
+              - '.github/workflows/emails-build.yaml'
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: paths-filter
+    if: needs.paths-filter.outputs.matches == 'true'
+    strategy:
+      matrix:
+        node: [20, 22]
+    steps:
+      # Setup:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: "yarn"
+          cache-dependency-path: emails/yarn.lock
+
+      - name: Install dependencies
+        working-directory: emails
+        env:
+          HUSKY: 0
+        run: yarn install --frozen-lockfile
+
+      # Check build:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
+      - name: Build Email Service
+        uses: docker/build-push-action@v3
+        with:
+          context: emails
+          load: true
+          tags: webrecorder/browsertrix-emails:latest
+          cache-from: type=gha,scope=emails
+          cache-to: type=gha,scope=emails,mode=max

.github/workflows/k3d-ci.yaml

@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 2  # important, to fetch previous commit
+          fetch-depth: 2 # important, to fetch previous commit
 
       # workaround for https://github.com/dorny/paths-filter/issues/240
       - id: previous-sha
@@ -36,6 +36,7 @@ jobs:
             matches:
               - 'backend/**'
               - 'chart/**'
+              - 'emails/**'
               - '.github/workflows/k3d-ci.yaml'
 
   btrix-k3d-test:
@@ -51,6 +52,7 @@ jobs:
           cluster-name: btrix-1
           args: >-
             -p "30870:30870@agent:0:direct"
+            -p "30872:30872@agent:0:direct"
             --agents 1
             --no-lb
             --k3s-arg "--disable=traefik,servicelb,metrics-server@server:*"
@@ -73,6 +75,16 @@ jobs:
           cache-from: type=gha,scope=backend
           cache-to: type=gha,scope=backend,mode=max
 
+      - name: Build Emails
+        uses: docker/build-push-action@v3
+        with:
+          context: emails
+          load: true
+          #outputs: type=tar,dest=emails.tar
+          tags: webrecorder/browsertrix-emails:latest
+          cache-from: type=gha,scope=emails
+          cache-to: type=gha,scope=emails,mode=max
+
       - name: Build Frontend
         uses: docker/build-push-action@v3
         with:
@@ -83,15 +95,16 @@ jobs:
           cache-from: type=gha,scope=frontend
           cache-to: type=gha,scope=frontend,mode=max
 
-      - name: 'Import Images'
+      - name: "Import Images"
         run: |
           k3d image import webrecorder/browsertrix-backend:latest -m direct -c btrix-1 --verbose
+          k3d image import webrecorder/browsertrix-emails:latest -m direct -c btrix-1 --verbose
           k3d image import webrecorder/browsertrix-frontend:latest -m direct -c btrix-1 --verbose
 
       - name: Install Kubectl
         uses: azure/setup-kubectl@v3
         with:
-            version: v1.28.2
+          version: v1.28.2
 
       - name: Install Helm
         uses: azure/setup-helm@v3
@@ -121,13 +134,17 @@ jobs:
         run: pytest -vv ./backend/test/test_*.py
 
       - name: Print Backend Logs (API)
-        if: ${{ failure() }}
+        if: ${{ always() }}
         run: kubectl logs svc/browsertrix-cloud-backend -c api
 
       - name: Print Backend Logs (Operator)
-        if: ${{ failure() }}
+        if: ${{ always() }}
         run: kubectl logs svc/browsertrix-cloud-backend -c op
 
+      - name: Print Emails Logs
+        if: ${{ always() }}
+        run: kubectl logs svc/browsertrix-cloud-emails -c api
+
       - name: Print K8S Events
-        if: ${{ failure() }}
+        if: ${{ always() }}
         run: kubectl events --all-namespaces

.github/workflows/k3d-log-ci.yaml

@@ -5,8 +5,9 @@ on:
     branches:
       - main
     paths:
-      - 'backend/**'
-      - 'chart/**'
+      - "backend/**"
+      - "chart/**"
+      - "emails/**"
 
 jobs:
   btrix-k3d-admin-logging-test:
@@ -30,7 +31,7 @@ jobs:
       - name: Install Helm
         uses: azure/setup-helm@v3
         with:
-           version: 3.10.2
+          version: 3.10.2
 
       - name: Add hosts to /etc/hosts
         run: |
@@ -43,7 +44,7 @@ jobs:
           POD=$(kubectl get pods -A | grep nginx-ingress | head -n 1 | awk '{print $2}')
           while [[ $(kubectl get pods $POD -n kube-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do kubectl get pods -n kube-system; sleep 5; done
           sleep 20
- 
+
       - name: Start Cluster with Helm (chart/admin/logging)
         run: |
           ./chart/admin/logging/scripts/eck_install.sh

.github/workflows/k3d-nightly-ci.yaml

@@ -3,7 +3,7 @@ name: Nightly tests (K3d)
 on:
   schedule:
     # Run daily at 8am UTC
-    - cron:  '0 8 * * *'
+    - cron: "0 8 * * *"
 
   workflow_dispatch:
 
@@ -59,6 +59,16 @@ jobs:
           cache-from: type=gha,scope=backend
           cache-to: type=gha,scope=backend,mode=max
 
+      - name: Build Emails
+        uses: docker/build-push-action@v3
+        with:
+          context: emails
+          load: true
+          #outputs: type=tar,dest=emails.tar
+          tags: webrecorder/browsertrix-emails:latest
+          cache-from: type=gha,scope=emails
+          cache-to: type=gha,scope=emails,mode=max
+
       - name: Build Frontend
         uses: docker/build-push-action@v3
         with:
@@ -69,9 +79,10 @@ jobs:
           cache-from: type=gha,scope=frontend
           cache-to: type=gha,scope=frontend,mode=max
 
-      - name: 'Import Images'
+      - name: "Import Images"
         run: |
           k3d image import webrecorder/browsertrix-backend:latest -m direct -c btrix-nightly --verbose
+          k3d image import webrecorder/browsertrix-emails:latest -m direct -c btrix-nightly --verbose
           k3d image import webrecorder/browsertrix-frontend:latest -m direct -c btrix-nightly --verbose
 
       - name: Install Kubectl
@@ -89,7 +100,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
+          python-version: "3.x"
 
       - name: Install Python Libs
         run: pip install -r ./backend/test-requirements.txt
@@ -98,15 +109,19 @@ jobs:
         run: kubectl wait --for=condition=ready pod --all --timeout=240s
 
       - name: Create Extra Test Buckets
-        run:  kubectl exec -i deployment/local-minio -c minio -- mkdir /data/replica-0
+        run: kubectl exec -i deployment/local-minio -c minio -- mkdir /data/replica-0
 
       - name: Run Tests
         run: pytest -vv ./backend/test_nightly/${{ matrix.module }}
 
       - name: Print Backend Logs (API)
-        if: ${{ failure() }}
+        if: ${{ always() }}
         run: kubectl logs svc/browsertrix-cloud-backend -c api
 
       - name: Print Backend Logs (Operator)
-        if: ${{ failure() }}
+        if: ${{ always() }}
         run: kubectl logs svc/browsertrix-cloud-backend -c op
+
+      - name: Print Emails Logs
+        if: ${{ always() }}
+        run: kubectl logs svc/browsertrix-cloud-emails -c api

.github/workflows/microk8s-ci.yaml

@@ -5,9 +5,9 @@ on:
     branches:
       - main
     paths:
-      - 'backend/**'
-      - 'chart/**'
-      - '.github/workflows/microk8s-ci.yaml'
+      - "backend/**"
+      - "chart/**"
+      - ".github/workflows/microk8s-ci.yaml"
   workflow_dispatch:
 
 env:
@@ -19,7 +19,7 @@ jobs:
     steps:
       - uses: balchua/microk8s-actions@v0.3.1
         with:
-          channel: '1.25/stable'
+          channel: "1.25/stable"
           addons: '["dns", "helm3", "hostpath-storage", "registry", "host-access"]'
       - name: Checkout
         uses: actions/checkout@v3
@@ -41,6 +41,15 @@ jobs:
           cache-from: type=gha,scope=backend
           cache-to: type=gha,scope=backend,mode=max
 
+      - name: Build Emails
+        uses: docker/build-push-action@v3
+        with:
+          context: emails
+          push: true
+          tags: localhost:32000/webrecorder/browsertrix-emails:latest
+          cache-from: type=gha,scope=emails
+          cache-to: type=gha,scope=emails,mode=max
+
       - name: Build Frontend
         uses: docker/build-push-action@v3
         with:
@@ -60,7 +69,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
+          python-version: "3.x"
 
       - name: Install Python Libs
         run: pip install -r ./backend/test-requirements.txt
@@ -72,5 +81,9 @@ jobs:
         run: pytest -vv ./backend/test/test_*.py
 
       - name: Print Backend Logs
-        if: ${{ failure() }}
+        if: ${{ always() }}
         run: sudo microk8s kubectl logs svc/browsertrix-cloud-backend
+
+      - name: Print Emails Logs
+        if: ${{ always() }}
+        run: sudo microk8s kubectl logs svc/browsertrix-cloud-emails

.github/workflows/release.yaml

@@ -9,34 +9,28 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v3
 
-      -
-        name: Set up QEMU
+      - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
-      -
-        name: Set up Docker Buildx
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      -
-        name: Login to Docker Hub
+      - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      -
-        name: Set Env Vars
+      - name: Set Env Vars
         run: |
           echo VERSION=`cat version.txt` >> $GITHUB_ENV
           echo GIT_COMMIT_HASH=`git rev-parse --short HEAD` >> $GITHUB_ENV
           echo GIT_BRANCH_NAME=`git rev-parse --abbrev-ref HEAD` >> $GITHUB_ENV
 
-      -
-        name: Build Backend
+      - name: Build Backend
         uses: docker/build-push-action@v4
         with:
           context: backend
@@ -46,8 +40,17 @@ jobs:
           cache-from: type=gha,scope=backend
           cache-to: type=gha,scope=backend,mode=max
 
-      -
-        name: Build Frontend
+      - name: Build Emails
+        uses: docker/build-push-action@v4
+        with:
+          context: emails
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.REPO_PREFIX }}webrecorder/browsertrix-emails:${{ env.VERSION }}, webrecorder/browsertrix-emails:latest
+          cache-from: type=gha,scope=emails
+          cache-to: type=gha,scope=emails,mode=max
+
+      - name: Build Frontend
         uses: docker/build-push-action@v4
         env:
           HUSKY: 0