m

Author: ceroberoz

.github/workflows/scrape.yml

@@ -1,64 +1,32 @@
-name: Scrape Job Data and Upload to Google Sheets
+ame: Scrape and Upload to Google Sheets
 
 on:
   push:
-    branches:
-      - master
-  pull_request_target:
+    branches: [master]
+  pull_request:
+    branches: [master]
   schedule:
     - cron: "0 0 * * *"
-  workflow_dispatch:
 
 jobs:
   scrape-and-upload:
     runs-on: ubuntu-latest
-    permissions:
-      contents: write
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.MEONG }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.head_ref }}
-
+      - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.x"
-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install scrapy google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client
-
-      # - name: Run scrape.sh
-      #   run: bash scrape.sh
-
-      # - name: Check for CSV files
-      #   run: |
-      #     if ls output/*.csv 1> /dev/null 2>&1; then
-      #       echo "CSV files found."
-      #     else
-      #       echo "No CSV files found. Exiting."
-      #       exit 1
-      #     fi
-
-      # - name: Move CSV files to public directory
-      #   run: |
-      #     mkdir -p public
-      #     mv output/*.csv public/
-
+          pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client
+      - name: Run scraper
+        run: |
+          # Your scraping script here
+          # This should create public/merged.csv
       - name: Upload to Google Sheets
         env:
           GCP_JSON: ${{ secrets.GCP_JSON }}
           GOOGLE_SHEETS_ID: ${{ secrets.GOOGLE_SHEETS_ID }}
         run: python upload_to_sheets.py
-
-      - name: Commit and push changes
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git add public/*.csv
-          git commit -m "[skip ci] Updated job data" || echo "No changes to commit"
-          git push

upload_to_sheets.py

@@ -1,40 +1,31 @@
 import os
 import csv
 import json
-import sys
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 
+# If modifying these scopes, update the GCP_JSON secret accordingly.
+SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]
+
 def get_env_var(var_name):
     value = os.environ.get(var_name)
     if value is None:
-        print(f"Error: {var_name} environment variable is not set")
-        sys.exit(1)
+        raise ValueError(f"{var_name} environment variable is not set")
     return value
 
 def setup_credentials():
-    creds_json = get_env_var('GCP_JSON')  # Changed from GCP_SA_KEY to GCP_JSON
-    try:
-        creds_dict = json.loads(creds_json)
-        return service_account.Credentials.from_service_account_info(
-            creds_dict,
-            scopes=['https://www.googleapis.com/auth/spreadsheets']
-        )
-    except json.JSONDecodeError:
-        print("Error: Invalid JSON in GCP_JSON")  # Changed error message
-        sys.exit(1)
+    gcp_json = get_env_var('GCP_JSON')
+    creds_dict = json.loads(gcp_json)
+    return service_account.Credentials.from_service_account_info(
+        creds_dict, scopes=SCOPES)
 
 def read_csv(file_path):
-    try:
-        with open(file_path, 'r') as file:
-            return list(csv.reader(file))
-    except FileNotFoundError:
-        print(f"Error: CSV file not found at {file_path}")
-        sys.exit(1)
+    with open(file_path, 'r') as file:
+        return list(csv.reader(file))
 
 def upload_to_sheets(service, spreadsheet_id, data):
-    sheet_range = 'Sheet1'
+    sheet_range = 'Sheet1'  # Update this if you want to use a different sheet name
     body = {'values': data}
 
     try:
@@ -52,19 +43,20 @@ def upload_to_sheets(service, spreadsheet_id, data):
             body=body
         ).execute()
         print(f"{result.get('updatedCells')} cells updated.")
-    except HttpError as e:
-        print(f"HTTP error occurred: {e}")
-        if e.resp.status == 404:
-            print("Error 404: Make sure the Google Sheets ID is correct and the service account has access to the sheet.")
-        sys.exit(1)
+    except HttpError as err:
+        print(f"An error occurred: {err}")
+        raise
 
 def main():
-    creds = setup_credentials()
-    service = build('sheets', 'v4', credentials=creds)
-    spreadsheet_id = get_env_var('GOOGLE_SHEETS_ID')
-
-    csv_content = read_csv('public/merged.csv')
-    upload_to_sheets(service, spreadsheet_id, csv_content)
+    try:
+        creds = setup_credentials()
+        service = build("sheets", "v4", credentials=creds)
+        spreadsheet_id = get_env_var('GOOGLE_SHEETS_ID')
+
+        csv_content = read_csv('public/merged.csv')
+        upload_to_sheets(service, spreadsheet_id, csv_content)
+    except HttpError as err:
+        print(err)
 
 if __name__ == "__main__":
     main()