Merge pull request #1706 from microbiomedata/issue-1612-gcs-images

Add API endpoints for handling submission-related image uploads

Author: pkalita-lbl

.env.example

@@ -38,6 +38,14 @@ NMDC_API_JWT_SECRET=generateme
 # Change this value to "testing" to run tests outside tox
 NMDC_ENVIRONMENT=development
 
+# == Cloud Storage ==
+NMDC_GCS_USE_FAKE=True
+# Get this from the Google Cloud Console; only needed if NMDC_GCS_USE_FAKE=False
+# NMDC_GCS_PROJECT_ID=changeme
+# Use `openssl rand -hex 4` to generate the random string once. Keep the `local_` prefix for local development.
+# NMDC_GCS_OBJECT_NAME_PREFIX=local_[random_string]
+
+
 # (Optional) Slack incoming webhook URL the ingester can use to post messages to Slack.
 # Reference: https://api.slack.com/messaging/webhooks#create_a_webhook
 # SLACK_WEBHOOK_URL_FOR_INGESTER=changeme

.github/workflows/server.yml

@@ -43,6 +43,17 @@ jobs:
         python-version: [3.12]
 
     steps:
+    # Use docker run directly instead of the `services` key because otherwise it's not possible
+    # to pass the `-scheme` flag.
+    # See:
+    #   - https://github.com/fsouza/fake-gcs-server/issues/561
+    #   - https://github.com/orgs/community/discussions/26688
+    - name: Start Fake GCS Server
+      run: |
+        docker run -d --name fake-gcs-server \
+          -p 4443:4443 \
+          fsouza/fake-gcs-server \
+          -scheme http
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
@@ -54,6 +65,9 @@ jobs:
         echo 'export NMDC_TESTING_DATABASE_URI="postgresql://nmdc:nmdc@localhost:5432/nmdc"' >> .env
         echo 'export NMDC_MONGO_USER="${{ secrets.NMDC_MONGO_USER }}"' >> .env
         echo 'export NMDC_MONGO_PASSWORD="${{ secrets.NMDC_MONGO_PASSWORD }}"' >> .env
+        echo 'export NMDC_GCS_USE_FAKE="true"' >> .env
+        echo 'export NMDC_GCS_OBJECT_NAME_PREFIX="testing"' >> .env
+        echo 'export NMDC_GCS_FAKE_API_ENDPOINT="http://localhost:4443"' >> .env
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

docker-compose.yml

@@ -55,6 +55,7 @@ services:
     environment:
       - PGHOST=db
       - UVICORN_RELOAD=True
+      - GOOGLE_APPLICATION_CREDENTIALS=/gcp/credentials.json
     networks:
       - public
       - default
@@ -63,6 +64,7 @@ services:
     volumes:
       - ./data/ingest:/data/ingest
       - ./nmdc_server:/app/nmdc_server
+      - $HOME/.config/gcloud/application_default_credentials.json:/gcp/credentials.json
     # Use a TTY so colors are preserved in the log output
     tty: true
 
@@ -90,9 +92,18 @@ services:
     ports:
       - "4008:4008"
 
+  storage:
+    image: fsouza/fake-gcs-server
+    ports:
+      - "4443:4443"
+    volumes:
+      - storage-data:/storage
+    command: "-scheme http -public-host localhost:4443 -filesystem-root /storage"
+
 volumes:
   app-db-data:
   app-db-ingest:
+  storage-data:
 
 networks:
   public:

docs/development.md

@@ -72,6 +72,32 @@ NMDC_MONGO_USER=changeme
 NMDC_MONGO_PASSWORD=changeme
 ```
 
+### Google Cloud Storage
+
+Google Cloud Storage (GCS) is used to store images associated with Submission Portal submissions. By default, local development uses a local mock GCS server. This is controlled by the `NMDC_GCS_USE_FAKE` variable in `.env`. If you want to use the real GCS server, set this variable to `false`.
+
+Whether you use the real or fake GCS server, you will need to set up authentication. The recommended way to do this is to use Application Default Credentials (ADC) with service account impersonation. Service account impersonation is required for generating signed URLs for uploading/downloading images directly to/from GCS.
+
+1. Ask a team member with the necessary GCS permissions to associate your Google Cloud account with the NMDC Google Cloud project and service account. 
+2. Install the Google Cloud Command Line Interface (CLI) by following the instructions at https://cloud.google.com/sdk/docs/install.
+3. Run the following command to set up Application Default Credentials (ADC):
+    ```bash
+    gcloud auth application-default login --impersonate-service-account <service account email will be provided by team member>
+    ```
+   
+You also must generate a local object name prefix. This prefix is used to differentiate which system uploaded to the shared GCS bucket. Local development systems should use the prefix `local_<random_suffix>`.
+
+1. Generate a random suffix using the following command:
+    ```bash
+    openssl rand -hex 4
+    ```
+2. Set the `NMDC_GCS_OBJECT_NAME_PREFIX` variable in `.env` to `local_<random_suffix>`.
+
+    ```bash
+   NMDC_GCS_OBJECT_NAME_PREFIX=local_1234abcd  # replace 1234abcd with your random suffix
+   ```
+
+
 ## Load production data
 
 The `nmdc-server` CLI has a `load-db` subcommand which populates your local database using a nightly production backup. These backups are stored on NERSC. You must have NERSC credentials to use this subcommand.