further cleanup:

- consolidate UserUploadFile into SeedFile, can split again if needed
- add get_absolute_presigned_url() to UserFile for convenience in getting absolute url directly

Author: ikreymer

backend/btrixcloud/crawlconfigs.py

@@ -282,7 +282,9 @@ async def add_crawl_config(
 
         if config_in.config.seedFileId:
             # Validate file with that id exists
-            seed_file = await self.file_ops.get_file(config_in.config.seedFileId, org)
+            seed_file = await self.file_ops.get_seed_file(
+                config_in.config.seedFileId, org
+            )
 
             # Validate seeds not set
             if config_in.config.seeds:
@@ -504,7 +506,7 @@ async def update_crawl_config(
 
         if update.config and update.config.seedFileId:
             # Validate file with that id exists
-            seed_file = await self.file_ops.get_file(update.config.seedFileId, org)
+            seed_file = await self.file_ops.get_seed_file(update.config.seedFileId, org)
 
             # Validate seeds not set
             if update.config.seeds or (
@@ -636,7 +638,7 @@ async def update_crawl_config(
             and update.config.seedFileId is None
         ):
             try:
-                await self.file_ops.delete_user_file(
+                await self.file_ops.delete_seed_file(
                     orig_crawl_config.config.seedFileId, org
                 )
             except HTTPException:
@@ -976,7 +978,7 @@ async def make_inactive_or_delete(
         if not crawlconfig.crawlAttemptCount:
             if crawlconfig.config and crawlconfig.config.seedFileId:
                 try:
-                    await self.file_ops.delete_user_file(
+                    await self.file_ops.delete_seed_file(
                         crawlconfig.config.seedFileId, org
                     )
                 except HTTPException:
@@ -1127,7 +1129,7 @@ async def run_now_internal(
                     status_code=400, detail="seed_file_not_supported_by_crawler"
                 )
 
-            seed_file_out = await self.file_ops.get_file_out(
+            seed_file_out = await self.file_ops.get_seed_file_out(
                 crawlconfig.config.seedFileId, org
             )
             seed_file_url = seed_file_out.path

backend/btrixcloud/file_uploads.py

@@ -12,8 +12,7 @@
 import pymongo
 
 from .models import (
-    UserUploadFile,
-    UserUploadFileOut,
+    SeedFileOut,
     SeedFile,
     UserFile,
     UserFilePreparer,
@@ -88,36 +87,36 @@ async def get_file_raw(
 
         return res
 
-    async def get_file(
+    async def get_seed_file(
         self,
         file_id: UUID,
         org: Optional[Organization] = None,
         type_: Optional[str] = None,
-    ) -> UserUploadFile:
+    ) -> SeedFile:
         """Get file by UUID"""
         file_raw = await self.get_file_raw(file_id, org, type_)
-        return UserUploadFile.from_dict(file_raw)
+        return SeedFile.from_dict(file_raw)
 
-    async def get_file_out(
+    async def get_seed_file_out(
         self,
         file_id: UUID,
         org: Optional[Organization] = None,
         type_: Optional[str] = None,
         headers: Optional[dict] = None,
-    ) -> UserUploadFileOut:
+    ) -> SeedFileOut:
         """Get file output model by UUID"""
-        user_file = await self.get_file(file_id, org, type_)
+        user_file = await self.get_seed_file(file_id, org, type_)
         return await user_file.get_file_out(org, self.storage_ops, headers)
 
-    async def list_user_files(
+    async def list_seed_files(
         self,
         org: Organization,
         page_size: int = DEFAULT_PAGE_SIZE,
         page: int = 1,
         sort_by: str = "created",
         sort_direction: int = -1,
         headers: Optional[dict] = None,
-    ) -> Tuple[list[UserUploadFileOut], int]:
+    ) -> Tuple[list[SeedFileOut], int]:
         """list all user-uploaded files"""
         # pylint: disable=too-many-locals
 
@@ -171,7 +170,7 @@ async def list_user_files(
 
         user_files = []
         for res in items:
-            file_ = UserUploadFile.from_dict(res)
+            file_ = SeedFile.from_dict(res)
             file_out = await file_.get_file_out(org, self.storage_ops, headers)
             user_files.append(file_out)
 
@@ -278,12 +277,13 @@ async def _parse_seed_info_from_file(
         first_seed = ""
         seed_count = 0
 
-        file_out = await file_obj.get_file_out(org, self.storage_ops)
-        print("PATH", file_out.path)
+        file_url = await file_obj.get_absolute_presigned_url(
+            org, self.storage_ops, None
+        )
 
         with tempfile.TemporaryFile() as fp:
             async with aiohttp.ClientSession() as session:
-                async with session.get(file_out.path) as resp:
+                async with session.get(file_url) as resp:
                     async for chunk in resp.content.iter_chunked(CHUNK_SIZE):
                         fp.write(chunk)
 
@@ -299,11 +299,11 @@ async def _parse_seed_info_from_file(
 
         return first_seed, seed_count
 
-    async def delete_user_file(
+    async def delete_seed_file(
         self, file_id: UUID, org: Organization
     ) -> Dict[str, bool]:
         """Delete user-uploaded file from storage and db"""
-        file = await self.get_file(file_id, org)
+        file = await self.get_seed_file(file_id, org)
 
         # Make sure seed file isn't currently referenced by any workflows
         if file.type == "seedFile":
@@ -358,7 +358,7 @@ async def cleanup_unused_seed_files(self):
 
             try:
                 org = await self.org_ops.get_org_by_id(file_dict["oid"])
-                await self.delete_user_file(file_id, org)
+                await self.delete_seed_file(file_id, org)
                 print(f"Deleted unused seed file {file_id}", flush=True)
             # pylint: disable=broad-exception-caught
             except Exception as err:
@@ -401,7 +401,7 @@ async def upload_seedfile_stream(
 
     # pylint: disable=too-many-arguments
     @router.get("", response_model=PaginatedUserFileResponse)
-    async def list_user_files(
+    async def list_seed_files(
         request: Request,
         org: Organization = Depends(org_viewer_dep),
         pageSize: int = DEFAULT_PAGE_SIZE,
@@ -410,7 +410,7 @@ async def list_user_files(
         sortDirection: int = -1,
     ):
         # pylint: disable=duplicate-code
-        user_files, total = await ops.list_user_files(
+        user_files, total = await ops.list_seed_files(
             org,
             page_size=pageSize,
             page=page,
@@ -420,17 +420,17 @@ async def list_user_files(
         )
         return paginated_format(user_files, total, page, pageSize)
 
-    @router.get("/{file_id}", response_model=UserUploadFileOut)
-    async def get_user_file(
+    @router.get("/{file_id}", response_model=SeedFileOut)
+    async def get_seed_file(
         file_id: UUID, request: Request, org: Organization = Depends(org_viewer_dep)
     ):
-        return await ops.get_file_out(file_id, org, headers=dict(request.headers))
+        return await ops.get_seed_file_out(file_id, org, headers=dict(request.headers))
 
     @router.delete("/{file_id}", response_model=SuccessResponse)
     async def delete_user_file(
         file_id: UUID, org: Organization = Depends(org_crawl_dep)
     ):
-        return await ops.delete_user_file(file_id, org)
+        return await ops.delete_seed_file(file_id, org)
 
     if org_ops.router:
         org_ops.router.include_router(router)

backend/btrixcloud/models.py

@@ -1146,16 +1146,14 @@ def prepare_filename(self, filename):
 
 # ============================================================================
 
-### USER-UPLOADED IMAGES ###
+### USER-UPLOADED FILES ###
 
 
 # ============================================================================
-class UserFileOut(BaseModel):
-    """output for user-uploaded file as stored on other document
+class PublicUserFileOut(BaseModel):
+    """public output for user-uploaded file stored on other document
 
-    Used for collection thumbnails.
-    Should merge with UserUploadFile models below (used to store files in
-    distinct files mongo collection) eventually.
+    Public User Upload File (used for collection thumbnails).
     Conforms to Data Resource Spec.
     """
 
@@ -1164,38 +1162,28 @@ class UserFileOut(BaseModel):
     hash: str
     size: int
 
-    originalFilename: str
     mime: str
-    userid: UUID
-    userName: str
-    created: datetime
 
 
 # ============================================================================
-class PublicUserFileOut(BaseModel):
-    """public output for user-uploaded file stored on other document
-
-    Used for collection thumbnails.
-    Should merge with UserUploadFile models below (used to store files in
-    distinct files mongo collection) eventually.
+class UserFileOut(PublicUserFileOut):
+    """output for user-uploaded file as stored on other document,
+    additional non-public fields included
     Conforms to Data Resource Spec.
     """
 
-    name: str
-    path: str
-    hash: str
-    size: int
-
+    originalFilename: str
     mime: str
+    userid: UUID
+    userName: str
+    created: datetime
 
 
 # ============================================================================
 class UserFile(BaseFile):
     """User-uploaded file stored on anther mongo document
 
-    Used for collection thumbnails.
-    Should merge with UserUploadFile models below (used to store files in
-    distinct files mongo collection) eventually.
+    Base user uploaded file (currently used for collection thumbnails).
     Conforms to Data Resource Spec.
     """
 
@@ -1205,16 +1193,20 @@ class UserFile(BaseFile):
     userName: str
     created: datetime
 
+    async def get_absolute_presigned_url(
+        self, org, storage_ops, headers: Optional[dict]
+    ) -> str:
+        """Get presigned URL as absolute URL"""
+        presigned_url, _ = await storage_ops.get_presigned_url(org, self)
+        return storage_ops.resolve_relative_access_path(presigned_url, headers) or ""
+
     async def get_file_out(
         self, org, storage_ops, headers: Optional[dict] = None
     ) -> UserFileOut:
         """Get UserFileOut with new presigned url"""
-        presigned_url, _ = await storage_ops.get_presigned_url(org, self)
-        presigned_url = storage_ops.resolve_relative_access_path(presigned_url, headers)
-
         return UserFileOut(
             name=self.filename,
-            path=presigned_url or "",
+            path=await self.get_absolute_presigned_url(org, storage_ops, headers),
             hash=self.hash,
             size=self.size,
             originalFilename=self.originalFilename,
@@ -1228,12 +1220,9 @@ async def get_public_file_out(
         self, org, storage_ops, headers: Optional[dict] = None
     ) -> PublicUserFileOut:
         """Get PublicUserFileOut with new presigned url"""
-        presigned_url, _ = await storage_ops.get_presigned_url(org, self)
-        presigned_url = storage_ops.resolve_relative_access_path(presigned_url, headers)
-
         return PublicUserFileOut(
             name=self.filename,
-            path=presigned_url or "",
+            path=await self.get_absolute_presigned_url(org, storage_ops, headers),
             hash=self.hash,
             size=self.size,
             mime=self.mime,
@@ -1281,8 +1270,8 @@ def get_user_file(
 
 
 # ============================================================================
-class UserUploadFileOut(UserFileOut):
-    """Output model for all user-uploaded files stored in files mongo collection"""
+class SeedFileOut(UserFileOut):
+    """Output model for user-uploaded seed files"""
 
     id: UUID
     oid: UUID
@@ -1293,26 +1282,26 @@ class UserUploadFileOut(UserFileOut):
 
 
 # ============================================================================
-class UserUploadFile(UserFile, BaseMongoModel):
-    """User-uploaded file saved in files mongo collection"""
+class SeedFile(UserFile, BaseMongoModel):
+    """Stores user-uploaded file files in 'file_uploads' mongo collection
+    Used with crawl workflows
+    """
+
+    type: Literal["seedFile"] = "seedFile"
 
     id: UUID
     oid: UUID
-    type: str
 
     firstSeed: Optional[str] = None
     seedCount: Optional[int] = None
 
     async def get_file_out(
         self, org, storage_ops, headers: Optional[dict] = None
-    ) -> UserUploadFileOut:
-        """Get UserUploadFileOut with new presigned url"""
-        presigned_url, _ = await storage_ops.get_presigned_url(org, self)
-        presigned_url = storage_ops.resolve_relative_access_path(presigned_url, headers)
-
-        return UserUploadFileOut(
+    ) -> SeedFileOut:
+        """Get SeedFileOut with new presigned url"""
+        return SeedFileOut(
             name=self.filename,
-            path=presigned_url or "",
+            path=await self.get_absolute_presigned_url(org, storage_ops, headers),
             hash=self.hash,
             size=self.size,
             originalFilename=self.originalFilename,
@@ -1328,13 +1317,6 @@ async def get_file_out(
         )
 
 
-# ============================================================================
-class SeedFile(UserUploadFile):
-    """Seed file for crawl workflows"""
-
-    type: Literal["seedFile"] = "seedFile"
-
-
 # ============================================================================
 
 ### PAGES ###
@@ -3061,7 +3043,7 @@ class PaginatedUserOutResponse(PaginatedResponse):
 class PaginatedUserFileResponse(PaginatedResponse):
     """Response model for user-uploaded files (e.g. seed files)"""
 
-    items: List[UserUploadFileOut]
+    items: List[SeedFileOut]
 
 
 # ============================================================================

backend/btrixcloud/storages.py

@@ -56,7 +56,6 @@
     PresignedUrl,
     SuccessResponse,
     User,
-    UserUploadFile,
 )
 
 from .utils import slug_from_name, dt_now, get_origin
@@ -630,7 +629,7 @@ async def get_presigned_urls_bulk(
         return urls, now + self.signed_duration_delta
 
     async def delete_file_object(
-        self, org: Organization, crawlfile: Union[BaseFile, UserUploadFile]
+        self, org: Organization, crawlfile: Union[BaseFile]
     ) -> bool:
         """delete crawl file from storage."""
         return await self._delete_file(org, crawlfile.filename, crawlfile.storage)