fix: migrate media location and thumbnails to agnostic space provider

ascariandrea · ascariandrea · commit 463dc506af7f · 2025-08-19T18:41:12.000+02:00
diff --git a/packages/@liexp/backend/src/flows/tg/parseVideo.flow.ts b/packages/@liexp/backend/src/flows/tg/parseVideo.flow.ts
@@ -73,7 +73,7 @@ export const parseVideo =
                 TE.mapLeft(ServerError.fromUnknown),
               );
             }),
-            TE.map((r) => ensureHTTPS(r.Location)),
+            TE.map((r) => ensureHTTPS(ctx.env.SPACE_ENDPOINT, r.Location)),
           ),
       ),
     );
diff --git a/packages/@liexp/backend/src/io/media.io.ts b/packages/@liexp/backend/src/io/media.io.ts
@@ -30,9 +30,12 @@ const encodeMedia = (
   return pipe(
     Schema.encodeEither(io.http.Media.AdminMedia)({
       ...media,
+      location: ensureHTTPS(spaceEndpoint, media.location),
       label: media.label ?? media.location,
       description: media.description ?? undefined,
-      thumbnail: media.thumbnail ? ensureHTTPS(media.thumbnail) : undefined,
+      thumbnail: media.thumbnail
+        ? ensureHTTPS(spaceEndpoint, media.thumbnail)
+        : undefined,
       transferable: !media.location.includes(spaceEndpoint),
       creator: Schema.is(UUID)(media.creator)
         ? media.creator
@@ -65,7 +68,7 @@ const decodeMedia = (
       ...media,
       label: media.label ?? media.location,
       description: media.description ?? undefined,
-      location: ensureHTTPS(media.location),
+      location: ensureHTTPS(spaceEndpoint, media.location),
       creator: Schema.is(UUID)(media.creator)
         ? media.creator
         : media.creator?.id,
@@ -76,8 +79,10 @@ const decodeMedia = (
       areas: (media.areas ?? []).map((a) => (Schema.is(UUID)(a) ? a : a.id)),
       featuredInStories: media.featuredInStories ?? [],
       socialPosts: media.socialPosts ?? [],
-      thumbnail: media.thumbnail ? ensureHTTPS(media.thumbnail) : undefined,
-      transferable: !media.location.includes(spaceEndpoint),
+      thumbnail: media.thumbnail
+        ? ensureHTTPS(spaceEndpoint, media.thumbnail)
+        : undefined,
+      transferable: !media.location.startsWith("/"),
       createdAt: media.createdAt.toISOString(),
       updatedAt: media.updatedAt.toISOString(),
       deletedAt: media.deletedAt?.toISOString() ?? undefined,
diff --git a/packages/@liexp/backend/src/providers/puppeteer.provider.ts b/packages/@liexp/backend/src/providers/puppeteer.provider.ts
@@ -172,10 +172,11 @@ export const GetPuppeteerProvider = (
         });
         b.on("disconnected", (e) => {
           puppeteerLogger.debug.log(
-            "browser disconnected %ds",
+            "browser disconnected %ds: %O",
             differenceInSeconds(new Date(), connectedAt, {
               roundingMethod: "ceil",
             }),
+            e,
           );
         });
         return b;
diff --git a/packages/@liexp/backend/src/providers/wikipedia/wikipedia.provider.ts b/packages/@liexp/backend/src/providers/wikipedia/wikipedia.provider.ts
@@ -52,6 +52,7 @@ interface WikipediaProviderOpts {
   logger: Logger;
   client: Bot;
   restClient: AxiosInstance;
+  spaceEndpoint: string;
 }
 
 const toMWError = (e: unknown): Error => {
@@ -62,6 +63,7 @@ export const WikipediaProvider = ({
   logger,
   client,
   restClient,
+  spaceEndpoint,
 }: WikipediaProviderOpts): WikipediaProvider => {
   logger.debug.log("Wikipedia provider created");
 
@@ -127,7 +129,7 @@ export const WikipediaProvider = ({
             items.filter((i) => i.type === "image"),
             fp.A.head,
             fp.O.chainNullableK((r) => r.srcset?.[0]?.src),
-            fp.O.map((url) => ensureHTTPS(url)),
+            fp.O.map((url) => ensureHTTPS(spaceEndpoint, url)),
             fp.O.toUndefined,
           );
         }),
diff --git a/packages/@liexp/shared/src/utils/url.utils.ts b/packages/@liexp/shared/src/utils/url.utils.ts
@@ -1,7 +1,8 @@
+import { Schema } from "effect/index";
 import * as R from "fp-ts/lib/Record.js";
 import { pipe } from "fp-ts/lib/function.js";
 import qs from "query-string";
-import { type URL } from "../io/http/Common/URL.js";
+import { URL } from "../io/http/Common/URL.js";
 
 export const sanitizeURL = (url: URL): URL => {
   const [cleanURL, query] = url.split("?");
@@ -14,20 +15,28 @@ export const sanitizeURL = (url: URL): URL => {
   );
 
   if (!R.isEmpty(cleanQuery)) {
-    return `${cleanURL}?${qs.stringify(cleanQuery)}` as URL;
+    return Schema.decodeSync(URL)(`${cleanURL}?${qs.stringify(cleanQuery)}`);
   }
 
-  return cleanURL as URL;
+  return Schema.decodeSync(URL)(cleanURL);
 };
 
-export const ensureHTTPS = (url: string): URL => {
-  if (url.startsWith("https://") || url.startsWith("http://")) {
-    return url as URL;
+const encodeWithSpaceEndpoint = (spaceHost: string, url: string): string => {
+  if (url.startsWith("//")) {
+    return `https:${url}`;
   }
 
-  if (url.startsWith("//")) {
-    return `https:${url}` as URL;
+  if (url.startsWith("/")) {
+    return `https://${spaceHost}${url}`;
+  }
+
+  if (url.startsWith("https://") || url.startsWith("http://")) {
+    return url;
   }
 
-  return `https://${url}` as URL;
+  return `https://${url}`;
+};
+
+export const ensureHTTPS = (spaceHost: string, url: string): URL => {
+  return pipe(encodeWithSpaceEndpoint(spaceHost, url), Schema.decodeSync(URL));
 };
diff --git a/packages/@liexp/ui/src/components/Cards/Events/EventCard.tsx b/packages/@liexp/ui/src/components/Cards/Events/EventCard.tsx
@@ -87,6 +87,11 @@ const EventCard = <E extends SearchEvent.SearchEvent>({
                   height: props.style?.maxHeight ?? 150,
                   width: isVertical ? "100%" : 150,
                 }}
+                onError={(e) => {
+                  if (defaultImage && e.currentTarget.src !== defaultImage) {
+                    e.currentTarget.src = defaultImage;
+                  }
+                }}
               />
             </Stack>
           ) : null}
diff --git a/packages/@liexp/ui/src/components/Media/ExpandableImageElement.tsx b/packages/@liexp/ui/src/components/Media/ExpandableImageElement.tsx
@@ -64,6 +64,7 @@ interface ExpandableImageElementProps {
   onLoad?: (rect: DOMRect) => void;
   onClick?: (e: React.MouseEvent<HTMLDivElement, MouseEvent>) => void;
   disableZoom?: boolean;
+  fallbackImage?: string;
 }
 
 const ExpandableImageElement: React.FC<ExpandableImageElementProps> = ({
@@ -73,6 +74,7 @@ const ExpandableImageElement: React.FC<ExpandableImageElementProps> = ({
   onLoad,
   onClick,
   disableZoom = false,
+  fallbackImage,
 }) => {
   const [modal, showModal] = useModal({ disablePortal: false });
 
@@ -135,6 +137,11 @@ const ExpandableImageElement: React.FC<ExpandableImageElementProps> = ({
           const rect = e.currentTarget.getBoundingClientRect();
           onLoad?.(rect);
         }}
+        onError={(e) => {
+         if (fallbackImage) {
+            e.currentTarget.src = fallbackImage;
+          }
+        }}
         loading="lazy"
       />
       {!disableZoom ? (
diff --git a/packages/@liexp/ui/src/components/Media/MediaElement.tsx b/packages/@liexp/ui/src/components/Media/MediaElement.tsx
@@ -1,6 +1,7 @@
 import { Media } from "@liexp/shared/lib/io/http/index.js";
 import { clsx } from "clsx";
 import * as React from "react";
+import { useConfiguration } from "../../context/ConfigurationContext.js";
 import { styled } from "../../theme/index.js";
 import { Video } from "../Video/Video.js";
 import { Box, Typography } from "../mui/index.js";
@@ -44,6 +45,7 @@ export interface MediaElementProps {
   onLoad?: (rect: DOMRect) => void;
   enableDescription?: boolean;
   disableZoom?: boolean;
+  fallbackImage?: string;
   onClick?: (e: any) => void;
   options?: {
     iframe: {
@@ -64,8 +66,12 @@ const MediaElement: React.FC<MediaElementProps> = ({
   enableDescription = false,
   options,
   disableZoom = false,
+  fallbackImage: _fallbackImage,
   ...props
 }) => {
+  const { platforms } = useConfiguration();
+  const fallbackImage = _fallbackImage ?? platforms.web.defaultImage;
+
   const mediaElement = React.useMemo(() => {
     switch (media.type) {
       case Media.IframeVideoType.literals[0]:
@@ -129,6 +135,7 @@ const MediaElement: React.FC<MediaElementProps> = ({
             className={clsx(classes.item, itemClassName)}
             media={{ ...media, type: media.type }}
             disableZoom={disableZoom}
+            fallbackImage={fallbackImage}
           />
         );
     }
diff --git a/services/api/.env b/services/api/.env
@@ -16,8 +16,8 @@ DB_DATABASE=liexp
 DB_SSL_MODE=off
 WEB_URL=http://liexp.dev
 
-SPACE_BUCKET=local
-SPACE_ENDPOINT=space.liexp.dev:9000
+SPACE_BUCKET=public
+SPACE_ENDPOINT=space.liexp.dev
 SPACE_REGION=fra1
 SPACE_ACCESS_KEY_ID=space-access-key-id
 SPACE_ACCESS_KEY_SECRET=space-access-key-secret-id
diff --git a/services/api/.env.test b/services/api/.env.test
@@ -18,7 +18,7 @@ DB_HOST=127.0.0.1
 DB_PORT=8432
 DB_DATABASE=liexp_test
 DB_SSL_MODE=off
-WEB_URL=http://localhost:4020
+WEB_URL=http://liexp.dev
 
 SPACE_BUCKET=media
 SPACE_ENDPOINT=localhost:9000
@@ -35,14 +35,6 @@ DEFAULT_PAGE_SIZE=20
 
 JWT_SECRET=my-secret
 
-# Cron jobs
-DOWNLOAD_VACCINE_DATA_CRON="0 * * * * *"
-SOCIAL_POSTING_CRON="0 * * * * *"
-TEMP_FOLDER_CLEAN_UP_CRON="* * 0 * *"
-GENERATE_MISSING_THUMBNAILS_CRON="* * 0 * *"
-PROCESS_DONE_JOB_CRON="* * 0 * *"
-REGENERATE_MEDIA_THUMBNAILS_CRON="* * 0 * *"
-
 TG_BOT_TOKEN=invalid
 TG_BOT_CHAT=invalid
 TG_BOT_USERNAME=invalid
diff --git a/services/api/ormconfig.js b/services/api/ormconfig.js
@@ -4,8 +4,7 @@ import { loadENV } from "@liexp/core/lib/env/utils.js";
 import D from "debug";
 import { DataSource } from "typeorm";
 import { Schema } from "effect";
-import {GetLogger} from '@liexp/core/lib/logger/Logger.js'
-
+import { GetLogger } from "@liexp/core/lib/logger/Logger.js";
 
 
 if (process.env.DOTENV_CONFIG_PATH) {
diff --git a/services/api/scripts/dump-remote-db.sh b/services/api/scripts/dump-remote-db.sh
@@ -3,7 +3,7 @@
 set -ex
 
 # Load environment variables
-source .env.alpha
+source .env.prod
 
 user=$DB_USERNAME
 password=$DB_PASSWORD
diff --git a/services/api/src/context/index.ts b/services/api/src/context/index.ts
@@ -70,6 +70,7 @@ export const makeContext =
       restClient: axios.default.create({
         baseURL: "https://en.wikipedia.org/api/rest_v1",
       }),
+      spaceEndpoint: env.SPACE_ENDPOINT,
     });
 
     const rationalWikiProvider = WikipediaProvider({
@@ -84,6 +85,7 @@ export const makeContext =
       restClient: axios.default.create({
         baseURL: "https://rationalwiki.org/api/rest_v1",
       }),
+      spaceEndpoint: env.SPACE_ENDPOINT,
     });
 
     const fsClient = GetFSClient({ client: fs });
diff --git a/services/api/src/migrations/1754153060855-MediaEntityNoCDNHost.ts b/services/api/src/migrations/1754153060855-MediaEntityNoCDNHost.ts
@@ -0,0 +1,36 @@
+import { type MigrationInterface, type QueryRunner } from "typeorm";
+
+export class MediaEntityNoCDNHost1754153060855 implements MigrationInterface {
+  name = "MediaEntityNoCDNHost1754153060855";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      "update image i set \"location\" = replace(\"location\", 'https://econnessione-alpha.fra1.digitaloceanspaces.com', '') where i.\"location\" like 'https://econnessione-alpha.fra1.digitaloceanspaces.com%'",
+    );
+    await queryRunner.query(
+      "update image i set \"location\" = replace(\"location\", 'https://fra1.digitaloceanspaces.com/econnessione-alpha', '') where i.\"location\" like 'https://fra1.digitaloceanspaces.com/econnessione-alpha/%'",
+    );
+    await queryRunner.query(
+      "update image i set \"location\" = replace(\"location\", 'fra1.digitaloceanspaces.com/econnessione-alpha', '') where i.\"location\" like 'fra1.digitaloceanspaces.com/econnessione-alpha/%'",
+    );
+
+    await queryRunner.query(
+      "update image i set \"thumbnail\" = replace(\"thumbnail\", 'https://econnessione-alpha.fra1.digitaloceanspaces.com', '') where i.\"thumbnail\" like 'https://econnessione-alpha.fra1.digitaloceanspaces.com%'",
+    );
+    await queryRunner.query(
+      "update image i set \"thumbnail\" = replace(\"thumbnail\", 'https://fra1.digitaloceanspaces.com/econnessione-alpha', '') where i.\"thumbnail\" like 'https://fra1.digitaloceanspaces.com/econnessione-alpha/%'",
+    );
+    await queryRunner.query(
+      "update image i set \"thumbnail\" = replace(\"thumbnail\", 'fra1.digitaloceanspaces.com/econnessione-alpha', '') where i.\"thumbnail\" like 'fra1.digitaloceanspaces.com/econnessione-alpha/%'",
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      "update image i set \"location\" = replace(\"location\", '/public', 'https://econnessione-alpha.fra1.digitaloceanspaces.com/public') where i.\"location\" like '/%'",
+    );
+    await queryRunner.query(
+      "update image i set \"thumbnail\" = replace(\"thumbnail\", '/public', 'https://econnessione-alpha.fra1.digitaloceanspaces.com/public') where i.\"thumbnail\" like '/%'",
+    );
+  }
+}
diff --git a/services/api/src/routes/actors/__tests__/listActor.e2e.ts b/services/api/src/routes/actors/__tests__/listActor.e2e.ts
@@ -30,6 +30,7 @@ describe("List Actor", () => {
           diedOn: undefined,
           death: undefined,
           avatar: a.avatar,
+          nationalities: [],
         })),
       ),
     );
diff --git a/services/api/src/routes/social-posts/__tests__/createSocialPost.e2e.ts b/services/api/src/routes/social-posts/__tests__/createSocialPost.e2e.ts
@@ -1,5 +1,4 @@
 import { MediaEntity } from "@liexp/backend/lib/entities/Media.entity.js";
-import { toMediaEntity } from "@liexp/backend/lib/test/utils/entities/index.js";
 import { uuid } from "@liexp/shared/lib/io/http/Common/UUID.js";
 import { ImageType } from "@liexp/shared/lib/io/http/Media/MediaType.js";
 import {
@@ -89,6 +88,7 @@ describe("Create Social Post", () => {
       areas: [],
       featuredInStories: [],
       socialPosts: [],
+      creator: undefined,
     }));
 
     await throwTE(Test.ctx.db.save(MediaEntity, [media]));
diff --git a/services/web/.env b/services/web/.env
@@ -1,7 +1,5 @@
-# build env
 VIRTUAL_HOST=liexp.dev
 VIRTUAL_PORT=3000
-
 # app env
 VITE_NODE_ENV=development
 VITE_PUBLIC_URL=http://liexp.dev
diff --git a/services/worker/src/context/make.ts b/services/worker/src/context/make.ts
@@ -83,12 +83,14 @@ export const makeContext = (
     logger: GetLogger("wp"),
     client: impl.wp.wiki,
     restClient: impl.wp.http,
+    spaceEndpoint: env.SPACE_ENDPOINT,
   });
 
   const rationalWikiProvider = WikipediaProvider({
     logger: GetLogger("rw"),
     client: impl.rw.wiki,
     restClient: impl.rw.http,
+    spaceEndpoint: env.SPACE_ENDPOINT,
   });
 
   const urlMetadataClient = MakeURLMetadata(impl.urlMetadata);
diff --git a/services/worker/src/jobs/generateMissingMedia.job.ts b/services/worker/src/jobs/generateMissingMedia.job.ts
@@ -28,7 +28,7 @@ export const generateMissingThumbnailsCron = (
                 pipe(
                   createThumbnail({
                     id: m.id,
-                    location: ensureHTTPS(m.location),
+                    location: ensureHTTPS(ctx.env.SPACE_ENDPOINT, m.location),
                     type: m.type,
                     thumbnail: null,
                   })(ctx),
