feat(pdf): add pdf sanitization

Author: AlbanSdl

.dockerignore

@@ -5,3 +5,5 @@ docker-compose.*yml
 node_modules
 npm-debug.log
 dist
+uploads
+qpdf

.gitignore

@@ -114,7 +114,11 @@ docs/build/
 # OLD DATABASE, DO NOT PUSH, IT CONTAINS SENSITIVE DATA
 migration/etuutt_old/etuutt_old.sql
 # UE Data
-scripts/**/*.csv
+scripts/**/*
+!scripts/**/
+!scripts/**/*.ts
+uploads/
+qpdf/
 
 # Dev files
 dev

package.json

@@ -6,6 +6,7 @@
   "private": true,
   "license": "UNLICENSED",
   "scripts": {
+    "postinstall": "rm -rf qpdf && wget -q \"https://github.com/qpdf/qpdf/releases/download/v12.1.0/qpdf-12.1.0-bin-linux-x86_64.zip\" -O qpdf.zip && unzip qpdf.zip -d qpdf && rm qpdf.zip",
     "build": "npx nest build",
     "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
     "start": "NODE_ENV=development npx nest start",
@@ -29,10 +30,10 @@
     "test:db:editor": "env-cmd -f .env.test -- pnpm prisma studio",
     "seed:base": "env-cmd -f .env.dev -- ts-node scripts/seed/base.ts",
     "seed:ue": "env-cmd -f .env.dev -- ts-node scripts/seed/ue.ts",
-    "seed:ue:aliases": "env-cmd -f .env.dev -- ts-node scripts/seed/aliases.ts",
+    "seed:ue:annals": "env-cmd -f .env.dev -- ts-node scripts/seed/ue-annals.ts",
     "seed:base:prod": "node scripts/seed/base.js",
     "seed:ue:prod": "node scripts/seed/ue.js",
-    "seed:ue:aliases:prod": "node scripts/seed/aliases.js",
+    "seed:ue:annals:prod": "node scripts/seed/ue-annals.js",
     "script:deps:graph": "ts-node scripts/dependency_graph.ts"
   },
   "dependencies": {
@@ -56,6 +57,7 @@
     "multer": "1.4.5-lts.1",
     "pactum-matchers": "^1.1.7",
     "passport-jwt": "^4.0.1",
+    "pdf-lib": "^1.17.1",
     "pdfjs-dist": "^5.0.375",
     "pdfkit": "^0.14.0",
     "prisma": "^5.17.0",

pnpm-lock.yaml

@@ -310,7 +310,7 @@ model UeAnnal {
 
 model UeAnnalType {
   id   String @id @default(uuid())
-  name String @db.VarChar(255)
+  name String @unique @db.VarChar(255)
 
   annals UeAnnal[]
 }
@@ -812,7 +812,7 @@ model UTTBranch {
 
 model UTTBranchOption {
   id                       String @id @default(uuid())
-  code                     String @db.VarChar(10)
+  code                     String @db.VarChar(10) @unique
   name                     String @db.VarChar(255)
   branchCode               String
   descriptionTranslationId String @unique

prisma/schema.prisma

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.json",
+  "files": ["../../src/types.d.ts"]
+}

prisma/seed/tsconfig.json

@@ -32,4 +32,4 @@ async function main() {
   console.info('\x1b[42;30m✅ Aliases have been created/updated\x1b[0m');
 }
 
-main();
+export { main as seedUeAliases };

scripts/seed/aliases.ts

@@ -180,6 +180,22 @@ async function main() {
       }),
     ),
   );
+
+  console.log('Updating Annal Types');
+  const annalTypes = ['Final', 'Médian', 'Partiel', 'Examen de TP', 'Projet', 'Devoir Maison'];
+  await Promise.all(
+    annalTypes.map((type) =>
+      prisma.ueAnnalType.upsert({
+        where: {
+          name: type,
+        },
+        update: {},
+        create: {
+          name: type,
+        },
+      }),
+    ),
+  );
 }
 
 main();

scripts/seed/base.ts

@@ -0,0 +1,112 @@
+import { readdir, readFile, writeFile } from 'fs/promises';
+import { PrismaClient } from '@prisma/client';
+
+const prisma = new PrismaClient();
+
+const annalTypes = {
+  final: 'Final',
+  median: 'Médian',
+  dm: 'Devoir Maison',
+  partiel: 'Partiel',
+};
+
+async function main() {
+  const folderPath = process.argv[2] || 'scripts/seed/ues';
+
+  const ueDirectories = Object.fromEntries(
+    (
+      await Promise.all(
+        (
+          await readdir(folderPath, { withFileTypes: true })
+        )
+          .filter((dirent) => dirent.isDirectory())
+          .map(async (dirent) => {
+            const dirname = dirent.name;
+            return (await readdir(`${folderPath}/${dirname}`, { withFileTypes: true }))
+              .filter((dir) => dir.isDirectory())
+              .map((dir) => [dir.name, `${folderPath}/${dirname}/${dir.name}`]);
+          }),
+      )
+    ).flat(),
+  ) as Record<string, string>;
+
+  const databaseUeofs = (
+    await prisma.ueof.findMany({
+      select: { code: true },
+    })
+  ).map((ueof) => ueof.code);
+
+  // console.log(ueDirectories);
+  for (const [ueName, uePath] of Object.entries(ueDirectories)) {
+    const annals = (await readdir(uePath, { withFileTypes: true })).filter((dirent) =>
+      dirent.name.match(/\.(png|jpeg|jpg|pdf|tiff)$/),
+    );
+    const other = (await readdir(uePath, { withFileTypes: true })).filter(
+      (dirent) => !dirent.name.match(/\.(png|jpeg|jpg|pdf|tiff)$/) && !dirent.name.match(/^ue(?:\.md)?\.docx$/),
+    );
+    if (other.length > 0) console.log(other);
+
+    // Import annal files
+    for (const annal of annals) {
+      const name = annal.name
+        .replace(/\.[^.]+$/, '')
+        .replace(
+          /[-_](?=(?<sem1>[AP]\d{4})?(?<ue1>[^-_]*(?![AP]\d{4}))?)[^-_]*[-_](?=(?<sem2>[AP]\d{4})?(?<ue2>[^-_]*(?![AP]\d{4}))?)[^-_]*/i,
+          '-$<sem1>$<sem2>-$<ue1>$<ue2>',
+        )
+        .replace(/(?<=-[AP])20(?=\d{2}-)/, '');
+      const [type, semester] = name.split('-');
+      if (!(type in annalTypes)) console.warn(`Unknown annal type: ${type}`);
+      const ueofs = databaseUeofs
+        .filter(
+          (ueof) =>
+            ueof.startsWith(`${ueName.replace(/A$/, '')}_`) &&
+            (Number(semester.slice(1)) >= Number(ueof.slice(-2)) || ueof.endsWith('U23')),
+        )
+        .sort((a, b) => {
+          const ueofVector = Number(ueName.endsWith('A')) && Number(/_EN_/.test(b)) - Number(/_EN_/.test(a));
+          if (ueofVector) return ueofVector;
+          return Number(a.slice(-2)) - Number(b.slice(-2));
+        });
+
+      // console.info(`[NAME FIX] ${annal.name} -> ${name}`);
+      // console.log(`[UEOF SELECTION] ${ueName} -> ${ueofs[0]}`);
+
+      if (!ueofs.length) console.warn(`[UEOF SELECTION FAILED] ${ueName}`);
+      else {
+        const { id } = await prisma.ueAnnal.create({
+          data: {
+            type: {
+              connect: {
+                name: annalTypes[type],
+              },
+            },
+            uploadComplete: true,
+            ueof: {
+              connect: {
+                code: ueofs[0],
+              },
+            },
+            semester: {
+              connectOrCreate: {
+                where: {
+                  code: semester,
+                },
+                create: {
+                  code: semester,
+                  start: new Date(),
+                  end: new Date(),
+                },
+              },
+            },
+          },
+        });
+        await writeFile(`uploads/exams/${id}.pdf`, (await readFile(`${uePath}/${annal.name}`)) as Uint8Array);
+      }
+    }
+  }
+
+  console.info('\x1b[42;30m✅ Import complete\x1b[0m');
+}
+
+main();

scripts/seed/ue-annals.ts

@@ -3,6 +3,7 @@ import { createInterface } from 'readline/promises';
 import { parse } from '@fast-csv/parse';
 import { PrismaClient } from '@prisma/client';
 import type { TextItem } from 'pdfjs-dist/types/src/display/api';
+import { seedUeAliases } from './aliases';
 
 const prisma = new PrismaClient();
 
@@ -291,7 +292,7 @@ async function main() {
               },
             },
             branchOptions: {
-              connect: credit.branchOptions.map((id) => ({ id })),
+              connect: credit.branchOptions.map((id) => ({ code: id })),
             },
           })),
         },
@@ -358,39 +359,45 @@ async function main() {
     });
   }
 
+  console.info('\x1b[42;30mImporting UE aliases\x1b[0m');
+  await seedUeAliases();
+
   const aliases = await prisma.ueAlias.findMany();
 
   console.info('\x1b[42;30mImporting UE requirements...\x1b[0m');
-
-  try {
-    await Promise.all(
-      ues.map((ueof) =>
-        prisma.ueof.update({
-          where: {
-            code: ueof.ueof_code,
-          },
-          data: {
-            requirements: {
-              connect: ueof.requirements
-                .map((code) => {
-                  const result = aliases.find((alias) => alias.code === code);
-                  return result ? result.standsFor : code;
-                })
-                .filter((code) => code)
-                .map((code) => ({
-                  code: code,
-                })),
-            },
+  const results = await Promise.allSettled(
+    ues.map((ueof) =>
+      prisma.ueof.update({
+        where: {
+          code: ueof.ueof_code,
+        },
+        data: {
+          requirements: {
+            connect: ueof.requirements
+              .map((code) => {
+                const result = aliases.find((alias) => alias.code === code);
+                return result ? result.standsFor : code;
+              })
+              .filter((code) => code)
+              .map((code) => ({
+                code: code,
+              })),
           },
-        }),
-      ),
-    );
-  } catch (error) {
-    console.error(
-      '\x1b[41;30mAn error occurred while importing UE requirements. Try `$ pnpm seed:ue:aliases` first.\x1b[0m',
+        },
+      }),
+    ),
+  );
+  const fails = [];
+  for (let i = 0; i < results.length; i++) if (results[i].status === 'rejected') fails.push(ues[i]);
+  if (fails.length)
+    return console.warn(
+      `\x1b[41;30m[UNKNOWN_UE_REFERENCE] Unknown UE reference${fails.length ? 's' : ''}: ${[
+        ...new Set(fails.map((ueof) => ueof.requirements).flat()),
+      ]
+        .filter((mUe) => ues.every((ue) => ue.code !== mUe))
+        .join(', ')}. Consider adding UE aliases\x1b[0m`,
     );
-    return;
-  }
+
   console.info('\x1b[42;30mFetching current INSUEs\x1b[0m');
   try {
     const insues = await fetchINSUE();