Merge branch 'lindylearn-match-by-frontmatter'

Author: weichenw

package.json

@@ -18,6 +18,7 @@
   "license": "MIT",
   "dependencies": {
     "axios": "^0.26.0",
+    "gray-matter": "^4.0.3",
     "lodash.pickby": "^4.6.0",
     "nunjucks": "^3.2.3"
   },
@@ -49,4 +50,4 @@
     "webpack-cli": "^4.6.0",
     "webpack-node-externals": "^2.5.2"
   }
-}
+}

src/fileManager.ts

@@ -1,9 +1,15 @@
-import type {  Vault } from 'obsidian';
+import type { Vault, MetadataCache, TFile } from 'obsidian';
 import { get } from 'svelte/store';
 import { Renderer } from '~/renderer';
 import { settingsStore } from '~/store';
 import { sanitizeTitle } from '~/utils/sanitizeTitle';
 import type { Article } from '~/models';
+import { frontMatterDocType, addFrontMatter } from "~/utils/frontmatter"
+
+type AnnotationFile = {
+  articleUrl?: string;
+  file: TFile;
+};
 
 const articleFolderPath = (article: Article): string => {
   const settings = get(settingsStore);
@@ -17,52 +23,68 @@ const articleFolderPath = (article: Article): string => {
 
 export default class FileManager {
   private vault: Vault;
+  private metadataCache: MetadataCache;
   private renderer: Renderer;
 
-  constructor(vault: Vault) {
+  constructor(vault: Vault, metadataCache: MetadataCache) {
     this.vault = vault;
+    this.metadataCache = metadataCache;
     this.renderer = new Renderer();
   }
 
-  public async createFolder(folderPath: string): Promise<void> {
-    await this.vault.createFolder(folderPath);
-  }
+  // Save an article as markdown file, replacing its existing file if present
+  public async saveArticle(article: Article): Promise<boolean> {
+    const existingFile = await this.getArticleFile(article);
 
-  public async createFile(filePath: string, content: string): Promise<void> {
-    await this.vault.create(filePath, content);
-  }
+    if (existingFile) {
+      console.debug(`Updating ${existingFile.path}`);
 
-  public async createOrUpdate(article: Article): Promise<boolean> {
-    const folderPath = articleFolderPath(article);
-    const fileName = `${sanitizeTitle(article.metadata.title)}.md`;
-    const filePath = `${folderPath}/${fileName}`
-    let createdNewArticle = false;
+      const newMarkdownContent = this.renderer.render(article, false);
+      const existingFileContent = await this.vault.cachedRead(existingFile);
+      const fileContent = existingFileContent + newMarkdownContent;
 
-    if (!(await this.vault.adapter.exists(folderPath))) {
-
-      console.info(`Folder ${folderPath} not found. Will be created`);
-      await this.createFolder(folderPath);
-    }
+      await this.vault.modify(existingFile, fileContent);
+      return false;
+    } else {
+      const newFilePath = this.getNewArticleFilePath(article);
+      console.debug(`Creating ${newFilePath}`);
 
-    if (!(await this.vault.adapter.exists(filePath))) {
+      const markdownContent = this.renderer.render(article, true);
+      const fileContent = addFrontMatter(markdownContent, article);
 
-      console.info(`Document ${filePath} not found. Will be created`);
-      const content = this.renderer.render(article);
-      await this.createFile(filePath, content);
-      createdNewArticle = true;
-      await settingsStore.actions.addSyncedFile({filename: fileName, uri: encodeURIComponent(article.metadata.url)});
+      await this.vault.create(newFilePath, fileContent);
+      return true;
     }
-    else {
+  }
 
-      console.info(`Document ${article.metadata.title} found. Loading content and updating highlights`);
-      const content = this.renderer.render(article, false);
-      const fileContent = await this.vault.adapter.read(filePath);
-      const contentToSave = fileContent + content;
-      await this.vault.adapter.write(filePath, contentToSave);
+  public async isArticleSaved(article: Article): Promise<boolean> {
+    const file = await this.getArticleFile(article);
+    return !!file
+  }
 
-    }
+  private async getArticleFile(article: Article): Promise<TFile | null> {
+    const files = await this.getAnnotationFiles()
+    return files.find((file) => file.articleUrl === article.metadata.url)?.file || null;
+  }
 
-    return createdNewArticle;
+  // TODO cache this method for performance?
+  private async getAnnotationFiles(): Promise<AnnotationFile[]> {
+    const files = this.vault.getMarkdownFiles();
+
+    return files
+      .map((file) => {
+        const cache = this.metadataCache.getFileCache(file);
+        return { file, frontmatter: cache?.frontmatter };
+      })
+      .filter(({ frontmatter }) => frontmatter?.["doc_type"] === frontMatterDocType)
+      .map(({ file, frontmatter }): AnnotationFile => ({ file, articleUrl: frontmatter["url"] }))
+  }
+
+  public getNewArticleFilePath(article: Article): string {
+    const folderPath = articleFolderPath(article);
+    const fileName = `${sanitizeTitle(article.metadata.title)}.md`;
+    const filePath = `${folderPath}/${fileName}`
+    return filePath;
   }
 
 }

src/main.ts

@@ -18,7 +18,7 @@ export default class HypothesisPlugin extends Plugin {
 
 		await initialise(this);
 
-		const fileManager = new FileManager(this.app.vault);
+		const fileManager = new FileManager(this.app.vault, this.app.metadataCache);
 
 		this.syncHypothesis = new SyncHypothesis(fileManager);
 

src/modals/resyncDelFileModal.ts

@@ -4,18 +4,22 @@ import { settingsStore } from '~/store';
 import { get } from 'svelte/store';
 import SyncHypothesis from '~/sync/syncHypothesis';
 import FileManager from '~/fileManager';
-import type { SyncedFile } from '~/models';
+import type { Article, SyncedFile } from '~/models';
+import ApiManager from '~/api/api';
+import parseSyncResponse from '~/parser/parseSyncResponse';
 
 export default class ResyncDelFileModal extends Modal {
     private syncHypothesis!: SyncHypothesis;
     public waitForClose: Promise<void>;
     private resolvePromise: () => void;
     private modalContent: ResyncDelFileModalContent;
-    vault: Vault;
+    private vault: Vault;
+    private fileManager: FileManager
 
     constructor(app: App) {
         super(app);
         this.vault = app.vault;
+        this.fileManager = new FileManager(this.vault, this.app.metadataCache);
 
         this.waitForClose = new Promise(
             (resolve) => (this.resolvePromise = resolve)
@@ -27,9 +31,8 @@ export default class ResyncDelFileModal extends Modal {
 
     async onOpen() {
         super.onOpen()
-        const fileManager = new FileManager(this.vault);
-		this.syncHypothesis = new SyncHypothesis(fileManager);
-        const deletedFiles = await this.retrieveDeletedFiles(this.vault);
+		this.syncHypothesis = new SyncHypothesis(this.fileManager);
+        const deletedFiles = await this.retrieveDeletedFiles();
 
         this.titleEl.innerText = "Hypothes.is: Resync deleted file(s)";
 
@@ -56,21 +59,20 @@ export default class ResyncDelFileModal extends Modal {
         this.resolvePromise();
     }
 
-    async retrieveDeletedFiles(vault:Vault) {
+    async retrieveDeletedFiles(): Promise<SyncedFile[]> {
+        const token = get(settingsStore).token;
+        const userid = get(settingsStore).user;
+        const apiManager = new ApiManager(token, userid);
 
-        const folder = get(settingsStore).highlightsFolder;
-        const syncedFiles = get(settingsStore).syncedFiles;
+        // Fetch all annotated articles that *should* be present
+        const allAnnotations = await apiManager.getHighlights()
+        const allArticles: [] = Object.values(await parseSyncResponse(allAnnotations));
 
-        const existingFiles = await vault.adapter.list(`/${folder}`);
-
-        // eslint-disable-next-line
-        const r = /[^\/]*$/;
-
-        existingFiles.files.forEach(function(value, index) {
-            this[index] = (value.match(r))[0]
-        }, existingFiles.files);
-
-        return syncedFiles.filter((file) =>  !existingFiles.files.includes(file.filename));
+        // Check which files are actually present
+        const deletedArticles = await Promise.all(allArticles.filter(async article => !(await this.fileManager.isArticleSaved(article))));
+        return deletedArticles.map((article: Article) => 
+            ({ uri: article.metadata.url, filename: this.fileManager.getNewArticleFilePath(article)})
+        );
     }
 
     async startResync(selectedFiles: SyncedFile[]): Promise<void> {

src/store/settings.ts

@@ -1,4 +1,4 @@
-import type { Group, SyncedFile } from '~/models';
+import type { Group } from '~/models';
 import { writable } from 'svelte/store';
 import defaultTemplate from '~/assets/defaultTemplate.njk';
 import type HypothesisPlugin from '~/main';
@@ -19,7 +19,6 @@ type Settings = {
   history: SyncHistory;
   dateTimeFormat: string;
   autoSyncInterval: number;
-  syncedFiles: SyncedFile[];
   groups: Group[];
   useDomainFolders: boolean;
 };
@@ -37,9 +36,8 @@ const DEFAULT_SETTINGS: Settings = {
     totalArticles: 0,
     totalHighlights: 0,
   },
-  syncedFiles: [],
   groups: [],
-  useDomainFolders: false,
+  useDomainFolders: false
 };
 
 const createSettingsStore = () => {
@@ -107,7 +105,6 @@ const createSettingsStore = () => {
       state.history.totalArticles = 0;
       state.history.totalHighlights = 0;
       state.lastSyncDate = undefined;
-      state.syncedFiles = [];
       return state;
     });
   };
@@ -155,20 +152,6 @@ const createSettingsStore = () => {
     });
   };
 
-  const addSyncedFile = (value: SyncedFile) => {
-    store.update((state) => {
-      const uniqueValuesSet = new Set();
-      const syncFiles = [...state.syncedFiles, value];
-      state.syncedFiles = syncFiles.filter((obj) => {
-        const isPresentInSet = uniqueValuesSet.has(obj.filename);
-        uniqueValuesSet.add(obj.filename);
-        return !isPresentInSet;
-      });
-
-      return state;
-    });
-  }
-
   const setGroups = async (value: Group[]) => {
     store.update((state) => {
       state.groups = value;
@@ -204,7 +187,6 @@ const createSettingsStore = () => {
       setSyncOnBoot,
       incrementHistory,
       setDateTimeFormat,
-      addSyncedFile,
       setGroups,
       resetGroups,
       setUseDomainFolder,

src/sync/syncHypothesis.ts

@@ -68,7 +68,7 @@ export default class SyncHypothesis {
 
     private async syncArticle(article: Article): Promise<void> {
 
-        const createdNewArticle = await this.fileManager.createOrUpdate(article);
+        const createdNewArticle = await this.fileManager.saveArticle(article);
 
         if (createdNewArticle) {
             this.syncState.newArticlesSynced += 1;

src/utils/frontmatter.ts

@@ -0,0 +1,17 @@
+import matter from "gray-matter"
+import type { Article } from '~/models';
+
+type FrontMatterContent = {
+    doc_type?: string;
+    url?: string;
+}
+
+export const frontMatterDocType = "hypothesis-highlights"
+
+export const addFrontMatter = (markdownContent: string, article: Article) => {
+    const frontMatter: FrontMatterContent = {
+        doc_type: frontMatterDocType,
+        url: article.metadata.url,
+    };
+    return matter.stringify(markdownContent, frontMatter);
+}