import Note from '../models/Note'; import { NoteEntity } from '../services/database/types'; import { MdFrontMatterExport } from '../services/interop/types'; import time from '../time'; import * as yaml from 'js-yaml'; import moment = require('moment'); export interface ParsedMeta { metadata: NoteEntity; tags: string[]; } const convertDate = (datetime: number): string => { return time.unixMsToRfc3339Sec(datetime); }; const dateStringToDate = (dateString: string, defaultValue: number) => { try { // When exporting with Joplin, we encode the date in this format, so try this first. const ms = time.rfc3339SecToUnixMs(dateString); return ms; } catch { // If it fails, try to parse with `moment`: const m = moment(dateString); return m.isValid() ? m.toDate().getTime() : defaultValue; } }; export const fieldOrder = ['title', 'id', 'updated', 'created', 'source', 'author', 'latitude', 'longitude', 'altitude', 'completed?', 'due', 'tags']; // There is a special case (negative numbers) where the yaml library will force quotations // These need to be stripped function trimQuotes(rawOutput: string): string { return rawOutput.split('\n').map(line => { const index = line.indexOf(': \'-'); const indexWithSpace = line.indexOf(': \'- '); // We don't apply this processing if the string starts with a dash // followed by a space. Those should actually be in quotes, otherwise // they are detected as invalid list items when we later try to import // the file. if (index === indexWithSpace) return line; if (index >= 0) { // The plus 2 eats the : and space characters const start = line.substring(0, index + 2); // The plus 3 eats the quote character const end = line.substring(index + 3, line.length - 1); return start + end; } return line; }).join('\n'); } export const noteToFrontMatter = (note: NoteEntity, tagTitles: string[]) => { const md: MdFrontMatterExport = {}; // Every variable needs to be converted separately, so they will be handles in groups // // title if (note.title) { md['title'] = note.title; } // source, author if (note.source_url) { md['source'] = note.source_url; } if (note.author) { md['author'] = note.author; } // locations // non-strict inequality is used here to interpret the location strings // as numbers i.e 0.000000 is the same as 0. // This is necessary because these fields are officially numbers, but often // contain strings. // eslint-disable-next-line eqeqeq if (note.latitude != 0 || note.longitude != 0 || note.altitude != 0) { md['latitude'] = note.latitude; md['longitude'] = note.longitude; md['altitude'] = note.altitude; } // todo if (note.is_todo) { // boolean is not support by the yaml FAILSAFE_SCHEMA md['completed?'] = note.todo_completed ? 'yes' : 'no'; } if (note.todo_due) { md['due'] = convertDate(note.todo_due); } // time if (note.user_updated_time) { md['updated'] = convertDate(note.user_updated_time); } if (note.user_created_time) { md['created'] = convertDate(note.user_created_time); } // tags if (tagTitles.length) md['tags'] = tagTitles; // This guarentees that fields will always be ordered the same way // which can be useful if users are using this for generating diffs const sort = (a: string, b: string) => { return fieldOrder.indexOf(a) - fieldOrder.indexOf(b); }; // The FAILSAFE_SCHEMA along with noCompatMode allows this to export strings that look // like numbers (or yes/no) without the added '' quotes around the text const rawOutput = yaml.dump(md, { sortKeys: sort, noCompatMode: true, schema: yaml.FAILSAFE_SCHEMA }); // The additional trimming is the unfortunate result of the yaml library insisting on // quoting negative numbers. // For now the trimQuotes function only trims quotes associated with a negative number // but it can be extended to support more special cases in the future if necessary. return trimQuotes(rawOutput); }; export const serialize = async (modNote: NoteEntity, tagTitles: string[]) => { const noteContent = await Note.replaceResourceInternalToExternalLinks(await Note.serialize(modNote, ['body'])); const metadata = noteToFrontMatter(modNote, tagTitles); return `---\n${metadata}---\n\n${noteContent}`; }; function isTruthy(str: string): boolean { return ['true', 'yes'].includes(str.toLowerCase()); } // Enforces exactly 2 spaces in front of list items function normalizeYamlWhitespace(yaml: string[]): string[] { return yaml.map(line => { const l = line.trimStart(); if (l.startsWith('-')) { return ` ${l}`; } return line; }); } // This is a helper function to convert an arbitrary author variable into a string // the use case is for loading from r-markdown/pandoc style notes // references: // https://pandoc.org/MANUAL.html#extension-yaml_metadata_block // https://github.com/hao203/rmarkdown-YAML function extractAuthor(author: unknown): string { if (!author) return ''; if (typeof(author) === 'string') { return author; } else if (Array.isArray(author)) { // Joplin only supports a single author, so we take the first one return extractAuthor(author[0]); } else if (typeof(author) === 'object') { if ('name' in author) { return (author as { name: string }).name; } } return ''; } const getNoteHeader = (note: string) => { // Ignore the leading `---` const lines = note.split('\n').slice(1); let inHeader = true; const headerLines: string[] = []; const bodyLines: string[] = []; for (let i = 0; i < lines.length; i++) { const line = lines[i]; const nextLine = i + 1 <= lines.length - 1 ? lines[i + 1] : ''; if (inHeader && line.startsWith('---')) { inHeader = false; // Need to eat the extra newline after the yaml block. Note that // if the next line is not an empty line, we keep it. Fixes // https://github.com/laurent22/joplin/issues/8802 if (nextLine.trim() === '') i++; continue; } if (inHeader) { headerLines.push(line); } else { bodyLines.push(line); } } const normalizedHeaderLines = normalizeYamlWhitespace(headerLines); const header = normalizedHeaderLines.join('\n'); const body = bodyLines.join('\n'); return { header, body }; }; const toLowerCase = (obj: Record): Record => { const newObj: Record = {}; for (const key of Object.keys(obj)) { newObj[key.toLowerCase()] = obj[key]; } return newObj; }; export const parse = (note: string): ParsedMeta => { if (!note.startsWith('---')) return { metadata: { body: note }, tags: [] }; const { header, body } = getNoteHeader(note); const md = toLowerCase((yaml.load(header, { schema: yaml.FAILSAFE_SCHEMA }) as Record) ?? {}); const asString = (v: unknown): string => typeof v === 'string' ? v : v === null || v === undefined ? '' : String(v); const asNumber = (v: unknown): number => typeof v === 'number' ? v : Number(v); const metadata: NoteEntity = { title: asString(md['title']), source_url: asString(md['source']), is_todo: ('completed?' in md) ? 1 : 0, }; if ('id' in md && typeof md['id'] === 'string' && md['id'].match(/^[0-9a-zA-Z]{32}$/)) { metadata['id'] = md['id']; } if ('author' in md) { metadata['author'] = extractAuthor(md['author']); } // The date fallback gives support for MultiMarkdown format, r-markdown, and pandoc formats if ('created' in md) { metadata['user_created_time'] = dateStringToDate(asString(md['created']), Date.now()); } else if ('date' in md) { metadata['user_created_time'] = dateStringToDate(asString(md['date']), Date.now()); } else if ('created_at' in md) { // Add support for Notesnook metadata['user_created_time'] = dateStringToDate(asString(md['created_at']), Date.now()); } if ('updated' in md) { metadata['user_updated_time'] = dateStringToDate(asString(md['updated']), Date.now()); } else if ('lastmod' in md) { // Add support for hugo metadata['user_updated_time'] = dateStringToDate(asString(md['lastmod']), Date.now()); } else if ('date' in md) { metadata['user_updated_time'] = dateStringToDate(asString(md['date']), Date.now()); } else if ('updated_at' in md) { // Notesnook metadata['user_updated_time'] = dateStringToDate(asString(md['updated_at']), Date.now()); } if ('latitude' in md) { metadata['latitude'] = asNumber(md['latitude']); } if ('longitude' in md) { metadata['longitude'] = asNumber(md['longitude']); } if ('altitude' in md) { metadata['altitude'] = asNumber(md['altitude']); } if (metadata.is_todo) { if (isTruthy(asString(md['completed?']))) { // Completed time isn't preserved, so we use a sane choice here metadata['todo_completed'] = metadata['user_updated_time'] ?? Date.now(); } if ('due' in md) { const due_date = dateStringToDate(asString(md['due']), null); if (due_date) { metadata['todo_due'] = due_date; } } } // Tags are handled separately from typical metadata let tags: string[] = []; if ('tags' in md && Array.isArray(md['tags'])) { // Only create unique tags tags = md['tags'] as string[]; } else if ('keywords' in md && Array.isArray(md['keywords'])) { // Support for r-markdown/pandoc. Note that "keywords" may be an empty field in the input // document, which would be parsed as just "null", so this is why we need to check that it // is an array. Fixes: https://github.com/laurent22/joplin/issues/13008 tags = tags.concat(md['keywords'] as string[]); } // Only create unique tags tags = [...new Set(tags)]; metadata['body'] = body; return { metadata, tags }; };