feat: Initial clean release of CHM

packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.test.ts

@@ -0,0 +1,223 @@
+import checkIfPluginCanBeAdded from './checkIfPluginCanBeAdded';
+
+describe('checkIfPluginCanBeAdded', () => {
+
+	test('should not add if already a plugin with this ID but different package name', () => {
+		const testCases = [
+			[
+				{
+					'test': {
+						id: 'test',
+						_npm_package_name: 'original',
+					},
+				},
+				{
+					id: 'test',
+					_npm_package_name: 'original',
+				},
+				true,
+			],
+			[
+				{
+					'test': {
+						id: 'test',
+						_npm_package_name: 'original',
+					},
+				},
+				{
+					id: 'test',
+					_npm_package_name: 'cantdothat',
+				},
+				false,
+			],
+		];
+
+		for (const t of testCases) {
+			const [existingManifests, manifest, shouldWork] = t;
+
+			let hasThrown = false;
+			try {
+				checkIfPluginCanBeAdded(existingManifests as unknown as Parameters<typeof checkIfPluginCanBeAdded>[0], manifest as unknown as Parameters<typeof checkIfPluginCanBeAdded>[1], { source: 'npm' });
+			} catch (error) {
+				hasThrown = true;
+			}
+
+			expect(!hasThrown).toBe(shouldWork);
+		}
+	});
+
+	test('should not add if already a plugin with this ID but a different case', () => {
+		const testCases = [
+			[
+				{
+					'test': {
+						id: 'test',
+						_npm_package_name: 'original',
+					},
+				},
+				{
+					id: 'newone',
+					_npm_package_name: 'otherpackage',
+				},
+				true,
+			],
+			[
+				{
+					'test': {
+						id: 'test',
+						_npm_package_name: 'original',
+					},
+				},
+				{
+					id: 'Test',
+					_npm_package_name: 'original',
+				},
+				false,
+			],
+		];
+
+		for (const t of testCases) {
+			const [existingManifests, manifest, shouldWork] = t;
+
+			let hasThrown = false;
+			try {
+				checkIfPluginCanBeAdded(existingManifests as unknown as Parameters<typeof checkIfPluginCanBeAdded>[0], manifest as unknown as Parameters<typeof checkIfPluginCanBeAdded>[1], { source: 'npm' });
+			} catch (error) {
+				hasThrown = true;
+			}
+
+			expect(!hasThrown).toBe(shouldWork);
+		}
+	});
+
+	test('should not add if already a plugin with this ID but different repository url', () => {
+		const testCases = [
+			[
+				{
+					'test': {
+						id: 'test',
+						repository_url: 'https://github.com/laurent22/joplin',
+					},
+				},
+				{
+					id: 'test',
+					repository_url: 'https://github.com/laurent22/joplin.git',
+				},
+				true,
+			],
+			[
+				{
+					'test': {
+						id: 'test',
+						repository_url: 'https://github.com/laurent22/joplin',
+					},
+				},
+				{
+					id: 'test',
+					repository_url: 'https://github.com/someone/else',
+				},
+				false,
+			],
+		];
+
+		for (const t of testCases) {
+			const [existingManifests, manifest, shouldWork] = t;
+
+			let hasThrown = false;
+			try {
+				checkIfPluginCanBeAdded(existingManifests as unknown as Parameters<typeof checkIfPluginCanBeAdded>[0], manifest as unknown as Parameters<typeof checkIfPluginCanBeAdded>[1], { source: 'repository' });
+			} catch (error) {
+				hasThrown = true;
+			}
+
+			expect(!hasThrown).toBe(shouldWork);
+		}
+	});
+
+	test('should reject moving a legacy npm plugin to an unverified repository', () => {
+		const existingManifests = {
+			'test': {
+				id: 'test',
+				_npm_package_name: 'original',
+			},
+		};
+		const manifest = {
+			id: 'test',
+			repository_url: 'https://github.com/attacker/plugin',
+		};
+
+		expect(
+			() => checkIfPluginCanBeAdded(
+				existingManifests as unknown as Parameters<typeof checkIfPluginCanBeAdded>[0],
+				manifest as unknown as Parameters<typeof checkIfPluginCanBeAdded>[1],
+				{ source: 'repository' },
+			),
+		).toThrow('A maintainer must verify and register its repository URL');
+	});
+
+	test('should reject a legacy repository migration even if the npm package name is copied', () => {
+		const existingManifests = {
+			'test': {
+				id: 'test',
+				_npm_package_name: 'original',
+			},
+		};
+		const manifest = {
+			id: 'test',
+			repository_url: 'https://github.com/attacker/plugin',
+			_npm_package_name: 'original',
+		};
+
+		expect(
+			() => checkIfPluginCanBeAdded(
+				existingManifests as unknown as Parameters<typeof checkIfPluginCanBeAdded>[0],
+				manifest as unknown as Parameters<typeof checkIfPluginCanBeAdded>[1],
+				{ source: 'repository' },
+			),
+		).toThrow('A maintainer must verify and register its repository URL');
+	});
+
+	test('should not fall back to an npm package name after repository ownership is registered', () => {
+		const existingManifests = {
+			'test': {
+				id: 'test',
+				repository_url: 'https://github.com/original/plugin',
+				_npm_package_name: 'original',
+			},
+		};
+		const manifest = {
+			id: 'test',
+			_npm_package_name: 'original',
+		};
+
+		expect(
+			() => checkIfPluginCanBeAdded(
+				existingManifests as unknown as Parameters<typeof checkIfPluginCanBeAdded>[0],
+				manifest as unknown as Parameters<typeof checkIfPluginCanBeAdded>[1],
+				{ source: 'npm' },
+			),
+		).toThrow('has already been published under repository');
+	});
+
+	test('should report repository ownership when npm targets a repository-origin plugin', () => {
+		const existingManifests = {
+			'test': {
+				id: 'test',
+				repository_url: 'https://github.com/original/plugin',
+			},
+		};
+		const manifest = {
+			id: 'test',
+			_npm_package_name: 'attacker-package',
+		};
+
+		expect(
+			() => checkIfPluginCanBeAdded(
+				existingManifests as unknown as Parameters<typeof checkIfPluginCanBeAdded>[0],
+				manifest as unknown as Parameters<typeof checkIfPluginCanBeAdded>[1],
+				{ source: 'npm' },
+			),
+		).toThrow('is owned by repository "https://github.com/original/plugin" and cannot be updated from npm package "attacker-package"');
+	});
+
+});

packages/plugin-repo-cli/lib/checkIfPluginCanBeAdded.ts

@@ -0,0 +1,59 @@
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+import { normalizeRepoUrl } from './utils';
+import { PluginSubmissionOptions } from './types';
+
+type PluginManifests = Record<string, PluginManifest>;
+
+function caseInsensitiveFindManifest(manifests: PluginManifests, manifestId: string): PluginManifest | null {
+	for (const id of Object.keys(manifests)) {
+		if (id.toLowerCase() === manifestId.toLowerCase()) return manifests[id];
+	}
+	return null;
+}
+
+export default function(existingManifests: PluginManifests, manifest: PluginManifest, options: PluginSubmissionOptions) {
+	// If there's already a plugin with this ID published under a different
+	// repository/package name, we skip it. Otherwise it would allow anyone to overwrite
+	// someone else plugin just by using the same ID. So the first plugin with
+	// this ID that was originally added is kept.
+	//
+	// We need case insensitive match because the filesystem might be case
+	// insensitive too.
+	const originalManifest = caseInsensitiveFindManifest(existingManifests, manifest.id);
+
+	if (originalManifest) {
+		const originalRepo = normalizeRepoUrl(originalManifest.repository_url);
+		const newRepo = normalizeRepoUrl(manifest.repository_url);
+
+		if (options.source === 'npm') {
+			if (originalRepo && !originalManifest._npm_package_name) {
+				throw new Error(`Plugin "${manifest.id}" is owned by repository "${originalManifest.repository_url}" and cannot be updated from npm package "${manifest._npm_package_name}".`);
+			}
+
+			if (originalManifest._npm_package_name !== manifest._npm_package_name) {
+				throw new Error(`Plugin "${manifest.id}" from npm package "${manifest._npm_package_name}" has already been published under npm package "${originalManifest._npm_package_name}". Plugin from package "${originalManifest._npm_package_name}" will not be imported.`);
+			}
+
+			// We will include repository check as a mandatory check even in npm
+			// The repository URL must not be removed or changed, but this will allow
+			// updating of plugin if both old and new manifest does not have repository url
+			if (originalRepo && (!newRepo || originalRepo !== newRepo)) {
+				throw new Error(`Plugin "${manifest.id}" from repository "${manifest.repository_url}" has already been published under repository "${originalManifest.repository_url}".`);
+			}
+		} else if (options.source === 'repository') {
+			if (originalRepo) {
+				if (!newRepo || originalRepo !== newRepo) {
+					throw new Error(`Plugin "${manifest.id}" from repository "${manifest.repository_url}" has already been published under repository "${originalManifest.repository_url}".`);
+				}
+			} else if (newRepo) {
+				throw new Error(`Plugin "${manifest.id}" has already been published from npm package "${originalManifest._npm_package_name}". A maintainer must verify and register its repository URL before it can be published from a repository.`);
+			}
+		}
+
+		// Don't add a plugin if there is already a plugin with the same ID
+		// but different casing
+		if (originalManifest.id !== manifest.id) {
+			throw new Error(`Plugin "${manifest.id}" cannot be published because there is already a plugin with ID "${originalManifest.id}". A new package, under a different name, should be published if the plugin ID needs to change.`);
+		}
+	}
+}

packages/plugin-repo-cli/lib/errorsHaveChanged.test.ts

@@ -0,0 +1,66 @@
+import errorsHaveChanged from './errorsHaveChanged';
+import { ImportErrors } from './types';
+
+describe('errorsHaveChanged', () => {
+
+	test('should tell if an errors object has changed', () => {
+		const testCases: [ImportErrors, ImportErrors, boolean][] = [
+			[
+				{
+					'one': '111',
+					'two': '222',
+				},
+				{
+					'two': '222',
+					'one': '111',
+				},
+				false,
+			],
+			[
+				{
+					'one': '111',
+					'two': '222',
+				},
+				{
+					'one': '111',
+					'two': '222',
+				},
+				false,
+			],
+			[
+				{
+					'one': '111',
+					'two': '222',
+				},
+				{
+					'one': '111',
+				},
+				true,
+			],
+			[
+				{
+					'one': '111',
+					'two': '222',
+				},
+				{
+					'one': '222',
+					'two': '111',
+				},
+				true,
+			],
+			[
+				{},
+				{},
+				false,
+			],
+		];
+
+		for (const t of testCases) {
+			const [oldErrors, newErrors, expected] = t;
+
+			const result = errorsHaveChanged(oldErrors, newErrors);
+			expect(result).toBe(expected);
+		}
+	});
+
+});

packages/plugin-repo-cli/lib/errorsHaveChanged.ts

@@ -0,0 +1,11 @@
+import { ImportErrors } from './types';
+
+export default function(previousErrors: ImportErrors, newErrors: ImportErrors): boolean {
+	if (Object.keys(previousErrors).length !== Object.keys(newErrors).length) return true;
+
+	for (const packageName of Object.keys(previousErrors)) {
+		if (newErrors[packageName] !== previousErrors[packageName]) return true;
+	}
+
+	return false;
+}

packages/plugin-repo-cli/lib/gitCompareUrl.test.ts

@@ -0,0 +1,56 @@
+import gitCompareUrl from './gitCompareUrl';
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+
+type GitCompareManifest = Pick<PluginManifest, 'repository_url' | '_publish_commit'>;
+
+describe('gitCompareUrl', () => {
+
+	test('should create a compare URL', () => {
+		const testCases: [GitCompareManifest, GitCompareManifest | null, string | null][] = [
+			[
+				{
+					repository_url: 'https://github.com/JackGruber/joplin-plugin-copytags',
+					_publish_commit: 'master:9ec4a476a54440ac43422c34e179dcabfca1e5a0',
+				},
+				{
+					repository_url: 'https://github.com/JackGruber/joplin-plugin-copytags',
+					_publish_commit: 'master:b52b01f6d3b709a811ac214253636a7c207c87dd',
+				},
+				'https://github.com/JackGruber/joplin-plugin-copytags/compare/b52b01f6d3b709a811ac214253636a7c207c87dd..9ec4a476a54440ac43422c34e179dcabfca1e5a0',
+			],
+			[
+				{
+					repository_url: 'https://github.com/JackGruber/joplin-plugin-copytags.git',
+					_publish_commit: 'master:9ec4a476a54440ac43422c34e179dcabfca1e5a0',
+				},
+				{
+					repository_url: 'https://github.com/JackGruber/joplin-plugin-copytags.git',
+					_publish_commit: 'master:b52b01f6d3b709a811ac214253636a7c207c87dd',
+				},
+				'https://github.com/JackGruber/joplin-plugin-copytags/compare/b52b01f6d3b709a811ac214253636a7c207c87dd..9ec4a476a54440ac43422c34e179dcabfca1e5a0',
+			],
+			[
+				{
+					repository_url: 'https://github.com/JackGruber/joplin-plugin-copytags',
+					_publish_commit: 'master:9ec4a476a54440ac43422c34e179dcabfca1e5a0',
+				},
+				null,
+				'https://github.com/JackGruber/joplin-plugin-copytags/tree/9ec4a476a54440ac43422c34e179dcabfca1e5a0',
+			],
+			[
+				{
+					_publish_commit: 'master:9ec4a476a54440ac43422c34e179dcabfca1e5a0',
+				},
+				null,
+				null,
+			],
+		];
+
+		for (const t of testCases) {
+			const [manifest, previousManifest, expected] = t;
+			const actual = gitCompareUrl(manifest, previousManifest);
+			expect(actual).toBe(expected);
+		}
+	});
+
+});

packages/plugin-repo-cli/lib/gitCompareUrl.ts

@@ -0,0 +1,36 @@
+import { rtrimSlashes } from '@joplin/lib/path-utils';
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+
+function removeBranch(commit: string): string {
+	if (!commit) return '';
+
+	if (commit.indexOf(':') >= 0) {
+		const s = commit.split(':');
+		return s[1];
+	}
+
+	return commit;
+}
+
+const formatRepoUrl = (url: string): string => {
+	if (!url) return url;
+	if (url.endsWith('.git')) return url.substring(0, url.length - 4);
+	return url;
+};
+
+export default function(manifest: Pick<PluginManifest, 'repository_url' | '_publish_commit'>, previousManifest: Pick<PluginManifest, '_publish_commit'> | null = null): string {
+	// "repository_url": "https://github.com/JackGruber/joplin-plugin-copytags",
+	// "_publish_commit": "master:b52b01f6d3b709a811ac214253636a7c207c87dd",
+
+	// https://github.com/JackGruber/joplin-plugin-copytags/compare/9ec4a476a54440ac43422c34e179dcabfca1e5a0..b52b01f6d3b709a811ac214253636a7c207c87dd
+
+	const repoUrl: string = formatRepoUrl(manifest.repository_url);
+	const commit: string = removeBranch(manifest._publish_commit);
+	const previousCommit: string = previousManifest ? removeBranch(previousManifest._publish_commit) : '';
+
+	if (!repoUrl) return null;
+	if (!commit && !previousCommit) return repoUrl;
+	if (commit && !previousCommit) return `${rtrimSlashes(repoUrl)}/tree/${commit}`;
+
+	return `${rtrimSlashes(repoUrl)}/compare/${previousCommit}..${commit}`;
+}

packages/plugin-repo-cli/lib/overrideUtils.test.ts

@@ -0,0 +1,107 @@
+import { applyManifestOverrides, getObsoleteManifests, getSupersededPackages, ManifestOverrides } from './overrideUtils';
+
+describe('overrideUtils', () => {
+
+	test('should get the obsolete manifests', () => {
+		const manifestOverrides: Record<string, { _obsolete?: boolean; description?: string; [key: string]: unknown }> = {
+			'ambrt.backlinksToNote': {
+				'manifest_version': 1,
+				'id': 'ambrt.backlinksToNote',
+				'app_min_version': '1.5',
+				'version': '1.0.2',
+				'name': 'Backlinks to note',
+				'description': 'Creates backlinks to opened note',
+				'author': 'a',
+				'homepage_url': 'https://discourse.joplinapp.org/t/insert-referencing-notes-backlinks-plugin/13632',
+				'_publish_hash': 'sha256:5676da6b9ad71fc5a9779d3bde13f17de5352344711e135f0db8c62c6dbb5696',
+				'_publish_commit': 'master:19e515bd67e51cc37bd270a32d2898ca009a0de2',
+				'_npm_package_name': 'joplin-plugin-backlinks',
+				'_obsolete': true,
+			},
+			'MyPlugin': {
+				'manifest_version': 1,
+				'id': 'MyPlugin',
+				'app_min_version': '1.6',
+				'version': '1.0.0',
+				'name': 'Testing New Plugin',
+				'description': 'bla',
+				'author': 'bla',
+				'homepage_url': 'bla',
+				'repository_url': 'bla',
+				'_publish_hash': 'sha256:065285d06ea3c084e7f8f8c23583de8d70c4d586274a242c4c750f6faad8c7cb',
+				'_publish_commit': '',
+				'_npm_package_name': 'joplin-plugin-testing-new-plugin',
+				'_obsolete': true,
+			},
+			'io.github.jackgruber.copytags': {
+				'_recommended': true,
+			},
+		};
+
+		const obsoletes = getObsoleteManifests(manifestOverrides);
+		expect(Object.keys(obsoletes).sort()).toEqual(['MyPlugin', 'ambrt.backlinksToNote']);
+		expect(obsoletes['ambrt.backlinksToNote'].description).toBe('Creates backlinks to opened note');
+	});
+
+	test('should apply the overrides', () => {
+		const manifests = {
+			'io.github.jackgruber.copytags': {
+				'manifest_version': 1,
+				'id': 'io.github.jackgruber.copytags',
+				'app_min_version': '1.6.2',
+				'version': '1.0.1',
+				'name': 'Tagging',
+				'description': 'Plugin to extend the Joplin tagging menu with a copy all tags and a tagging dialog with more control. (Formerly Copy Tags).',
+				'author': 'JackGruber',
+				'homepage_url': 'https://github.com/JackGruber/joplin-plugin-tagging/blob/master/README.md',
+				'repository_url': 'https://github.com/JackGruber/joplin-plugin-tagging',
+				'keywords': [
+					'duplicate',
+					'copy',
+					'tags',
+					'tagging',
+					'tag',
+				],
+				'_publish_hash': 'sha256:88daaf234a9b47e5644a8de6f830a801d12edbe41ea5364d994773e89eeafeef',
+				'_publish_commit': 'master:64c0510e3236df7788a8d10ec28dcfbb4c2bdbb7',
+				'_npm_package_name': 'joplin-plugin-copytags',
+			},
+			'joplin.plugin.ambrt.backlinksToNote': {
+				'manifest_version': 1,
+				'id': 'joplin.plugin.ambrt.backlinksToNote',
+				'app_min_version': '1.7',
+				'version': '2.0.11',
+				'name': 'Automatic Backlinks to note',
+				'description': 'Creates backlinks to opened note, also in automatic way',
+				'author': 'ambrt,cyingfan',
+				'homepage_url': 'https://discourse.joplinapp.org/t/insert-referencing-notes-backlinks-plugin/13632',
+				'_publish_hash': 'sha256:df57930d1ab62d4297dad0bb1764888935fcbf6ca8c04e3a843e86a260735c51',
+				'_publish_commit': 'master:98102718a9c0fa9416d654451b18602798c4d3bb',
+				'_npm_package_name': 'joplin-plugin-backlinks',
+			},
+		};
+
+		const overrides: ManifestOverrides = {
+			'io.github.jackgruber.copytags': {
+				_recommended: true,
+			},
+		};
+
+		const updatedManifests = applyManifestOverrides(manifests, overrides);
+		expect(updatedManifests['io.github.jackgruber.copytags']._recommended).toBe(true);
+		expect(updatedManifests['joplin.plugin.ambrt.backlinksToNote']._recommended).toBe(undefined);
+	});
+
+	test('should get superseded packages', () => {
+		const manifestOverrides = {
+			'joplin.plugin.benji.favorites': {
+				_superseded_package: 'joplin-plugin-benji-favorites',
+			},
+			'io.github.jackgruber.copytags': {
+				_recommended: true,
+			},
+		};
+		const supersededPackages = getSupersededPackages(manifestOverrides);
+		expect(supersededPackages).toMatchObject(['joplin-plugin-benji-favorites']);
+	});
+});

packages/plugin-repo-cli/lib/overrideUtils.ts

@@ -0,0 +1,67 @@
+import * as path from 'path';
+import { readJsonFile } from './utils';
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+
+export interface ManifestOverride {
+	_obsolete?: boolean;
+	_recommended?: boolean;
+	_superseded_package?: string;
+}
+
+export type ManifestOverrides = Record<string, ManifestOverride>;
+type PluginManifests = Record<string, PluginManifest>;
+
+export function applyManifestOverrides(manifests: PluginManifests, overrides: ManifestOverrides) {
+	for (const [pluginId, override] of Object.entries(overrides)) {
+		const manifest = manifests[pluginId] as PluginManifest & Record<string, unknown>;
+
+		if (!manifest) {
+			// If the manifest does not exist in the destination, it means the
+			// plugin has been taken out, so the obsolete property should be set
+			// to `true`. If not, it's an error.
+			if (!override._obsolete) {
+				console.error(`Could not find manifest for plugin ${pluginId} when applying override`);
+			}
+			continue;
+		}
+
+		for (const [propName, propValue] of Object.entries(override)) {
+			manifest[propName] = propValue;
+		}
+	}
+
+	return manifests;
+}
+
+export function getObsoleteManifests<T extends { _obsolete?: boolean }>(overrides: Record<string, T>): Record<string, T> {
+	const output: Record<string, T> = {};
+
+	for (const [pluginId, override] of Object.entries(overrides)) {
+		if (override._obsolete) {
+			output[pluginId] = override;
+		}
+	}
+
+	return output;
+}
+
+function pluginManifestOverridesPath(repoDir: string): string {
+	return path.resolve(repoDir, 'manifestOverrides.json');
+}
+
+export async function readManifestOverrides(repoDir: string): Promise<ManifestOverrides> {
+	return readJsonFile(pluginManifestOverridesPath(repoDir), {});
+}
+
+export const getSupersededPackages = (manifestOverrides: ManifestOverrides): string[] => {
+	const supersededPackages = [];
+
+	for (const id in manifestOverrides) {
+		const supersededPackage = manifestOverrides[id]._superseded_package;
+		if (supersededPackage) {
+			supersededPackages.push(supersededPackage);
+		}
+	}
+
+	return supersededPackages;
+};

packages/plugin-repo-cli/lib/searchPlugins.ts

@@ -0,0 +1,79 @@
+import { URLSearchParams } from 'node:url';
+
+export interface PackageInfo {
+	name: string;
+	version: string;
+	date: string;
+	keywords: string[];
+}
+
+interface SearchResult {
+	package: PackageInfo;
+}
+
+interface SearchResponse {
+	objects: SearchResult[];
+	total: number;
+}
+
+function validateResponse(response: unknown): asserts response is SearchResponse {
+	if (typeof response !== 'object') throw new Error('Invalid response (must be an object)');
+	if (!('total' in response) || typeof response.total !== 'number') {
+		throw new Error('Invalid response.total');
+	}
+	if (!('objects' in response) || !Array.isArray(response.objects)) {
+		throw new Error('response.objects must be an array');
+	}
+	for (const object of response.objects) {
+		if (!('package' in object)) throw new Error('Missing "package" field');
+		const packageField = object.package;
+		if (typeof packageField.name !== 'string') throw new Error('package.name: Invalid type');
+		if (typeof packageField.version !== 'string') throw new Error('package.version: Invalid type');
+		if (typeof packageField.date !== 'string') throw new Error('package.date: Invalid type');
+		if (!Array.isArray(packageField.keywords)) throw new Error('package.keywords: Invalid type');
+	}
+}
+
+const searchPlugins = async () => {
+	const pageSize = 100;
+	const makeRequest = async (start: number): Promise<SearchResponse> => {
+		const urlParams = new URLSearchParams({
+			text: 'keywords:joplin-plugin',
+			size: String(pageSize),
+			from: String(start),
+		});
+		// API documentation: https://github.com/npm/registry/blob/main/docs/REGISTRY-API.md#get-v1search
+		const query = `https://registry.npmjs.org/-/v1/search?${urlParams.toString()}`;
+		const result = await fetch(query);
+		const json: unknown = await result.json();
+		validateResponse(json);
+		return json;
+	};
+
+	const packageInfos: PackageInfo[] = [];
+	const addPackageInfos = (response: SearchResponse) => {
+		for (const object of response.objects) {
+			packageInfos.push(object.package);
+		}
+	};
+
+	const firstResponse = await makeRequest(0);
+	let total = firstResponse.total;
+	addPackageInfos(firstResponse);
+
+	for (let page = 1; packageInfos.length < total; page++) {
+		// Cap the maximum number of requests: Fail early in the case where the search query breaks
+		// in the future.
+		if (page >= 100) {
+			throw new Error('More requests sent than expected. Does maximumRequests need to be increased?');
+		}
+
+		const response = await makeRequest(page * pageSize);
+		total = response.total;
+		addPackageInfos(response);
+	}
+
+	return packageInfos;
+};
+
+export default searchPlugins;

packages/plugin-repo-cli/lib/types.ts

@@ -0,0 +1,13 @@
+export type ImportErrors = Record<string, string>;
+
+export type PluginSubmissionSource = 'npm' | 'repository';
+
+export interface PluginSubmissionOptions {
+	source: PluginSubmissionSource;
+}
+
+export interface NpmPackage {
+	name: string;
+	version: string;
+	date: Date;
+}

packages/plugin-repo-cli/lib/updateReadme.test.ts

@@ -0,0 +1,69 @@
+import * as fs from 'fs-extra';
+import updateReadme from './updateReadme';
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+
+describe('updateReadme', () => {
+
+	test('should update the README file', async () => {
+		const manifests: Record<string, PluginManifest> = {
+			'io.github.jackgruber.copytags': {
+				'manifest_version': 1,
+				'id': 'io.github.jackgruber.copytags',
+				'app_min_version': '1.6.2',
+				'version': '1.0.1',
+				'name': 'Tagging',
+				'description': 'Plugin to extend the Joplin tagging menu with a copy all tags and a tagging dialog with more control. (Formerly Copy Tags).',
+				'author': 'JackGruber',
+				'homepage_url': 'https://github.com/JackGruber/joplin-plugin-tagging/blob/master/README.md',
+				'repository_url': 'https://github.com/JackGruber/joplin-plugin-tagging',
+				'keywords': [
+					'duplicate',
+					'copy',
+					'tags',
+					'tagging',
+					'tag',
+				],
+				'_publish_hash': 'sha256:88daaf234a9b47e5644a8de6f830a801d12edbe41ea5364d994773e89eeafeef',
+				'_publish_commit': 'master:64c0510e3236df7788a8d10ec28dcfbb4c2bdbb7',
+				'_npm_package_name': 'joplin-plugin-copytags',
+			},
+			'joplin.plugin.ambrt.backlinksToNote': {
+				'manifest_version': 1,
+				'id': 'joplin.plugin.ambrt.backlinksToNote',
+				'app_min_version': '1.7',
+				'version': '2.0.10',
+				'name': 'Automatic Backlinks to note',
+				'description': 'Creates backlinks to opened note, also in automatic way',
+				'author': 'ambrt',
+				'homepage_url': 'https://discourse.joplinapp.org/t/insert-referencing-notes-backlinks-plugin/13632',
+				'_publish_hash': 'sha256:b5dec8d00f19e34c4fe1dc0ed380b6743aa7debfd8f600ead0d6866ba21466f1',
+				'_publish_commit': 'master:05020664a6a7f567c466e3fbad1d7e7ec64dc369',
+				'_npm_package_name': 'joplin-plugin-backlinks',
+			},
+		};
+
+		const tempFilePath = `${__dirname}/test_README.md`;
+		await fs.writeFile(tempFilePath, '<!-- PLUGIN_LIST -->\n<!-- PLUGIN_LIST -->');
+
+		await updateReadme(tempFilePath, manifests);
+
+		const content = await fs.readFile(tempFilePath, 'utf8');
+
+		// Check header
+		expect(content.includes('| &nbsp; | &nbsp; | Name  | Version | Description | Author |')).toBe(true);
+
+		// Check plugin content
+		expect(content.includes('https://github.com/JackGruber/joplin-plugin-tagging/blob/master/README.md')).toBe(true);
+		expect(content.includes('https://github.com/joplin/plugins/raw/master/plugins/joplin.plugin.ambrt.backlinksToNote/plugin.jpl')).toBe(true);
+		expect(content.includes('Automatic Backlinks to note')).toBe(true);
+		expect(content.includes('2.0.10')).toBe(true);
+		expect(content.includes('Creates backlinks to opened note, also in automatic way')).toBe(true);
+		expect(content.includes('ambrt')).toBe(true);
+
+		// Check that it keeps the markers
+		expect(content.split('<!-- PLUGIN_LIST -->').length - 1).toBe(2);
+
+		await fs.remove(tempFilePath);
+	});
+
+});

packages/plugin-repo-cli/lib/updateReadme.ts

@@ -0,0 +1,66 @@
+import * as fs from 'fs-extra';
+import markdownUtils, { MarkdownTableHeader, MarkdownTableRow } from '@joplin/lib/markdownUtils';
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+
+export default async function(readmePath: string, manifests: Record<string, PluginManifest>) {
+	let rows: MarkdownTableRow[] = [];
+
+	for (const pluginId in manifests) {
+		rows.push(manifests[pluginId] as unknown as MarkdownTableRow);
+	}
+
+	rows = rows.map(row => {
+		return {
+			...row,
+			download_url: `https://github.com/joplin/plugins/raw/master/plugins/${row.id}/plugin.jpl`,
+		};
+	});
+
+	const headers: MarkdownTableHeader[] = [
+		{
+			name: 'homepage_url',
+			label: '&nbsp;',
+			filter: (value: string) => {
+				if (!value) return '-';
+				return `[🏠](${markdownUtils.escapeLinkUrl(value)})`;
+			},
+		},
+		{
+			name: 'download_url',
+			label: '&nbsp;',
+			filter: (value: string) => {
+				if (!value) return '-';
+				return `[⬇️](${markdownUtils.escapeLinkUrl(value)})`;
+			},
+		},
+		{
+			name: 'name',
+			label: 'Name',
+		},
+		{
+			name: 'version',
+			label: 'Version',
+		},
+		{
+			name: 'description',
+			label: 'Description',
+		},
+		{
+			name: 'author',
+			label: 'Author',
+		},
+	];
+
+	rows.sort((a, b) => {
+		return a.name.toLowerCase() < b.name.toLowerCase() ? -1 : +1;
+	});
+
+	const mdTable = markdownUtils.createMarkdownTable(headers, rows);
+
+	const tableRegex = /<!-- PLUGIN_LIST -->([^]*)<!-- PLUGIN_LIST -->/;
+
+	const content = await fs.pathExists(readmePath) ? await fs.readFile(readmePath, 'utf8') : '<!-- PLUGIN_LIST -->\n<!-- PLUGIN_LIST -->';
+	const newContent = content.replace(tableRegex, `<!-- PLUGIN_LIST -->\n${mdTable}\n<!-- PLUGIN_LIST -->`);
+
+	await fs.writeFile(readmePath, newContent, 'utf8');
+}

packages/plugin-repo-cli/lib/utils.ts

@@ -0,0 +1,34 @@
+import * as fs from 'fs-extra';
+
+export async function readJsonFile<T = unknown>(manifestPath: string, defaultValue: T = null): Promise<T> {
+	if (!(await fs.pathExists(manifestPath))) {
+		if (defaultValue === null) throw new Error(`No such file: ${manifestPath}`);
+		return defaultValue;
+	}
+
+	const content = await fs.readFile(manifestPath, 'utf8');
+	return JSON.parse(content);
+}
+
+function stripOffPackageOrg(name: string): string {
+	const n = name.split('/');
+	if (n[0][0] === '@') n.splice(0, 1);
+	return n.join('/');
+}
+
+export function isJoplinPluginPackage(pack: { keywords?: string[]; name: string }): boolean {
+	if (!pack.keywords || !pack.keywords.includes('joplin-plugin')) return false;
+	if (stripOffPackageOrg(pack.name).indexOf('joplin-plugin') !== 0) return false;
+	return true;
+}
+
+export const normalizeRepoUrl = (url: string) => {
+	if (!url) return '';
+	const normalized = url
+		.trim()
+		.toLowerCase()
+		.replace(/^(https?:\/\/)?(www\.)?github\.com\//, '')
+		.replace(/\.git$/, '')
+		.replace(/\/$/, '');
+	return normalized;
+};

packages/plugin-repo-cli/lib/validateUntrustedManifest.test.ts

@@ -0,0 +1,246 @@
+
+import validateUntrustedManifest from './validateUntrustedManifest';
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+
+const originalManifests = {
+	'joplin-plugin.this.is.a.test': {
+		id: 'joplin-plugin.this.is.a.test',
+		_npm_package_name: 'joplin-plugin-this-is-a-test',
+		version: '0.0.1',
+	},
+} as unknown as Record<string, PluginManifest>;
+
+// Note: Most of the checks below have additional tests in other files.
+// This test suite is primarily to ensure that the checks are present.
+
+describe('validateUntrustedManifest', () => {
+	test('should only allow valid plugin IDs', () => {
+		const badManifest = {
+			id: 'bad',
+			_npm_package_name: 'joplin-plugin-test',
+			version: '1.2.34',
+		};
+
+		expect(
+			() => validateUntrustedManifest(badManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).toThrow('ID cannot be shorter than');
+
+
+		const goodManifest = {
+			...badManifest,
+			id: 'this.plugin.has.a.better.id',
+		};
+
+		expect(
+			() => validateUntrustedManifest(goodManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).not.toThrow();
+	});
+
+	test('should only allow valid plugin versions', () => {
+		const badManifest = {
+			id: 'joplin-plugin.this.is.a.test',
+			_npm_package_name: 'joplin-plugin-this-is-a-test',
+			version: 'bad!',
+		};
+
+		expect(
+			() => validateUntrustedManifest(badManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).toThrow();
+
+
+		const goodManifest = {
+			...badManifest,
+			version: '0.0.2',
+		};
+
+		expect(
+			() => validateUntrustedManifest(goodManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).not.toThrow();
+	});
+
+	test('should only allow valid plugin platforms', () => {
+		const badManifest = {
+			id: 'com.example.a-plugin-for-a-fake-platform',
+			_npm_package_name: 'joplin-plugin-plugin-for-an-invalid-version',
+			version: '1.2.3',
+			platforms: [3, 4, 5],
+		};
+
+		expect(
+			() => validateUntrustedManifest(badManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).toThrow();
+
+		const goodManifest = {
+			...badManifest,
+			platforms: ['mobile', 'desktop'],
+		};
+
+		expect(
+			() => validateUntrustedManifest(goodManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).not.toThrow();
+	});
+
+	test('should not allow plugin authors to mark their own plugins as recommended', () => {
+		const newManifest1 = {
+			id: 'joplin-plugin.this.is.another.test',
+			_recommended: true,
+			_npm_package_name: 'joplin-plugin-another-test',
+			version: '1.2.3',
+		};
+
+		expect(
+			() => validateUntrustedManifest(newManifest1 as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).toThrow('mark itself as recommended');
+
+		// Should also throw for falsey values
+		const newManifest2 = {
+			id: 'joplin-plugin.yet.another.test',
+			_recommended: '',
+			_npm_package_name: 'another-test',
+			version: '1.2.3',
+		};
+
+		expect(
+			() => validateUntrustedManifest(newManifest2 as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).toThrow('mark itself as recommended');
+	});
+
+	test('should not allow changing the NPM package for an existing plugin', () => {
+		const newManifest = {
+			id: 'joplin-plugin.this.is.a.test',
+			_npm_package_name: 'joplin-plugin-test',
+			version: '0.0.2',
+		};
+
+		expect(
+			() => validateUntrustedManifest(newManifest as unknown as PluginManifest, originalManifests, { source: 'npm' }),
+		).toThrow();
+	});
+
+	test('should allow the same npm package to register its first repository URL', () => {
+		const newManifest = {
+			id: 'joplin-plugin.this.is.a.test',
+			_npm_package_name: 'joplin-plugin-this-is-a-test',
+			repository_url: 'https://github.com/example/this-is-a-test',
+			version: '0.0.2',
+		};
+
+		expect(
+			() => validateUntrustedManifest(
+				newManifest as unknown as PluginManifest,
+				originalManifests,
+				{ source: 'npm' },
+			),
+		).not.toThrow();
+	});
+
+	test('should require the same npm package after repository ownership is registered', () => {
+		const manifestsWithRepository = {
+			'joplin-plugin.this.is.a.test': {
+				...originalManifests['joplin-plugin.this.is.a.test'],
+				repository_url: 'https://github.com/example/this-is-a-test',
+			},
+		};
+		const newManifest = {
+			id: 'joplin-plugin.this.is.a.test',
+			_npm_package_name: 'joplin-plugin-attacker',
+			repository_url: 'https://github.com/example/this-is-a-test',
+			version: '0.0.2',
+		};
+
+		expect(
+			() => validateUntrustedManifest(
+				newManifest as unknown as PluginManifest,
+				manifestsWithRepository,
+				{ source: 'npm' },
+			),
+		).toThrow('has already been published under npm package');
+	});
+
+	test('should not allow the npm package to change an established repository URL', () => {
+		const manifestsWithRepository = {
+			'joplin-plugin.this.is.a.test': {
+				...originalManifests['joplin-plugin.this.is.a.test'],
+				repository_url: 'https://github.com/example/this-is-a-test',
+			},
+		};
+		const newManifest = {
+			id: 'joplin-plugin.this.is.a.test',
+			_npm_package_name: 'joplin-plugin-this-is-a-test',
+			repository_url: 'https://github.com/attacker/this-is-a-test',
+			version: '0.0.2',
+		};
+
+		expect(
+			() => validateUntrustedManifest(
+				newManifest as unknown as PluginManifest,
+				manifestsWithRepository,
+				{ source: 'npm' },
+			),
+		).toThrow('has already been published under repository');
+	});
+
+	test('should reject an unverified repository migration for a legacy npm plugin', () => {
+		const newManifest = {
+			id: 'joplin-plugin.this.is.a.test',
+			repository_url: 'https://github.com/attacker/this-is-a-test',
+			version: '0.0.2',
+		};
+
+		expect(
+			() => validateUntrustedManifest(
+				newManifest as unknown as PluginManifest,
+				originalManifests,
+				{ source: 'repository' },
+			),
+		).toThrow('A maintainer must verify and register its repository URL');
+	});
+
+	test('should require a repository URL for repository submissions', () => {
+		const manifest = {
+			id: 'com.example.repository-plugin',
+			version: '1.0.0',
+		};
+
+		expect(
+			() => validateUntrustedManifest(
+				manifest as unknown as PluginManifest,
+				originalManifests,
+				{ source: 'repository' },
+			),
+		).toThrow('must specify repository_url');
+	});
+
+	test('should not accept an npm package name from a repository submission', () => {
+		const manifest = {
+			id: 'com.example.repository-plugin',
+			version: '1.0.0',
+			repository_url: 'https://github.com/example/repository-plugin',
+			_npm_package_name: 'copied-package-name',
+		};
+
+		expect(
+			() => validateUntrustedManifest(
+				manifest as unknown as PluginManifest,
+				originalManifests,
+				{ source: 'repository' },
+			),
+		).toThrow('cannot specify _npm_package_name');
+	});
+
+	test('should accept a new repository submission with a repository URL', () => {
+		const manifest = {
+			id: 'com.example.repository-plugin',
+			version: '1.0.0',
+			repository_url: 'https://github.com/example/repository-plugin',
+		};
+
+		expect(
+			() => validateUntrustedManifest(
+				manifest as unknown as PluginManifest,
+				originalManifests,
+				{ source: 'repository' },
+			),
+		).not.toThrow();
+	});
+});

packages/plugin-repo-cli/lib/validateUntrustedManifest.ts

@@ -0,0 +1,42 @@
+import validatePluginId from '@joplin/lib/services/plugins/utils/validatePluginId';
+import validatePluginVersion from '@joplin/lib/services/plugins/utils/validatePluginVersion';
+import validatePluginPlatforms from '@joplin/lib/services/plugins/utils/validatePluginPlatforms';
+import checkIfPluginCanBeAdded from './checkIfPluginCanBeAdded';
+import { PluginManifest } from '@joplin/lib/services/plugins/utils/types';
+import { PluginSubmissionOptions } from './types';
+
+// For npm submissions, assume that
+// 1. manifest._npm_package_name was assigned by the CLI.
+// 2. All fields in repository submissions are untrusted.
+const validateUntrustedManifest = (
+	manifest: PluginManifest,
+	existingManifests: Record<string, PluginManifest>,
+	options: PluginSubmissionOptions,
+) => {
+	// At this point, we need to check the manifest ID as it's used in various
+	// places including as directory name and object key in manifests.json, so
+	// it needs to be correct. It's mostly for security reasons. The other
+	// manifest properties are checked when the plugin is loaded into the app.
+	validatePluginId(manifest.id);
+	validatePluginVersion(manifest.version);
+	validatePluginPlatforms(manifest.platforms);
+
+	// This prevents a plugin author from marking their own plugin as _recommended.
+	if (typeof manifest._recommended !== 'undefined') {
+		throw new Error(`Plugin ${manifest.id} cannot mark itself as recommended.`);
+	}
+
+	if (options.source === 'repository') {
+		if (!manifest.repository_url?.trim()) {
+			throw new Error(`Plugin ${manifest.id} must specify repository_url when published from a repository.`);
+		}
+
+		if (typeof manifest._npm_package_name !== 'undefined') {
+			throw new Error(`Plugin ${manifest.id} cannot specify _npm_package_name when published from a repository.`);
+		}
+	}
+
+	checkIfPluginCanBeAdded(existingManifests, manifest, options);
+};
+
+export default validateUntrustedManifest;