import getLicenses, { DependencyType, LicenseInfo } from './getLicenses'; import { readFile, readdir } from 'fs/promises'; import { dirname, join } from 'path'; import licenseOverrides from './licenseOverrides'; import { LicenseOverride } from './licenseOverrides/types'; import { exists } from 'fs-extra'; import matchApache2 from './utils/matchApache2'; import matchMit from './utils/matchMit'; // Important: Review the output of this tool for correctness interface PackageInfo extends LicenseInfo { packageName: string; } const cachedFetchResults: Map = new Map(); const readOrFetchRepositoryFile = async (pkg: PackageInfo, allowedPaths: string[]): Promise => { for (const path of allowedPaths) { const targetPath = join(pkg.path, path); if (await exists(targetPath)) { const licenseText = await readFile(targetPath, 'utf8'); return licenseText; } } for (const path of allowedPaths) { const cacheKey = `${pkg.repository}/${path}`; if (cachedFetchResults.has(cacheKey)) { const cacheValue = cachedFetchResults.get(cacheKey); if (cacheValue) { return cacheValue; } // Otherwise, try the next allowed path } else { const repositoryMatch = pkg.repository?.match(/^git@github\.com:([^/]+)\/([^.]+)(?:\.git)?$/) ?? pkg.repository?.match(/^https:\/\/(?:www\.)?github\.com\/([^/]+)\/([^/]+)$/); if (repositoryMatch) { const organization = repositoryMatch[1]; const project = repositoryMatch[2]; console.error('Attempting to fetch', path, 'for repository', pkg.repository, 'from GitHub...'); const noticeResult = await fetch(`https://raw.githubusercontent.com/${organization}/${project}/HEAD/${path}`); if (noticeResult.ok) { const result = await noticeResult.text(); cachedFetchResults.set(cacheKey, result); console.error('Fetch success.'); return result; } else { cachedFetchResults.set(cacheKey, null); console.error(`Fetch failed: ${noticeResult.statusText}`); } } else { console.warn('No repository for package', pkg.packageName); } } } return null; }; const getNotice = async (pkg: PackageInfo) => { // Some package overrides lack a `path` if (!pkg.path) { console.error('Not including notices for', pkg.packageName, ' -- no path.'); return ''; } const files = await readdir(pkg.path); const noticeLines = []; for (const fileName of files) { if (/NOTICE(?:S)?(?:\.\w+)?$/i.exec(fileName)) { const noticeContent = await readFile(join(pkg.path, fileName), 'utf8'); noticeLines.push(`${fileName}:\n\n${noticeContent}`); } } // If no notices were found, there may still be such a file in the package's repository // (as is the case for some Amazon AWS packages). if (noticeLines.length === 0 && pkg.licenses.includes('Apache')) { noticeLines.push(await readOrFetchRepositoryFile(pkg, ['NOTICE', 'NOTICE.md'])); } return noticeLines.join('\n\n'); }; const trimBeforeLicenseHeader = (text: string) => { const header = text.match(/#+ License[\n]/i) ?? text.match(/[\n]License[\n]--+[\n]/i); if (header) { return text.substring(header.index); } else { return text; } }; const readLicense = async (pkg: PackageInfo) => { let result = ''; if (pkg.licenseText && !pkg.licenses.includes('UNKNOWN')) { result = pkg.licenseText; } const resolvedLicenseToReadme = pkg.licenseFile && pkg.licenseFile.match(/\/README(\.\w+)?$/); // By default, license-checker-rseidelsohn uses the README when the license can't be // found. This is often wrong, and we can do better: if (pkg.path && (!pkg.licenseFile || resolvedLicenseToReadme)) { result = await readOrFetchRepositoryFile(pkg, ['LICENSE', 'LICENSE.md', 'LICENSE.txt', 'MIT-LICENSE.txt']); } if (!result && pkg.licenseFile) { result = await readFile(pkg.licenseFile, 'utf8'); if (resolvedLicenseToReadme) { result = trimBeforeLicenseHeader(result); } } return result; }; export enum EntryLicenseType { Mit = 'MIT', Apache2 = 'Apache2', Other = '', } type LicenseDetails = { type: EntryLicenseType.Apache2; notice: string; appendix: string; } | { type: EntryLicenseType.Mit; notice: string; copyright: string; } | { type: EntryLicenseType.Other; notice: string; fullText: string; }; interface ReportEntry { packageNames: string[]; license: LicenseDetails; licenseId: string; packageSource: string; } type Report = Record; const buildReport = async (packageNames: string[]): Promise => { const monorepoRootDir = dirname(dirname(dirname(__dirname))); const packagesRootDir = join(monorepoRootDir, 'packages'); const packageOverrides: LicenseOverride[] = []; for (const packageToCheck of packageNames) { if (licenseOverrides[packageToCheck]) { packageOverrides.push(...licenseOverrides[packageToCheck]); } } const report: Report = { [DependencyType.Production]: [], [DependencyType.Development]: [] }; for (const mode of [DependencyType.Production, DependencyType.Development]) { const output: ReportEntry[] = []; report[mode] = output; const dependencyLicenses: Map = new Map(); for (const packageName of packageNames) { const packageDir = join(packagesRootDir, packageName); const packageLicenses = await getLicenses(packageDir, mode, [], packageOverrides); for (const packageName in packageLicenses) { dependencyLicenses.set(packageName, packageLicenses[packageName]); } } // Group into per-repository (some dependencies are broken up into multiple // packages) const repositoryToPackages: Map = new Map(); for (const [packageName, packageData] of dependencyLicenses) { const packageInfo = { packageName, ...packageData, }; const repositoryData = repositoryToPackages.get(packageData.repository); if (repositoryData) { repositoryData.push(packageInfo); } else { repositoryToPackages.set(packageData.repository, [packageInfo]); } } for (const [repository, repositoryData] of repositoryToPackages) { let repositoryOutput: string[] = []; let relevantPackages: string[] = []; let previousLicenseId: string|null = null; let lastLicenseData: LicenseDetails|null = null; const flushOutput = () => { if (relevantPackages.length > 0 || repositoryOutput.length > 0) { if (!lastLicenseData) { throw new Error('lastLicenseData is not set'); } output.push({ packageNames: [...relevantPackages], license: lastLicenseData, licenseId: previousLicenseId, packageSource: repository && repository !== 'null' ? repository : null, }); } relevantPackages = []; repositoryOutput = []; lastLicenseData = null; }; // Try to fetch LICENSE and NOTICE from node_modules for (const pkg of repositoryData) { const currentNotice = await getNotice(pkg); let currentLicenses = pkg.licenses; const licenseText = await readLicense(pkg) ?? 'NONE AVAILABLE'; const addOrSetLicense = (matcher: RegExp, identifier: string) => { if (!currentLicenses.match(matcher)) { // If the license ID was previously unknown, it has now been identified if (currentLicenses === 'UNKNOWN') { currentLicenses = ''; } currentLicenses = [currentLicenses, identifier].filter(license => license.trim()).join(' AND '); } }; // Determine the license data and additional information based on the full license text. // This allows extracting copyright and other information from the full license. let licenseData: LicenseDetails; const apache2Match = matchApache2(licenseText); if (apache2Match) { licenseData = { type: EntryLicenseType.Apache2, appendix: apache2Match.appendix || null, notice: currentNotice, }; addOrSetLicense(/apache.?2/i, 'Apache-2'); } else { const mitMatch = matchMit(licenseText); if (mitMatch) { licenseData = { type: EntryLicenseType.Mit, copyright: mitMatch.copyright, notice: currentNotice, }; addOrSetLicense(/mit/i, 'MIT'); } else { licenseData = { type: EntryLicenseType.Other, fullText: licenseText, notice: currentNotice, }; } } const noticeChanged = lastLicenseData?.notice !== licenseData.notice; const copyrightChanged = lastLicenseData?.type === EntryLicenseType.Mit && licenseData.type === EntryLicenseType.Mit && lastLicenseData.copyright !== lastLicenseData.copyright; if (previousLicenseId !== currentLicenses || noticeChanged || copyrightChanged) { flushOutput(); previousLicenseId = currentLicenses; lastLicenseData = licenseData; } relevantPackages.push(pkg.packageName); } flushOutput(); } } return report; }; export default buildReport;