diff --git a/docs/man_pages/project/configuration/metadata-trim.md b/docs/man_pages/project/configuration/metadata-trim.md new file mode 100644 index 0000000000..72e316209c --- /dev/null +++ b/docs/man_pages/project/configuration/metadata-trim.md @@ -0,0 +1,39 @@ +<% if (isJekyll) { %>--- +title: ns metadata trim +position: 26 +---<% } %> + +# ns metadata trim + +### Description + +Reduces the native API metadata an app ships to what its JavaScript uses. The app's emitted bundle is analyzed for the native symbols it references, and the generator keeps those plus the types they depend on. + +Rules in `native-api-usage.json` still apply. Entries from `whitelist` and from plugins' `uses` are kept in addition to what the analysis finds, and `blacklist` entries are removed. A blacklisted type that a kept API's signature needs can't be removed: on Apple platforms it stays in the metadata, and on Android the build fails and names it. In a NativeScript project these files are found automatically for iOS and Android; other hosts pass them with `--native-api-usage`. + +If the bundle reads globals dynamically (an alias of `globalThis`, for example), its usage cannot be determined and nothing is trimmed. + +### Commands + +Usage | Synopsis +---|--- +Generate trimmed macOS metadata and embed it in a framework | `$ ns metadata trim macos --analyzer --generator [--framework ] [--out ]` +Generate trimmed iOS metadata | `$ ns metadata trim ios --analyzer --generator [--sdk ] [--framework ]` +Write the whitelist for an Android project | `$ ns metadata trim android --android-project ` + +### Arguments + +* `` - One or more emitted JavaScript files or directories. + +### Options + +* `--out` - Output directory. Defaults to `metadata-trim`. +* `--keep` - A `Module:Symbol` pattern to keep regardless of the analysis. Repeat for several. +* `--native-api-usage` - A `native-api-usage.json` file to apply, in the app or plugin format. Repeat for several. +* `--analyzer` - `ns-metadata-symbols` on Apple platforms; overrides the runtime's `seed.js` on Android. +* `--generator` - `objc-metadata-generator`. Use the one from the runtime release the app ships. +* `--framework` - A `NativeScript.framework` to copy into the output directory with the trimmed metadata embedded. +* `--android-project` - The Android project whose `whitelist.mdg` Gradle should use. Build it with `-PnsFilterMetadata`. +* `--sdk` - Xcode SDK name. Defaults to `macosx` or `iphonesimulator`. +* `--arch` - Target architecture. Defaults to `arm64`. +* `--min-version` - Minimum OS version to generate for. diff --git a/lib/bootstrap.ts b/lib/bootstrap.ts index 154a325910..9d6209a6d9 100644 --- a/lib/bootstrap.ts +++ b/lib/bootstrap.ts @@ -196,6 +196,12 @@ registerBuiltInCommand< registerBuiltInCommand< typeof import("./commands/generate").generateCommandDefinition >("generate", () => require("./commands/generate").generateCommandDefinition); +registerBuiltInCommand< + typeof import("./commands/metadata-trim").MetadataTrimCommand +>( + "metadata|trim", + () => require("./commands/metadata-trim").MetadataTrimCommand, +); registerBuiltInCommand< typeof import("./commands/list-platforms").listPlatformsCommandDefinition >( @@ -711,6 +717,7 @@ injector.require( "metadataFilteringService", "./services/metadata-filtering-service", ); +injector.require("metadataTrimService", "./services/metadata-trim-service"); injector.require("tempService", "./services/temp-service"); injector.require("sharedEventBus", "./shared-event-bus"); diff --git a/lib/commands/metadata-trim.ts b/lib/commands/metadata-trim.ts new file mode 100644 index 0000000000..56eca93497 --- /dev/null +++ b/lib/commands/metadata-trim.ts @@ -0,0 +1,129 @@ +import * as path from "path"; +import { + arrayOption, + Command, + CommandOptionsSchema, + stringOption, +} from "../common/define-command"; +import { inject } from "../common/di"; +import { IMetadataFilteringService } from "../definitions/metadata-filtering-service"; +import { + IDeclaredMetadataRules, + IMetadataTrimService, +} from "../definitions/metadata-trim-service"; +import { IProjectData } from "../definitions/project"; +import { IProjectHelper } from "../common/declarations"; +import { readDeclaredRules } from "../services/metadata-trim/declared-rules"; + +const PLATFORMS = ["ios", "macos", "android"]; + +const metadataTrimOptions = { + out: stringOption(), + keep: arrayOption(), + nativeApiUsage: arrayOption(), + analyzer: stringOption(), + generator: stringOption(), + framework: stringOption(), + androidProject: stringOption(), + sdk: stringOption(), + arch: stringOption(), + minVersion: stringOption(), +} satisfies CommandOptionsSchema; + +function formatBytes(bytes: number): string { + return `${(bytes / 1024 / 1024).toFixed(2)} MB`; +} + +function resolved(value: string | undefined): string | undefined { + return value ? path.resolve(value) : undefined; +} + +export class MetadataTrimCommand extends Command({ + name: "metadata|trim", + description: "Reduces native API metadata to what an app's JavaScript uses.", + options: metadataTrimOptions, + params: "any", +}) { + private $logger = inject("logger"); + private $metadataFilteringService = inject( + "metadataFilteringService", + ); + private $metadataTrimService = inject( + "metadataTrimService", + ); + private $projectData = inject("projectData"); + private $projectHelper = inject("projectHelper"); + + public canExecute(): boolean { + const [platform, ...bundles] = this.args; + if (!platform || !PLATFORMS.includes(platform.toLowerCase())) { + this.context.fail(`Specify a platform: ${PLATFORMS.join(", ")}.`); + } + if (!bundles.length) { + this.context.fail( + "Specify at least one emitted JavaScript bundle or directory.", + ); + } + return true; + } + + public async run(): Promise { + const [platform, ...bundles] = this.args; + const options = this.options; + const declared = this.declaredRules(platform.toLowerCase()); + const result = await this.$metadataTrimService.trim({ + platform: platform.toLowerCase(), + bundles: bundles.map((bundle) => path.resolve(bundle)), + outDir: path.resolve(options.out ?? "metadata-trim"), + keep: [...(options.keep ?? []), ...declared.keep], + blacklist: declared.blacklist, + analyzerPath: resolved(options.analyzer), + generatorPath: resolved(options.generator), + framework: resolved(options.framework), + androidProject: resolved(options.androidProject), + sdk: options.sdk, + arch: options.arch, + minVersion: options.minVersion, + }); + + if (declared.keep.length || declared.blacklist.length) { + this.$logger.info( + `Applied native-api-usage.json: ${declared.keep.length} kept, ${declared.blacklist.length} removed.`, + ); + } + if (result.metadataBytes !== undefined) { + this.$logger.info( + `Generated ${formatBytes(result.metadataBytes)} of metadata: ${result.metadataPath}`, + ); + } + if (result.installedTo) { + const saved = result.replacedBytes + ? ` (${formatBytes(result.replacedBytes)} smaller)` + : ""; + this.$logger.info(`Installed to ${result.installedTo}${saved}`); + } + if (result.nextStep) { + this.$logger.info(result.nextStep); + } + } + + private declaredRules(platform: string): IDeclaredMetadataRules { + const files = (this.options.nativeApiUsage ?? []).map((file) => + readDeclaredRules(path.resolve(file)), + ); + // A standard project's files only exist for iOS and Android. + if (this.$projectHelper.projectDir && platform !== "macos") { + this.$projectData.initializeProjectData(); + files.push( + this.$metadataFilteringService.getDeclaredRules( + this.$projectData, + platform, + ), + ); + } + return { + keep: files.flatMap((rules) => rules.keep), + blacklist: files.flatMap((rules) => rules.blacklist), + }; + } +} diff --git a/lib/definitions/metadata-filtering-service.d.ts b/lib/definitions/metadata-filtering-service.d.ts index 7fe802583b..9a455a1bcf 100644 --- a/lib/definitions/metadata-filtering-service.d.ts +++ b/lib/definitions/metadata-filtering-service.d.ts @@ -1,4 +1,5 @@ import { IProjectData } from "./project"; +import { IDeclaredMetadataRules } from "./metadata-trim-service"; /** * Describes service used to generate necessary files to filter the native metadata generation. @@ -45,4 +46,15 @@ interface IMetadataFilteringService { * @returns {void} */ generateMetadataFilters(projectData: IProjectData, platform: string): void; + + /** + * Reads the rules the app and its plugins declare in native-api-usage.json. + * @param {IProjectData} projectData Information about the current project. + * @param {string} platform The platform to read rules for. + * @returns {IDeclaredMetadataRules} The patterns to keep and to remove. + */ + getDeclaredRules( + projectData: IProjectData, + platform: string, + ): IDeclaredMetadataRules; } diff --git a/lib/definitions/metadata-trim-service.d.ts b/lib/definitions/metadata-trim-service.d.ts new file mode 100644 index 0000000000..08d10d2afa --- /dev/null +++ b/lib/definitions/metadata-trim-service.d.ts @@ -0,0 +1,71 @@ +/** What `ns metadata trim` was asked to do. */ +export interface IMetadataTrimOptions { + platform: string; + /** Emitted JavaScript (files or directories) the app actually runs. */ + bundles: string[]; + outDir: string; + /** Filter patterns (`Module:Symbol`) to keep regardless of what is found. */ + keep: string[]; + /** Filter patterns to remove even if the app references them. */ + blacklist?: string[]; + analyzerPath?: string; + generatorPath?: string; + /** A NativeScript.framework to embed the trimmed metadata into. */ + framework?: string; + /** An Android project root, the directory that owns `whitelist.mdg`. */ + androidProject?: string; + sdk?: string; + arch?: string; + minVersion?: string; +} + +/** What native-api-usage.json files declare, as filter patterns. */ +export interface IDeclaredMetadataRules { + keep: string[]; + blacklist: string[]; +} + +/** The native symbols an app's JavaScript can reach. */ +export interface IMetadataUsage { + whitelistPath: string; + patternCount: number; + /** The analysis could not bound the app's usage, so nothing can be removed. */ + unrestricted: boolean; +} + +export interface IMetadataTrimResult { + usage: IMetadataUsage; + /** Set when metadata was generated here rather than by a later build step. */ + metadataPath?: string; + metadataBytes?: number; + /** Where the result was installed, when an install target was given. */ + installedTo?: string; + replacedBytes?: number; + /** What the caller still has to do for the result to take effect. */ + nextStep?: string; +} + +/** + * One platform's way of turning usage into metadata. Everything host-specific + * lives behind this interface or in the options, so a new embedder of + * NativeScript needs an install target rather than a new pipeline. + */ +export interface IMetadataToolchain { + readonly platforms: string[]; + analyze(options: IMetadataTrimOptions): Promise; + generate( + usage: IMetadataUsage, + options: IMetadataTrimOptions, + ): Promise>; + install( + usage: IMetadataUsage, + metadataPath: string | undefined, + options: IMetadataTrimOptions, + ): Promise< + Pick + >; +} + +export interface IMetadataTrimService { + trim(options: IMetadataTrimOptions): Promise; +} diff --git a/lib/services/metadata-filtering-service.ts b/lib/services/metadata-filtering-service.ts index 3d46c5fe0e..7e6002b8c0 100644 --- a/lib/services/metadata-filtering-service.ts +++ b/lib/services/metadata-filtering-service.ts @@ -4,6 +4,8 @@ import { MetadataFilteringConstants } from "../constants"; import { IPluginsService } from "../definitions/plugins"; import { IPlatformsDataService } from "../definitions/platform"; import { IProjectData } from "../definitions/project"; +import { IDeclaredMetadataRules } from "../definitions/metadata-trim-service"; +import { mergeDeclaredRules } from "./metadata-trim/declared-rules"; import { IMetadataFilteringService, INativeApiUsageConfiguration, @@ -18,43 +20,66 @@ export class MetadataFilteringService implements IMetadataFilteringService { private $pluginsService: IPluginsService, private $mobileHelper: Mobile.IMobileHelper, private $platformsDataService: IPlatformsDataService, - private $logger: ILogger + private $logger: ILogger, ) {} public generateMetadataFilters( projectData: IProjectData, - platform: string + platform: string, ): void { this.generateWhitelist(projectData, platform); this.generateBlacklist(projectData, platform); } + public getDeclaredRules( + projectData: IProjectData, + platform: string, + ): IDeclaredMetadataRules { + const pluginConfigurations = this.$pluginsService + .getAllProductionPlugins(projectData, platform) + .map((plugin) => + path.join( + plugin.pluginPlatformsFolderPath(platform), + MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME, + ), + ) + .filter((configPath) => this.$fs.exists(configPath)) + .map( + (configPath): INativeApiUsagePluginConfiguration => + this.$fs.readJson(configPath) || {}, + ); + + return mergeDeclaredRules( + this.getNativeApiConfigurationForPlatform(projectData, platform), + pluginConfigurations, + ); + } + private generateWhitelist(projectData: IProjectData, platform: string): void { const platformsDirPath = this.getPlatformsDirPath(projectData, platform); const pathToWhitelistFile = path.join( platformsDirPath, - MetadataFilteringConstants.WHITELIST_FILE_NAME + MetadataFilteringConstants.WHITELIST_FILE_NAME, ); this.$fs.deleteFile(pathToWhitelistFile); const nativeApiConfiguration = this.getNativeApiConfigurationForPlatform( projectData, - platform + platform, ); if (nativeApiConfiguration) { const whitelistedItems: string[] = []; if (nativeApiConfiguration["whitelist-plugins-usages"]) { const plugins = this.$pluginsService.getAllProductionPlugins( projectData, - platform + platform, ); for (const pluginData of plugins) { - const pathToPlatformsDir = pluginData.pluginPlatformsFolderPath( - platform - ); + const pathToPlatformsDir = + pluginData.pluginPlatformsFolderPath(platform); const pathToPluginsMetadataConfig = path.join( pathToPlatformsDir, - MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME + MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME, ); if (this.$fs.exists(pathToPluginsMetadataConfig)) { const pluginConfig: INativeApiUsagePluginConfiguration = @@ -63,17 +88,17 @@ export class MetadataFilteringService implements IMetadataFilteringService { `Adding content of ${pathToPluginsMetadataConfig} to whitelisted items of metadata filtering: ${JSON.stringify( pluginConfig, null, - 2 - )}` + 2, + )}`, ); const itemsToAdd = pluginConfig.uses || []; if (itemsToAdd.length) { whitelistedItems.push( - `// Added from: ${pathToPluginsMetadataConfig}` + `// Added from: ${pathToPluginsMetadataConfig}`, ); whitelistedItems.push(...itemsToAdd); whitelistedItems.push( - `// Finished part from ${pathToPluginsMetadataConfig}${os.EOL}` + `// Finished part from ${pathToPluginsMetadataConfig}${os.EOL}`, ); } } @@ -87,8 +112,8 @@ export class MetadataFilteringService implements IMetadataFilteringService { `Adding content from application to whitelisted items of metadata filtering: ${JSON.stringify( applicationWhitelistedItems, null, - 2 - )}` + 2, + )}`, ); whitelistedItems.push(`// Added from application`); @@ -106,28 +131,27 @@ export class MetadataFilteringService implements IMetadataFilteringService { const platformsDirPath = this.getPlatformsDirPath(projectData, platform); const pathToBlacklistFile = path.join( platformsDirPath, - MetadataFilteringConstants.BLACKLIST_FILE_NAME + MetadataFilteringConstants.BLACKLIST_FILE_NAME, ); this.$fs.deleteFile(pathToBlacklistFile); const nativeApiConfiguration = this.getNativeApiConfigurationForPlatform( projectData, - platform + platform, ); if (nativeApiConfiguration) { const blacklistedItems: string[] = nativeApiConfiguration.blacklist || []; if (nativeApiConfiguration["whitelist-plugins-usages"]) { const plugins = this.$pluginsService.getAllProductionPlugins( projectData, - platform + platform, ); for (const pluginData of plugins) { - const pathToPlatformsDir = pluginData.pluginPlatformsFolderPath( - platform - ); + const pathToPlatformsDir = + pluginData.pluginPlatformsFolderPath(platform); const pathToPluginsMetadataConfig = path.join( pathToPlatformsDir, - MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME + MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME, ); if (this.$fs.exists(pathToPluginsMetadataConfig)) { const pluginConfig: INativeApiUsagePluginConfiguration = @@ -136,17 +160,17 @@ export class MetadataFilteringService implements IMetadataFilteringService { `Adding content of ${pathToPluginsMetadataConfig} to whitelisted items of metadata filtering: ${JSON.stringify( pluginConfig, null, - 2 - )}` + 2, + )}`, ); const itemsToAdd = pluginConfig.blacklist || []; if (itemsToAdd.length) { blacklistedItems.push( - `// Added from: ${pathToPluginsMetadataConfig}` + `// Added from: ${pathToPluginsMetadataConfig}`, ); blacklistedItems.push(...itemsToAdd); blacklistedItems.push( - `// Finished part from ${pathToPluginsMetadataConfig}${os.EOL}` + `// Finished part from ${pathToPluginsMetadataConfig}${os.EOL}`, ); } } @@ -157,20 +181,18 @@ export class MetadataFilteringService implements IMetadataFilteringService { } } else { this.$logger.trace( - `There's no application configuration for metadata filtering for platform ${platform}. Full metadata will be generated.` + `There's no application configuration for metadata filtering for platform ${platform}. Full metadata will be generated.`, ); } } private getNativeApiConfigurationForPlatform( projectData: IProjectData, - platform: string + platform: string, ): INativeApiUsageConfiguration { let config: INativeApiUsageConfiguration = null; - const pathToApplicationConfigurationFile = this.getPathToApplicationConfigurationForPlatform( - projectData, - platform - ); + const pathToApplicationConfigurationFile = + this.getPathToApplicationConfigurationForPlatform(projectData, platform); if (this.$fs.exists(pathToApplicationConfigurationFile)) { config = this.$fs.readJson(pathToApplicationConfigurationFile); } @@ -180,23 +202,23 @@ export class MetadataFilteringService implements IMetadataFilteringService { private getPlatformsDirPath( projectData: IProjectData, - platform: string + platform: string, ): string { const platformData = this.$platformsDataService.getPlatformData( platform, - projectData + projectData, ); return platformData.projectRoot; } private getPathToApplicationConfigurationForPlatform( projectData: IProjectData, - platform: string + platform: string, ): string { return path.join( projectData.appResourcesDirectoryPath, this.$mobileHelper.normalizePlatformName(platform), - MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME + MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME, ); } } diff --git a/lib/services/metadata-trim-service.ts b/lib/services/metadata-trim-service.ts new file mode 100644 index 0000000000..cce9d515da --- /dev/null +++ b/lib/services/metadata-trim-service.ts @@ -0,0 +1,67 @@ +import * as fs from "fs"; +import * as path from "path"; +import { IChildProcess, IErrors } from "../common/declarations"; +import { injector } from "../common/yok"; +import { + IMetadataToolchain, + IMetadataTrimOptions, + IMetadataTrimResult, + IMetadataTrimService, +} from "../definitions/metadata-trim-service"; +import { AndroidMetadataToolchain } from "./metadata-trim/android-toolchain"; +import { AppleMetadataToolchain } from "./metadata-trim/apple-toolchain"; + +export class MetadataTrimService implements IMetadataTrimService { + private toolchains: IMetadataToolchain[]; + + constructor( + $childProcess: IChildProcess, + private $errors: IErrors, + private $logger: ILogger, + ) { + this.toolchains = [ + new AppleMetadataToolchain($childProcess), + new AndroidMetadataToolchain($childProcess), + ]; + } + + public async trim( + options: IMetadataTrimOptions, + ): Promise { + const toolchain = this.toolchains.find((candidate) => + candidate.platforms.includes(options.platform), + ); + if (!toolchain) { + this.$errors.fail( + `Metadata trimming is not available for ${options.platform}.`, + ); + } + for (const bundle of options.bundles) { + if (!fs.existsSync(bundle)) { + this.$errors.fail(`Cannot find the bundle ${bundle}.`); + } + } + fs.mkdirSync(path.resolve(options.outDir), { recursive: true }); + + try { + const usage = await toolchain.analyze(options); + if (usage.unrestricted) { + this.$errors.fail( + "Could not determine which native APIs this app uses, usually because the bundle reads globals dynamically (for example an alias of globalThis, or eval). Nothing was trimmed.", + ); + } + this.$logger.info(`Found ${usage.patternCount} native API patterns.`); + const generated = await toolchain.generate(usage, options); + const installed = await toolchain.install( + usage, + generated.metadataPath, + options, + ); + return { usage, ...generated, ...installed }; + } catch (err) { + this.$errors.fail((err as Error).message); + } + } +} + +injector.register("metadataTrimService", MetadataTrimService); diff --git a/lib/services/metadata-trim/android-toolchain.ts b/lib/services/metadata-trim/android-toolchain.ts new file mode 100644 index 0000000000..87521198a6 --- /dev/null +++ b/lib/services/metadata-trim/android-toolchain.ts @@ -0,0 +1,100 @@ +import * as fs from "fs"; +import * as path from "path"; +import { IChildProcess } from "../../common/declarations"; +import { + IMetadataToolchain, + IMetadataTrimOptions, + IMetadataTrimResult, + IMetadataUsage, +} from "../../definitions/metadata-trim-service"; +import { readFilterPatterns } from "./apple-toolchain"; + +const SEED_SCRIPT = path.join("build-tools", "metadata-filter", "seed.js"); +const WHITELIST_NAME = "whitelist.mdg"; +const BLACKLIST_NAME = "blacklist.mdg"; + +/** Android's metadata generator runs in Gradle; this produces its whitelist. */ +export class AndroidMetadataToolchain implements IMetadataToolchain { + public readonly platforms = ["android"]; + + constructor(private $childProcess: IChildProcess) {} + + public async analyze(options: IMetadataTrimOptions): Promise { + if (!options.androidProject) { + throw new Error( + "Pass the Android project that owns whitelist.mdg with --android-project.", + ); + } + const project = options.androidProject; + const whitelistPath = path.join(options.outDir, "whitelist.android.mdg"); + const args = [ + options.analyzerPath ?? path.join(project, SEED_SCRIPT), + ...options.bundles, + "--out", + whitelistPath, + "--manifest", + path.join(project, "app", "src", "main", "AndroidManifest.xml"), + ]; + const resources = path.join(project, "app", "src", "main", "res"); + if (fs.existsSync(resources)) { + args.push("--res", resources); + } + await this.$childProcess.spawnFromEvent(process.execPath, args, "close", { + stdio: "pipe", + // The harvester's parser is installed with the project's build tools. + env: { + ...process.env, + NODE_PATH: path.join( + project, + "build-tools", + "jsparser", + "node_modules", + ), + }, + }); + + // The seed step refuses to write a partial analysis. + if (!fs.existsSync(whitelistPath)) { + return { whitelistPath, patternCount: 0, unrestricted: true }; + } + if (options.keep.length) { + fs.appendFileSync(whitelistPath, options.keep.join("\n") + "\n"); + } + const patterns = readFilterPatterns(whitelistPath); + return { + whitelistPath, + patternCount: patterns.length, + unrestricted: false, + }; + } + + public async generate(): Promise< + Pick + > { + return {}; + } + + public async install( + usage: IMetadataUsage, + _metadataPath: string | undefined, + options: IMetadataTrimOptions, + ): Promise< + Pick + > { + const installedTo = path.join(options.androidProject, WHITELIST_NAME); + fs.copyFileSync(usage.whitelistPath, installedTo); + if (options.blacklist?.length) { + fs.writeFileSync( + path.join(options.androidProject, BLACKLIST_NAME), + options.blacklist.join("\n") + "\n", + ); + } + return { + installedTo, + // Without this flag Gradle filters by the list as written, dropping + // the supertypes and signature types the listed classes depend on. + nextStep: + "Build with -PnsFilterMetadata so the generator also keeps the types these classes depend on.", + }; + } +} diff --git a/lib/services/metadata-trim/apple-toolchain.ts b/lib/services/metadata-trim/apple-toolchain.ts new file mode 100644 index 0000000000..4d286a1083 --- /dev/null +++ b/lib/services/metadata-trim/apple-toolchain.ts @@ -0,0 +1,186 @@ +import * as fs from "fs"; +import * as path from "path"; +import { IChildProcess } from "../../common/declarations"; +import { + IMetadataToolchain, + IMetadataTrimOptions, + IMetadataTrimResult, + IMetadataUsage, +} from "../../definitions/metadata-trim-service"; +import { + embeddedMetadataVersion, + metadataFileVersion, + replaceEmbeddedMetadata, +} from "./macho-metadata"; + +const ANALYZER_NAME = "ns-metadata-symbols"; +const GENERATOR_NAME = "objc-metadata-generator"; + +export const UNRESTRICTED_PATTERN = "*:*"; + +export function readFilterPatterns(filePath: string): string[] { + return fs + .readFileSync(filePath, "utf8") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#") && !line.startsWith("//")); +} + +function frameworkBinary(framework: string): string { + const versioned = path.join(framework, "Versions", "A", "NativeScript"); + return fs.existsSync(versioned) + ? versioned + : path.join(framework, "NativeScript"); +} + +export class AppleMetadataToolchain implements IMetadataToolchain { + public readonly platforms = ["ios", "macos"]; + + constructor(private $childProcess: IChildProcess) {} + + public async analyze(options: IMetadataTrimOptions): Promise { + const whitelistPath = path.join( + options.outDir, + `whitelist.${options.platform}.mdg`, + ); + const args = ["--output", whitelistPath]; + if (options.keep.length) { + const keepPath = path.join(options.outDir, "keep.mdg"); + fs.writeFileSync(keepPath, options.keep.join("\n") + "\n"); + args.push("--include-whitelist", keepPath); + } + await this.run( + this.tool(options.analyzerPath, ANALYZER_NAME, "--analyzer"), + [...args, ...options.bundles], + ); + const patterns = readFilterPatterns(whitelistPath); + return { + whitelistPath, + patternCount: patterns.length, + unrestricted: patterns.includes(UNRESTRICTED_PATTERN), + }; + } + + public async generate( + usage: IMetadataUsage, + options: IMetadataTrimOptions, + ): Promise> { + const arch = options.arch ?? "arm64"; + const base = path.join( + options.outDir, + `metadata.${options.platform}.${arch}`, + ); + const metadataPath = `${base}.nsmd`; + const blacklistArgs: string[] = []; + if (options.blacklist?.length) { + const blacklistPath = path.join( + options.outDir, + `blacklist.${options.platform}.mdg`, + ); + fs.writeFileSync(blacklistPath, options.blacklist.join("\n") + "\n"); + blacklistArgs.push("-blacklist-modules", blacklistPath); + } + const sdk = options.sdk ?? this.defaultSdk(options.platform); + const sdkPath = ( + await this.$childProcess.exec(`xcrun --sdk ${sdk} --show-sdk-path`) + ).trim(); + await this.run( + this.tool(options.generatorPath, GENERATOR_NAME, "--generator"), + [ + "-output-bin", + metadataPath, + "-output-umbrella", + `${base}.h`, + "-whitelist-modules", + usage.whitelistPath, + ...blacklistArgs, + "Xclang", + "-isysroot", + sdkPath, + "-std=gnu99", + "-target", + this.targetTriple(options.platform, sdk, arch, options.minVersion), + ], + ); + return { metadataPath, metadataBytes: fs.statSync(metadataPath).size }; + } + + public async install( + _usage: IMetadataUsage, + metadataPath: string | undefined, + options: IMetadataTrimOptions, + ): Promise> { + if (!options.framework || !metadataPath) { + return {}; + } + const installedTo = path.join( + options.outDir, + path.basename(options.framework), + ); + fs.rmSync(installedTo, { recursive: true, force: true }); + fs.cpSync(options.framework, installedTo, { + recursive: true, + verbatimSymlinks: true, + }); + + const binaryPath = frameworkBinary(installedTo); + const binary = fs.readFileSync(binaryPath); + const metadata = fs.readFileSync(metadataPath); + + // The runtime exits at launch on a format it can't read, so a generator + // that doesn't match the framework has to be caught here. + const readable = embeddedMetadataVersion(binary); + const written = metadataFileVersion(metadata); + if (written === undefined) { + throw new Error(`${metadataPath} is not NativeScript metadata.`); + } + if (readable !== undefined && written > readable) { + throw new Error( + `The generator writes metadata format ${written}, but this framework reads format ${readable}. Use the generator from the same runtime release as the framework.`, + ); + } + + const replaced = replaceEmbeddedMetadata(binary, metadata); + fs.writeFileSync(binaryPath, replaced); + await this.run("codesign", [ + "--force", + "--sign", + "-", + "--timestamp=none", + binaryPath, + ]); + return { installedTo, replacedBytes: binary.length - replaced.length }; + } + + private tool(explicit: string | undefined, name: string, flag: string) { + if (!explicit) { + throw new Error( + `Pass the path to ${name} from your NativeScript runtime with ${flag}.`, + ); + } + return explicit; + } + + private defaultSdk(platform: string): string { + return platform === "macos" ? "macosx" : "iphonesimulator"; + } + + private targetTriple( + platform: string, + sdk: string, + arch: string, + minVersion?: string, + ): string { + if (platform === "macos") { + return `${arch}-apple-macos${minVersion ?? "14.0"}`; + } + const simulator = sdk.startsWith("iphonesimulator") ? "-simulator" : ""; + return `${arch}-apple-ios${minVersion ?? "15.0"}${simulator}`; + } + + private async run(command: string, args: string[]): Promise { + await this.$childProcess.spawnFromEvent(command, args, "close", { + stdio: "pipe", + }); + } +} diff --git a/lib/services/metadata-trim/declared-rules.ts b/lib/services/metadata-trim/declared-rules.ts new file mode 100644 index 0000000000..8ff0880721 --- /dev/null +++ b/lib/services/metadata-trim/declared-rules.ts @@ -0,0 +1,44 @@ +import { readFileSync } from "fs"; +import { + INativeApiUsageConfiguration, + INativeApiUsagePluginConfiguration, +} from "../../definitions/metadata-filtering-service"; +import { IDeclaredMetadataRules } from "../../definitions/metadata-trim-service"; + +/** + * Follows the rules `ns prepare` applies to native-api-usage.json, with one + * difference: an app with no file of its own still keeps what its plugins + * declare, because declarations here only add to what the analysis found. + */ +export function mergeDeclaredRules( + app: INativeApiUsageConfiguration | null | undefined, + plugins: INativeApiUsagePluginConfiguration[], +): IDeclaredMetadataRules { + const usesPlugins = !app || app["whitelist-plugins-usages"]; + const pluginUses = usesPlugins + ? plugins.flatMap((plugin) => plugin.uses ?? []) + : []; + const pluginBlacklist = + app && usesPlugins + ? plugins.flatMap((plugin) => plugin.blacklist ?? []) + : []; + + return { + keep: [...(app?.whitelist ?? []), ...pluginUses], + blacklist: [...(app?.blacklist ?? []), ...pluginBlacklist], + }; +} + +/** Reads a native-api-usage.json in either the app or the plugin format. */ +export function readDeclaredRules(filePath: string): IDeclaredMetadataRules { + let content: any; + try { + content = JSON.parse(readFileSync(filePath, "utf8")); + } catch (err) { + throw new Error(`Cannot read ${filePath}: ${(err as Error).message}`); + } + return { + keep: [...(content.whitelist ?? []), ...(content.uses ?? [])], + blacklist: content.blacklist ?? [], + }; +} diff --git a/lib/services/metadata-trim/macho-metadata.ts b/lib/services/metadata-trim/macho-metadata.ts new file mode 100644 index 0000000000..76b41ff2eb --- /dev/null +++ b/lib/services/metadata-trim/macho-metadata.ts @@ -0,0 +1,168 @@ +const MH_MAGIC_64 = 0xfeedfacf; +const LC_SEGMENT_64 = 0x19; +const LC_SYMTAB = 0x2; +const LC_DYSYMTAB = 0xb; +const LC_DYLD_INFO = 0x22; +const LC_DYLD_INFO_ONLY = 0x80000022; +const LINKEDIT_DATA_COMMANDS = new Set([ + 0x1d, // LC_CODE_SIGNATURE + 0x1e, // LC_SEGMENT_SPLIT_INFO + 0x26, // LC_FUNCTION_STARTS + 0x29, // LC_DATA_IN_CODE + 0x2b, // LC_DYLIB_CODE_SIGN_DRS + 0x2e, // LC_LINKER_OPTIMIZATION_HINT + 0x36, // LC_ATOM_INFO + 0x80000033, // LC_DYLD_EXPORTS_TRIE + 0x80000034, // LC_DYLD_CHAINED_FIXUPS +]); +const OFFSET_FIELDS: Record = { + [LC_SYMTAB]: [8, 16], + [LC_DYSYMTAB]: [32, 40, 48, 56, 64, 72], + [LC_DYLD_INFO]: [8, 16, 24, 32, 40], + [LC_DYLD_INFO_ONLY]: [8, 16, 24, 32, 40], +}; + +const METADATA_SEGMENT = "__objc_metadata"; +const LINKEDIT_SEGMENT = "__LINKEDIT"; +const NSMD_MAGIC = "NSMD"; +const PAGE_SIZE = 0x4000; +const SECTION_64_SIZE = 80; + +interface Segment { + commandOffset: number; + vmAddress: bigint; + vmSize: bigint; + fileOffset: number; + fileSize: number; + sectionCount: number; +} + +function segmentName(binary: Buffer, offset: number): string { + const raw = binary.subarray(offset, offset + 16); + const end = raw.indexOf(0); + return raw.toString("latin1", 0, end === -1 ? 16 : end); +} + +function forEachCommand( + binary: Buffer, + visit: (type: number, offset: number) => void, +): void { + const commandCount = binary.readUInt32LE(16); + let offset = 32; + for (let i = 0; i < commandCount; i++) { + visit(binary.readUInt32LE(offset), offset); + offset += binary.readUInt32LE(offset + 4); + } +} + +function findSegments(binary: Buffer): Record { + if (binary.length < 32 || binary.readUInt32LE(0) !== MH_MAGIC_64) { + throw new Error( + "Expected a thin 64-bit Mach-O binary. Pass the single-architecture slice of the framework.", + ); + } + const segments: Record = {}; + forEachCommand(binary, (type, offset) => { + if (type === LC_SEGMENT_64) { + segments[segmentName(binary, offset + 8)] = { + commandOffset: offset, + vmAddress: binary.readBigUInt64LE(offset + 24), + vmSize: binary.readBigUInt64LE(offset + 32), + fileOffset: Number(binary.readBigUInt64LE(offset + 40)), + fileSize: Number(binary.readBigUInt64LE(offset + 48)), + sectionCount: binary.readUInt32LE(offset + 64), + }; + } + }); + return segments; +} + +function readFormatVersion(data: Buffer): number | undefined { + if (data.toString("latin1", 0, 4) !== NSMD_MAGIC) { + return undefined; + } + return data.readUInt16LE(4); +} + +/** The metadata format version a NativeScript binary embeds, if it has any. */ +export function embeddedMetadataVersion(binary: Buffer): number | undefined { + const segment = findSegments(binary)[METADATA_SEGMENT]; + return segment && readFormatVersion(binary.subarray(segment.fileOffset)); +} + +export function metadataFileVersion(metadata: Buffer): number | undefined { + return readFormatVersion(metadata); +} + +/** + * Returns `binary` with its embedded metadata replaced by `metadata` and the + * segment shrunk to fit, so the file itself gets smaller. The code signature + * no longer matches and the result has to be signed again. + */ +export function replaceEmbeddedMetadata( + binary: Buffer, + metadata: Buffer, +): Buffer { + const segments = findSegments(binary); + const target = segments[METADATA_SEGMENT]; + const linkedit = segments[LINKEDIT_SEGMENT]; + if (!target) { + throw new Error(`This binary has no ${METADATA_SEGMENT} segment.`); + } + // Only __LINKEDIT can move: it is addressed by file offsets alone, while + // every other segment is referenced by absolute addresses in the code. + if ( + !linkedit || + linkedit.fileOffset !== target.fileOffset + target.fileSize || + linkedit.vmAddress !== target.vmAddress + target.vmSize + ) { + throw new Error( + `${METADATA_SEGMENT} must be directly before ${LINKEDIT_SEGMENT} to be resized.`, + ); + } + + const newSize = Math.ceil(metadata.length / PAGE_SIZE) * PAGE_SIZE; + if (newSize > target.fileSize) { + throw new Error( + `The new metadata (${metadata.length} bytes) is larger than the embedded segment (${target.fileSize} bytes).`, + ); + } + const delta = target.fileSize - newSize; + + const head = Buffer.from(binary.subarray(0, target.fileOffset)); + + const moveOffset = (fieldOffset: number) => { + const value = head.readUInt32LE(fieldOffset); + if (value >= linkedit.fileOffset) { + head.writeUInt32LE(value - delta, fieldOffset); + } + }; + forEachCommand(head, (type, offset) => { + for (const field of OFFSET_FIELDS[type] ?? []) { + moveOffset(offset + field); + } + if (LINKEDIT_DATA_COMMANDS.has(type)) { + moveOffset(offset + 8); + } + }); + + const shrunk = BigInt(delta); + head.writeBigUInt64LE(target.vmSize - shrunk, target.commandOffset + 32); + head.writeBigUInt64LE(BigInt(newSize), target.commandOffset + 48); + for (let i = 0; i < target.sectionCount; i++) { + const section = target.commandOffset + 72 + i * SECTION_64_SIZE; + head.writeBigUInt64LE(BigInt(metadata.length), section + 40); + } + head.writeBigUInt64LE( + linkedit.vmAddress - shrunk, + linkedit.commandOffset + 24, + ); + head.writeBigUInt64LE( + BigInt(linkedit.fileOffset - delta), + linkedit.commandOffset + 40, + ); + + const payload = Buffer.alloc(newSize); + metadata.copy(payload); + return Buffer.concat([head, payload, binary.subarray(linkedit.fileOffset)]); +} diff --git a/test/services/metadata-filtering-service.ts b/test/services/metadata-filtering-service.ts index f092e24db0..a464d79928 100644 --- a/test/services/metadata-filtering-service.ts +++ b/test/services/metadata-filtering-service.ts @@ -25,12 +25,12 @@ describe("metadataFilteringService", () => { const appResourcesNativeApiUsageFilePath = path.join( projectData.appResourcesDirectoryPath, platform, - MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME + MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME, ); const pluginPlatformsDir = path.join("pluginDir", platform); const pluginNativeApiUsageFilePath = path.join( pluginPlatformsDir, - MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME + MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME, ); const pluginsUses: string[] = ["pluginUses1", "pluginUses2"]; @@ -41,14 +41,14 @@ describe("metadataFilteringService", () => { testInjector.register("pluginsService", { getAllProductionPlugins: ( prjData: IProjectData, - dependencies?: IDependencyData[] + dependencies?: IDependencyData[], ): IPluginData[] => { const plugins = !!(input && input.hasPlugins) ? [ { pluginPlatformsFolderPath: (pl: string) => pluginPlatformsDir, }, - ] + ] : []; return plugins; @@ -64,6 +64,46 @@ describe("metadataFilteringService", () => { return testInjector; }; + describe("getDeclaredRules", () => { + const service = (input: { hasPlugins: boolean }, files: any) => { + const testInjector = createTestInjector(input); + const fileSystem = testInjector.resolve("fs"); + fileSystem.exists = (filePath: string) => filePath in files; + fileSystem.readJson = (filePath: string) => files[filePath]; + return testInjector.resolve( + MetadataFilteringService, + ); + }; + + it("merges the app's rules with its plugins' declarations", () => { + const rules = service( + { hasPlugins: true }, + { + [appResourcesNativeApiUsageFilePath]: { + "whitelist-plugins-usages": true, + whitelist: whitelistArray, + blacklist: blacklistArray, + }, + [pluginNativeApiUsageFilePath]: { uses: pluginsUses }, + }, + ).getDeclaredRules(projectData, platform); + + assert.deepEqual(rules, { + keep: [...whitelistArray, ...pluginsUses], + blacklist: blacklistArray, + }); + }); + + it("returns nothing when no file exists", () => { + const rules = service({ hasPlugins: true }, {}).getDeclaredRules( + projectData, + platform, + ); + + assert.deepEqual(rules, { keep: [], blacklist: [] }); + }); + }); + describe("generateMetadataFilters", () => { const mockFs = (input: { testInjector: IInjector; @@ -96,14 +136,13 @@ describe("metadataFilteringService", () => { it("deletes previously generated files for metadata filtering", () => { const testInjector = createTestInjector(); - const metadataFilteringService: IMetadataFilteringService = testInjector.resolve( - MetadataFilteringService - ); + const metadataFilteringService: IMetadataFilteringService = + testInjector.resolve(MetadataFilteringService); const { fs } = mockFs({ testInjector, writeFileAction: (filePath: string, data: string) => { throw new Error( - `No data should be written when the ${MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME} does not exist in App_Resource/` + `No data should be written when the ${MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME} does not exist in App_Resource/`, ); }, }); @@ -119,9 +158,8 @@ describe("metadataFilteringService", () => { it(`generates ${MetadataFilteringConstants.BLACKLIST_FILE_NAME} when the file ${MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME} exists in App_Resources/`, () => { const testInjector = createTestInjector(); - const metadataFilteringService: IMetadataFilteringService = testInjector.resolve( - MetadataFilteringService - ); + const metadataFilteringService: IMetadataFilteringService = + testInjector.resolve(MetadataFilteringService); const { dataWritten } = mockFs({ testInjector, existingFiles: [appResourcesNativeApiUsageFilePath], @@ -137,7 +175,7 @@ describe("metadataFilteringService", () => { assert.deepStrictEqual(dataWritten, { [path.join( projectRoot, - MetadataFilteringConstants.BLACKLIST_FILE_NAME + MetadataFilteringConstants.BLACKLIST_FILE_NAME, )]: blacklistArray.join(EOL), }); }); @@ -149,7 +187,7 @@ describe("metadataFilteringService", () => { let finalContent = ""; if (input.pluginWhitelist) { finalContent += `// Added from: ${pluginNativeApiUsageFilePath}${EOL}${input.pluginWhitelist.join( - EOL + EOL, )}${EOL}// Finished part from ${pluginNativeApiUsageFilePath}${EOL}`; } @@ -159,7 +197,7 @@ describe("metadataFilteringService", () => { } finalContent += `// Added from application${EOL}${input.applicationWhitelist.join( - EOL + EOL, )}${EOL}// Finished part from application${EOL}`; } @@ -168,9 +206,8 @@ describe("metadataFilteringService", () => { it(`generates ${MetadataFilteringConstants.WHITELIST_FILE_NAME} when the file ${MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME} exists in App_Resources/`, () => { const testInjector = createTestInjector(); - const metadataFilteringService: IMetadataFilteringService = testInjector.resolve( - MetadataFilteringService - ); + const metadataFilteringService: IMetadataFilteringService = + testInjector.resolve(MetadataFilteringService); const { dataWritten } = mockFs({ testInjector, existingFiles: [appResourcesNativeApiUsageFilePath], @@ -185,7 +222,7 @@ describe("metadataFilteringService", () => { assert.deepStrictEqual(dataWritten, { [path.join( projectRoot, - MetadataFilteringConstants.WHITELIST_FILE_NAME + MetadataFilteringConstants.WHITELIST_FILE_NAME, )]: getExpectedWhitelistContent({ applicationWhitelist: whitelistArray, }), @@ -194,9 +231,8 @@ describe("metadataFilteringService", () => { it(`generates ${MetadataFilteringConstants.WHITELIST_FILE_NAME} with content from plugins when the file ${MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME} exists in App_Resources/ and whitelist-plugins-usages is true`, () => { const testInjector = createTestInjector({ hasPlugins: true }); - const metadataFilteringService: IMetadataFilteringService = testInjector.resolve( - MetadataFilteringService - ); + const metadataFilteringService: IMetadataFilteringService = + testInjector.resolve(MetadataFilteringService); const { dataWritten } = mockFs({ testInjector, existingFiles: [ @@ -215,16 +251,15 @@ describe("metadataFilteringService", () => { assert.deepStrictEqual(dataWritten, { [path.join( projectRoot, - MetadataFilteringConstants.WHITELIST_FILE_NAME + MetadataFilteringConstants.WHITELIST_FILE_NAME, )]: getExpectedWhitelistContent({ pluginWhitelist: whitelistArray }), }); }); it(`generates all files when both plugins and applications filters are included`, () => { const testInjector = createTestInjector({ hasPlugins: true }); - const metadataFilteringService: IMetadataFilteringService = testInjector.resolve( - MetadataFilteringService - ); + const metadataFilteringService: IMetadataFilteringService = + testInjector.resolve(MetadataFilteringService); const { dataWritten } = mockFs({ testInjector, existingFiles: [ @@ -250,20 +285,19 @@ describe("metadataFilteringService", () => { assert.deepStrictEqual(dataWritten, { [path.join( projectRoot, - MetadataFilteringConstants.WHITELIST_FILE_NAME + MetadataFilteringConstants.WHITELIST_FILE_NAME, )]: expectedWhitelist, [path.join( projectRoot, - MetadataFilteringConstants.BLACKLIST_FILE_NAME + MetadataFilteringConstants.BLACKLIST_FILE_NAME, )]: blacklistArray.join(EOL), }); }); it(`skips plugins ${MetadataFilteringConstants.NATIVE_API_USAGE_FILE_NAME} files when whitelist-plugins-usages in App_Resources is false`, () => { const testInjector = createTestInjector({ hasPlugins: true }); - const metadataFilteringService: IMetadataFilteringService = testInjector.resolve( - MetadataFilteringService - ); + const metadataFilteringService: IMetadataFilteringService = + testInjector.resolve(MetadataFilteringService); const { dataWritten } = mockFs({ testInjector, existingFiles: [ @@ -288,11 +322,11 @@ describe("metadataFilteringService", () => { assert.deepStrictEqual(dataWritten, { [path.join( projectRoot, - MetadataFilteringConstants.WHITELIST_FILE_NAME + MetadataFilteringConstants.WHITELIST_FILE_NAME, )]: expectedWhitelist, [path.join( projectRoot, - MetadataFilteringConstants.BLACKLIST_FILE_NAME + MetadataFilteringConstants.BLACKLIST_FILE_NAME, )]: blacklistArray.join(EOL), }); }); diff --git a/test/services/metadata-trim-service.ts b/test/services/metadata-trim-service.ts new file mode 100644 index 0000000000..36ff8b865c --- /dev/null +++ b/test/services/metadata-trim-service.ts @@ -0,0 +1,473 @@ +import { assert } from "chai"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { MetadataTrimService } from "../../lib/services/metadata-trim-service"; +import { + IMetadataToolchain, + IMetadataTrimOptions, + IMetadataUsage, +} from "../../lib/definitions/metadata-trim-service"; +import { AndroidMetadataToolchain } from "../../lib/services/metadata-trim/android-toolchain"; +import { AppleMetadataToolchain } from "../../lib/services/metadata-trim/apple-toolchain"; +import { + mergeDeclaredRules, + readDeclaredRules, +} from "../../lib/services/metadata-trim/declared-rules"; + +describe("metadataTrimService", () => { + let directory: string; + let bundle: string; + let calls: string[]; + + const errors = { + fail(message: string) { + throw new Error(message); + }, + }; + const logger = { info() {} }; + + function toolchain(usage: Partial): IMetadataToolchain { + return { + platforms: ["ios", "macos", "android"], + async analyze() { + calls.push("analyze"); + return { + whitelistPath: "whitelist.mdg", + patternCount: 3, + unrestricted: false, + ...usage, + }; + }, + async generate() { + calls.push("generate"); + return { metadataPath: "out.nsmd", metadataBytes: 12 }; + }, + async install(_usage, metadataPath) { + calls.push(`install:${metadataPath}`); + return { installedTo: "target", replacedBytes: 34 }; + }, + }; + } + + function service(usage: Partial = {}) { + const created = new MetadataTrimService( + {} as any, + errors as any, + logger as any, + ); + (created as any).toolchains = [toolchain(usage)]; + return created; + } + + function options( + overrides: Partial = {}, + ): IMetadataTrimOptions { + return { + platform: "macos", + bundles: [bundle], + outDir: path.join(directory, "out"), + keep: [], + ...overrides, + }; + } + + beforeEach(() => { + calls = []; + directory = fs.mkdtempSync(path.join(os.tmpdir(), "metadata-trim-")); + bundle = path.join(directory, "bundle.js"); + fs.writeFileSync(bundle, ""); + }); + + afterEach(() => { + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("runs analysis, generation and install in order", async () => { + const result = await service().trim(options()); + + assert.deepEqual(calls, ["analyze", "generate", "install:out.nsmd"]); + assert.equal(result.metadataBytes, 12); + assert.equal(result.installedTo, "target"); + assert.equal(result.replacedBytes, 34); + }); + + it("creates the output directory", async () => { + await service().trim(options()); + + assert.isTrue(fs.existsSync(path.join(directory, "out"))); + }); + + it("stops without generating when usage cannot be bounded", async () => { + let failure: Error; + try { + await service({ unrestricted: true }).trim(options()); + } catch (err) { + failure = err as Error; + } + + assert.match(failure.message, /Nothing was trimmed/); + assert.deepEqual(calls, ["analyze"]); + }); + + it("fails on a bundle that does not exist", async () => { + let failure: Error; + try { + await service().trim(options({ bundles: ["/missing/bundle.js"] })); + } catch (err) { + failure = err as Error; + } + + assert.match(failure.message, /Cannot find the bundle/); + assert.deepEqual(calls, []); + }); + + it("fails on a platform no toolchain supports", async () => { + let failure: Error; + try { + await service().trim(options({ platform: "windows" })); + } catch (err) { + failure = err as Error; + } + + assert.match(failure.message, /not available for windows/); + }); +}); + +describe("declared metadata rules", () => { + const plugin = { uses: ["java.util:List"], blacklist: ["java.util:Locale"] }; + + it("adds the app's whitelist and blacklist", () => { + const rules = mergeDeclaredRules( + { + "whitelist-plugins-usages": false, + whitelist: ["a:B"], + blacklist: ["c:D"], + }, + [], + ); + + assert.deepEqual(rules, { keep: ["a:B"], blacklist: ["c:D"] }); + }); + + it("includes plugin rules when the app opts in", () => { + const rules = mergeDeclaredRules( + { "whitelist-plugins-usages": true, whitelist: [], blacklist: [] }, + [plugin], + ); + + assert.deepEqual(rules, { + keep: ["java.util:List"], + blacklist: ["java.util:Locale"], + }); + }); + + it("ignores plugin rules when the app opts out", () => { + const rules = mergeDeclaredRules( + { "whitelist-plugins-usages": false, whitelist: [], blacklist: [] }, + [plugin], + ); + + assert.deepEqual(rules, { keep: [], blacklist: [] }); + }); + + it("keeps plugin uses but not their blacklist when the app has no file", () => { + assert.deepEqual(mergeDeclaredRules(null, [plugin]), { + keep: ["java.util:List"], + blacklist: [], + }); + }); + + describe("readDeclaredRules", () => { + let file: string; + + beforeEach(() => { + file = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "metadata-rules-")), + "native-api-usage.json", + ); + }); + + afterEach(() => { + fs.rmSync(path.dirname(file), { recursive: true, force: true }); + }); + + it("reads the app format", () => { + fs.writeFileSync( + file, + JSON.stringify({ whitelist: ["a:B"], blacklist: ["c:D"] }), + ); + + assert.deepEqual(readDeclaredRules(file), { + keep: ["a:B"], + blacklist: ["c:D"], + }); + }); + + it("reads the plugin format", () => { + fs.writeFileSync(file, JSON.stringify({ uses: ["a:B"] })); + + assert.deepEqual(readDeclaredRules(file), { + keep: ["a:B"], + blacklist: [], + }); + }); + + it("names the file when it can't be read", () => { + assert.throws(() => readDeclaredRules(`${file}.missing`), /Cannot read/); + }); + }); +}); + +describe("AndroidMetadataToolchain", () => { + let directory: string; + let project: string; + let spawnOptions: any; + + const childProcess = { + async spawnFromEvent( + _command: string, + args: string[], + _event: string, + options: any, + ) { + spawnOptions = options; + fs.writeFileSync( + args[args.indexOf("--out") + 1], + "# generated\nandroid.widget:SeekBar\n", + ); + }, + }; + + function options( + overrides: Partial = {}, + ): IMetadataTrimOptions { + return { + platform: "android", + bundles: ["/app/bundle.js"], + outDir: directory, + keep: [], + androidProject: project, + ...overrides, + }; + } + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "metadata-android-")); + project = path.join(directory, "project"); + fs.mkdirSync(project); + }); + + afterEach(() => { + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("resolves the harvester's parser from the project's build tools", async () => { + await new AndroidMetadataToolchain(childProcess as any).analyze(options()); + + assert.equal( + spawnOptions.env.NODE_PATH, + path.join(project, "build-tools", "jsparser", "node_modules"), + ); + }); + + it("appends kept patterns to the whitelist", async () => { + const usage = await new AndroidMetadataToolchain( + childProcess as any, + ).analyze(options({ keep: ["android.webkit:*"] })); + + assert.equal(usage.patternCount, 2); + assert.include( + fs.readFileSync(usage.whitelistPath, "utf8"), + "android.webkit:*", + ); + }); + + it("reports unbounded usage when the seed step wrote nothing", async () => { + const silent = { async spawnFromEvent() {} }; + + const usage = await new AndroidMetadataToolchain(silent as any).analyze( + options(), + ); + + assert.isTrue(usage.unrestricted); + }); + + it("requires an Android project", async () => { + let failure: Error; + try { + await new AndroidMetadataToolchain(childProcess as any).analyze( + options({ androidProject: undefined }), + ); + } catch (err) { + failure = err as Error; + } + + assert.match(failure.message, /--android-project/); + }); + + it("writes the blacklist beside the whitelist", async () => { + const toolchain = new AndroidMetadataToolchain(childProcess as any); + const trimOptions = options({ blacklist: ["java.util:Locale"] }); + const usage = await toolchain.analyze(trimOptions); + + await toolchain.install(usage, undefined, trimOptions); + + assert.equal( + fs.readFileSync(path.join(project, "blacklist.mdg"), "utf8"), + "java.util:Locale\n", + ); + }); + + it("installs the whitelist and asks for the closure build", async () => { + const toolchain = new AndroidMetadataToolchain(childProcess as any); + const usage = await toolchain.analyze(options()); + + const result = await toolchain.install(usage, undefined, options()); + + assert.equal(result.installedTo, path.join(project, "whitelist.mdg")); + assert.isTrue(fs.existsSync(result.installedTo)); + assert.match(result.nextStep, /-PnsFilterMetadata/); + }); +}); + +describe("AppleMetadataToolchain", () => { + let directory: string; + let spawned: { command: string; args: string[] }[]; + + const childProcess = { + async exec() { + return "/sdk/path\n"; + }, + async spawnFromEvent(command: string, args: string[]) { + spawned.push({ command, args }); + const output = args[args.indexOf("--output") + 1]; + if (args.includes("--output")) { + fs.writeFileSync(output, "# generated\n*:NSSlider\nFoundation:*\n"); + } + if (args.includes("-output-bin")) { + fs.writeFileSync(args[args.indexOf("-output-bin") + 1], "NSMD"); + } + }, + }; + + function options( + overrides: Partial = {}, + ): IMetadataTrimOptions { + return { + platform: "macos", + bundles: ["/app/bundle.js"], + outDir: directory, + keep: [], + analyzerPath: "/tools/analyzer", + generatorPath: "/tools/generator", + ...overrides, + }; + } + + beforeEach(() => { + spawned = []; + directory = fs.mkdtempSync(path.join(os.tmpdir(), "metadata-apple-")); + }); + + afterEach(() => { + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("counts the analyzer's patterns, ignoring comments", async () => { + const toolchain = new AppleMetadataToolchain(childProcess as any); + + const usage = await toolchain.analyze(options()); + + assert.equal(usage.patternCount, 2); + assert.isFalse(usage.unrestricted); + }); + + it("passes kept patterns to the analyzer", async () => { + const toolchain = new AppleMetadataToolchain(childProcess as any); + + await toolchain.analyze(options({ keep: ["UIKit:UIView", "MapKit:*"] })); + + const args = spawned[0].args; + const keepFile = args[args.indexOf("--include-whitelist") + 1]; + assert.equal(fs.readFileSync(keepFile, "utf8"), "UIKit:UIView\nMapKit:*\n"); + }); + + it("targets the iOS simulator by default", async () => { + const toolchain = new AppleMetadataToolchain(childProcess as any); + const usage = { + whitelistPath: "w.mdg", + patternCount: 1, + unrestricted: false, + }; + + await toolchain.generate(usage, options({ platform: "ios" })); + + const args = spawned[0].args; + assert.equal( + args[args.indexOf("-target") + 1], + "arm64-apple-ios15.0-simulator", + ); + }); + + it("passes blacklisted patterns to the generator", async () => { + const toolchain = new AppleMetadataToolchain(childProcess as any); + const usage = { + whitelistPath: "w.mdg", + patternCount: 1, + unrestricted: false, + }; + + await toolchain.generate( + usage, + options({ blacklist: ["UIKit:UIWebView"] }), + ); + + const args = spawned[0].args; + assert.equal( + fs.readFileSync(args[args.indexOf("-blacklist-modules") + 1], "utf8"), + "UIKit:UIWebView\n", + ); + }); + + it("names metadata by platform and architecture", async () => { + const toolchain = new AppleMetadataToolchain(childProcess as any); + const usage = { + whitelistPath: "w.mdg", + patternCount: 1, + unrestricted: false, + }; + + const result = await toolchain.generate(usage, options({ arch: "x86_64" })); + + assert.equal( + path.basename(result.metadataPath), + "metadata.macos.x86_64.nsmd", + ); + }); + + it("does nothing to install without a framework", async () => { + const toolchain = new AppleMetadataToolchain(childProcess as any); + + const result = await toolchain.install( + { whitelistPath: "w", patternCount: 1, unrestricted: false }, + "out.nsmd", + options(), + ); + + assert.deepEqual(result, {}); + }); + + it("requires a tool path", async () => { + const toolchain = new AppleMetadataToolchain(childProcess as any); + + let failure: Error; + try { + await toolchain.analyze(options({ analyzerPath: undefined })); + } catch (err) { + failure = err as Error; + } + + assert.match(failure.message, /--analyzer/); + }); +}); diff --git a/test/services/metadata-trim/macho-metadata.ts b/test/services/metadata-trim/macho-metadata.ts new file mode 100644 index 0000000000..e702e88a5e --- /dev/null +++ b/test/services/metadata-trim/macho-metadata.ts @@ -0,0 +1,192 @@ +import { assert } from "chai"; +import { + embeddedMetadataVersion, + metadataFileVersion, + replaceEmbeddedMetadata, +} from "../../../lib/services/metadata-trim/macho-metadata"; + +const PAGE = 0x4000; +const SEGMENT_COMMAND_SIZE = 72; +const SECTION_SIZE = 80; + +interface FakeBinaryOptions { + metadataPages?: number; + gapBeforeLinkedit?: number; +} + +function writeName(buffer: Buffer, offset: number, name: string) { + buffer.write(name, offset, "latin1"); +} + +function fakeBinary(options: FakeBinaryOptions = {}) { + const metadataPages = options.metadataPages ?? 4; + const gap = options.gapBeforeLinkedit ?? 0; + const metadataOffset = PAGE; + const metadataSize = metadataPages * PAGE; + const linkeditOffset = metadataOffset + metadataSize + gap; + const linkeditPayload = Buffer.from("LINKEDIT-CONTENT".repeat(8)); + + const binary = Buffer.alloc(linkeditOffset + linkeditPayload.length); + binary.writeUInt32LE(0xfeedfacf, 0); + binary.writeUInt32LE(4, 16); + + let at = 32; + binary.writeUInt32LE(0x19, at); + binary.writeUInt32LE(SEGMENT_COMMAND_SIZE + SECTION_SIZE, at + 4); + writeName(binary, at + 8, "__objc_metadata"); + binary.writeBigUInt64LE(BigInt(0x200000), at + 24); + binary.writeBigUInt64LE(BigInt(metadataSize), at + 32); + binary.writeBigUInt64LE(BigInt(metadataOffset), at + 40); + binary.writeBigUInt64LE(BigInt(metadataSize), at + 48); + binary.writeUInt32LE(1, at + 64); + writeName(binary, at + SEGMENT_COMMAND_SIZE, "__objc_metadata"); + binary.writeBigUInt64LE(BigInt(metadataSize), at + SEGMENT_COMMAND_SIZE + 40); + const metadataCommand = at; + at += SEGMENT_COMMAND_SIZE + SECTION_SIZE; + + binary.writeUInt32LE(0x19, at); + binary.writeUInt32LE(SEGMENT_COMMAND_SIZE, at + 4); + writeName(binary, at + 8, "__LINKEDIT"); + binary.writeBigUInt64LE(BigInt(0x200000 + metadataSize + gap), at + 24); + binary.writeBigUInt64LE(BigInt(PAGE), at + 32); + binary.writeBigUInt64LE(BigInt(linkeditOffset), at + 40); + binary.writeBigUInt64LE(BigInt(linkeditPayload.length), at + 48); + const linkeditCommand = at; + at += SEGMENT_COMMAND_SIZE; + + binary.writeUInt32LE(0x2, at); + binary.writeUInt32LE(24, at + 4); + binary.writeUInt32LE(linkeditOffset, at + 8); + binary.writeUInt32LE(linkeditOffset + 16, at + 16); + const symtabCommand = at; + at += 24; + + binary.writeUInt32LE(0x80000034, at); + binary.writeUInt32LE(16, at + 4); + binary.writeUInt32LE(linkeditOffset + 32, at + 8); + const fixupsCommand = at; + + binary.write("NSMD", metadataOffset, "latin1"); + binary.writeUInt16LE(1, metadataOffset + 4); + linkeditPayload.copy(binary, linkeditOffset); + + return { + binary, + linkeditOffset, + linkeditPayload, + metadataCommand, + linkeditCommand, + symtabCommand, + fixupsCommand, + metadataSize, + }; +} + +function metadataOf(length: number, version = 1): Buffer { + const metadata = Buffer.alloc(length, 7); + metadata.write("NSMD", 0, "latin1"); + metadata.writeUInt16LE(version, 4); + return metadata; +} + +describe("macho-metadata", () => { + it("shrinks the file and moves __LINKEDIT down by the saved pages", () => { + const fake = fakeBinary(); + const metadata = metadataOf(PAGE + 10); + + const result = replaceEmbeddedMetadata(fake.binary, metadata); + + const saved = fake.metadataSize - 2 * PAGE; + assert.equal(result.length, fake.binary.length - saved); + assert.equal( + Number(result.readBigUInt64LE(fake.linkeditCommand + 40)), + fake.linkeditOffset - saved, + ); + assert.equal( + Number(result.readBigUInt64LE(fake.linkeditCommand + 24)), + 0x200000 + 2 * PAGE, + ); + assert.deepEqual( + result.subarray(fake.linkeditOffset - saved), + fake.linkeditPayload, + ); + }); + + it("updates the segment and section sizes", () => { + const fake = fakeBinary(); + const result = replaceEmbeddedMetadata(fake.binary, metadataOf(100)); + + assert.equal( + Number(result.readBigUInt64LE(fake.metadataCommand + 48)), + PAGE, + ); + assert.equal( + Number(result.readBigUInt64LE(fake.metadataCommand + 32)), + PAGE, + ); + assert.equal( + Number( + result.readBigUInt64LE( + fake.metadataCommand + SEGMENT_COMMAND_SIZE + 40, + ), + ), + 100, + ); + }); + + it("rebases every offset that points into __LINKEDIT", () => { + const fake = fakeBinary(); + const result = replaceEmbeddedMetadata(fake.binary, metadataOf(100)); + const moved = fake.linkeditOffset - 3 * PAGE; + + assert.equal(result.readUInt32LE(fake.symtabCommand + 8), moved); + assert.equal(result.readUInt32LE(fake.symtabCommand + 16), moved + 16); + assert.equal(result.readUInt32LE(fake.fixupsCommand + 8), moved + 32); + }); + + it("writes the new metadata at the segment's offset", () => { + const metadata = metadataOf(100, 3); + const result = replaceEmbeddedMetadata(fakeBinary().binary, metadata); + + assert.deepEqual(result.subarray(PAGE, PAGE + 100), metadata); + assert.equal(embeddedMetadataVersion(result), 3); + }); + + it("refuses metadata larger than the segment", () => { + assert.throws( + () => + replaceEmbeddedMetadata( + fakeBinary({ metadataPages: 1 }).binary, + metadataOf(PAGE + 1), + ), + /larger than the embedded segment/, + ); + }); + + it("refuses a layout where __LINKEDIT does not follow the segment", () => { + assert.throws( + () => + replaceEmbeddedMetadata( + fakeBinary({ gapBeforeLinkedit: PAGE }).binary, + metadataOf(100), + ), + /directly before __LINKEDIT/, + ); + }); + + it("refuses fat and non-Mach-O files", () => { + assert.throws( + () => replaceEmbeddedMetadata(Buffer.alloc(64), metadataOf(100)), + /thin 64-bit Mach-O/, + ); + }); + + it("reads the embedded format version", () => { + assert.equal(embeddedMetadataVersion(fakeBinary().binary), 1); + }); + + it("reads a metadata file's format version and rejects other files", () => { + assert.equal(metadataFileVersion(metadataOf(10, 2)), 2); + assert.isUndefined(metadataFileVersion(Buffer.from("not metadata"))); + }); +});