diff --git a/docs/benchmarkdotnet.md b/docs/benchmarkdotnet.md index be429f57ad7..ed084e27876 100644 --- a/docs/benchmarkdotnet.md +++ b/docs/benchmarkdotnet.md @@ -27,7 +27,6 @@ BenchmarkDotNet is the benchmarking tool that allows to run benchmarks for .NET, - [Running In Process](#running-in-process) - [CoreRun](#corerun) - [dotnet cli](#dotnet-cli) - - [Private CLR Build](#private-clr-build) - [Private CoreRT Build](#private-corert-build) ## Main Concepts diff --git a/docs/benchmarking-workflow-dotnet-runtime.md b/docs/benchmarking-workflow-dotnet-runtime.md index 44d1ecd0d75..53ac9b1ceb1 100644 --- a/docs/benchmarking-workflow-dotnet-runtime.md +++ b/docs/benchmarking-workflow-dotnet-runtime.md @@ -11,6 +11,7 @@ - [Install v8 engine](#install-v8-engine) - [Run the benchmarks with the interpreter](#run-the-benchmarks-with-the-interpreter) - [Run the benchmarks with AOT](#run-the-benchmarks-with-aot) + - [Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK](#run-coreclr-wasm-readytorun-with-a-nightly-or-vmr-sdk) - [Note about "file ... being used by another process" error](#note-about-file--being-used-by-another-process-error) - [dotnet runtime testing for MonoAOT](#dotnet-runtime-testing-for-monoaot) - [Prerequisites (Files either built locally (with build.(sh/cmd) or downloaded from payload above (if same system setup) (in this order))](#prerequisites-files-either-built-locally-with-buildshcmd-or-downloaded-from-payload-above-if-same-system-setup-in-this-order) @@ -204,6 +205,35 @@ Essentially, add `--aotcompilermode wasm` to the `--bdn-arguments=".."`: --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --aotcompilermode wasm --logBuildOutput --buildTimeout 3600 --filter " ``` +#### Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK + +Use an exact SDK version together with a NuGet source containing the matching +browser runtime/ref, WebAssembly SDK, host-specific Crossgen2, and ILLink +packages, plus the complete `wasm-tools` pack closure. The source can be a +downloaded VMR package directory or a coherent feed. Workload manifest updates +are disabled and only exact versions recorded by the selected SDK are restored +from that source, so packages cannot float to another build cohort. + +```cmd +/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py \ + --csproj src/benchmarks/micro/MicroBenchmarks.csproj \ + -f net11.0 \ + --dotnet-versions \ + --wasm \ + --wasm-runtime-flavor CoreCLR \ + --wasm-ready-to-run \ + --wasm-workload-source /path/to/vmr/packages \ + --run-isolated \ + --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts \ + --bdn-arguments="--category-exclusion-filter NoWASM --logBuildOutput --buildTimeout 3600 --filter " +``` + +Omit `--dotnet-versions` to use the existing `main` daily-channel SDK +selection. In that case the package source must contain the product version +recorded in that SDK's `Microsoft.NETCoreSdk.BundledVersions.props`. +`PERFLAB_WASM_PACKAGE_VERSION` should remain unset; that override is reserved +for non-official runtime payloads. + #### Note about "file ... being used by another process" error If you are seeing warnings like: diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 0045cdb5a03..80a7f771cde 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -57,6 +57,7 @@ parameters: iOSStripSymbols: true # optional -- Whether to strip symbols from the iOS build machinePool: '' # optional -- Machine pool name (e.g. Pixel8, GalaxyA16, iPhone17) additionalSetupParameters: '' # optional -- Additional arguments to pass to the script + wasmWorkloadSource: '' # optional -- Exclusive coherent package source for SDK CoreCLR browser-WASM liveLibrariesBuildConfig: '' # optional -- Build configuration when generating Core_Root for libraries crossBuild: false # optional -- Whether the Core_Root is being cross-compiled @@ -241,6 +242,8 @@ jobs: - '--cross-build' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' + - ${{ if and(ne(parameters.wasmWorkloadSource, ''), eq(parameters.runtimeType, 'wasm_coreclr'), eq(parameters.r2rRunType, 'r2r')) }}: + - '--wasm-workload-source "${{ parameters.wasmWorkloadSource }}"' - template: /eng/pipelines/templates/send-to-helix-step.yml parameters: osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml index 479d40959a8..3c4fa42a959 100644 --- a/eng/pipelines/templates/runtime-perf-job.yml +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -16,6 +16,7 @@ parameters: isScenario: false downloadSpecificBuild: null # buildId, pipeline, branchName, project crossBuild: false + wasmWorkloadSource: '' runtimeRepoAlias: runtime performanceRepoAlias: self selfIsRuntime: true diff --git a/scripts/benchmarks_ci.py b/scripts/benchmarks_ci.py index 2aaea577db4..17131362e9e 100755 --- a/scripts/benchmarks_ci.py +++ b/scripts/benchmarks_ci.py @@ -101,6 +101,14 @@ def __is_valid_dotnet_path(dp: str) -> str: type=__is_valid_dotnet_path, help='Path to a custom dotnet' ) + parser.add_argument( + '--wasm-workload-source', + dest='wasm_workload_source', + required=False, + help=( + 'Exclusive NuGet source containing the coherent browser-WASM workload ' + 'packages for the selected SDK. Requires CoreCLR WASM ReadyToRun.') + ) parser.add_argument('--upload-to-perflab-container', dest="upload_to_perflab_container", @@ -220,6 +228,14 @@ def main(argv: list[str]): raise Exception("Framework version (-f) must be specified.") target_framework_monikers = dotnet.get_target_framework_monikers(args.frameworks) + if args.wasm_workload_source and not ( + args.wasm + and args.wasm_runtime_flavor == 'CoreCLR' + and args.wasm_ready_to_run): + raise ValueError( + '--wasm-workload-source requires --wasm ' + '--wasm-runtime-flavor CoreCLR --wasm-ready-to-run') + # Acquire necessary tools (dotnet) if not args.dotnet_path: init_tools( @@ -233,6 +249,15 @@ def main(argv: list[str]): else: dotnet.setup_dotnet(args.dotnet_path) + if args.wasm_workload_source: + os.environ.pop('PERFLAB_WASM_PACKAGE_VERSION', None) + dotnet.install_wasm_workload( + architecture=args.architecture, + target_framework_monikers=target_framework_monikers, + package_source=args.wasm_workload_source, + sdk_versions=args.dotnet_versions, + verbose=verbose) + # WORKAROUND # The MicroBenchmarks.csproj targets .NET Core 2.1, 3.0, 3.1 and 5.0 # to avoid a build failure when using older frameworks (error NETSDK1045: diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 90be9aaf918..10fe6582bd7 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -7,6 +7,10 @@ import re import json import datetime +import platform as platform_module +import shutil +import tempfile +import xml.etree.ElementTree as ET from argparse import ArgumentParser, ArgumentTypeError from glob import iglob from logging import getLogger @@ -68,6 +72,14 @@ def __log_script_header(message: str): CSharpProjFile = NamedTuple('CSharpProjFile', file_name=str, working_directory=str) +WasmPackage = NamedTuple('WasmPackage', package_id=str, version=str) +WasmWorkloadCohort = NamedTuple( + 'WasmWorkloadCohort', + sdk_version=str, + product_version=str, + host_rid=str, + workload_id=str, + packages=tuple[WasmPackage, ...]) @tracer.start_as_current_span("get_target_framework_moniker") def get_target_framework_moniker(framework: str) -> str: @@ -106,6 +118,294 @@ def get_target_framework_monikers(frameworks: list[str]) -> list[str]: # ['net6.0', 'nativeaot6.0'] should become ['net6.0'] return list(set(monikers)) + +def get_host_rid( + architecture: str, + system: Optional[str] = None, + libc: Optional[str] = None) -> str: + system = system or platform_module.system() + normalized_system = system.casefold() + + if normalized_system == 'darwin': + os_rid = 'osx' + elif normalized_system == 'windows': + os_rid = 'win' + elif normalized_system == 'linux': + if libc is None: + libc_name = platform_module.libc_ver()[0] + is_musl = ( + libc_name.casefold() == 'musl' + or path.isfile('/etc/alpine-release') + or any(iglob('/lib/ld-musl-*.so.1'))) + else: + is_musl = libc.casefold() == 'musl' + os_rid = 'linux-musl' if is_musl else 'linux' + else: + raise ValueError(f'Unsupported host operating system for Crossgen2: {system}') + + return f'{os_rid}-{architecture}' + + +def get_wasm_workload_cohort( + dotnet_root: str, + sdk_version: str, + target_framework: str, + host_rid: str) -> WasmWorkloadCohort: + sdk_major_match = re.match(r'^(\d+)\.', sdk_version) + framework_major_match = re.match(r'^net(\d+)\.0$', target_framework) + if not sdk_major_match or not framework_major_match: + raise ValueError( + f'Cannot determine a coherent WASM cohort for SDK {sdk_version} and {target_framework}') + if sdk_major_match.group(1) != framework_major_match.group(1): + raise ValueError( + f'WASM workload acquisition requires the SDK and target framework to have the same ' + f'major version, but got SDK {sdk_version} and {target_framework}') + + bundled_versions_path = path.join( + dotnet_root, 'sdk', sdk_version, 'Microsoft.NETCoreSdk.BundledVersions.props') + if not path.isfile(bundled_versions_path): + raise ValueError( + f'Cannot find SDK bundled product versions at {bundled_versions_path}') + + root = ET.parse(bundled_versions_path).getroot() + + def find_item(name: str, include: str) -> ET.Element: + for item in root.iter(name): + if (item.get('Include') == include + and item.get('TargetFramework') == target_framework): + return item + raise ValueError( + f'SDK {sdk_version} does not define {name} {include} for {target_framework}') + + framework_reference = find_item('KnownFrameworkReference', 'Microsoft.NETCore.App') + runtime_rids = framework_reference.get('RuntimePackRuntimeIdentifiers', '').split(';') + if 'browser-wasm' not in runtime_rids: + raise ValueError( + f'SDK {sdk_version} does not provide a browser-wasm CoreCLR runtime pack') + + crossgen = find_item('KnownCrossgen2Pack', 'Microsoft.NETCore.App.Crossgen2') + crossgen_rids = set( + crossgen.get('Crossgen2PortableRuntimeIdentifiers', '').split(';') + + crossgen.get('Crossgen2RuntimeIdentifiers', '').split(';')) + if host_rid not in crossgen_rids: + raise ValueError( + f'SDK {sdk_version} does not provide a Crossgen2 pack for host RID {host_rid}') + + illink = find_item('KnownILLinkPack', 'Microsoft.NET.ILLink.Tasks') + webassembly = find_item( + 'KnownWebAssemblySdkPack', 'Microsoft.NET.Sdk.WebAssembly.Pack') + package_versions = { + 'Microsoft.NETCore.App.Runtime.browser-wasm': + framework_reference.get('DefaultRuntimeFrameworkVersion'), + framework_reference.get('TargetingPackName', 'Microsoft.NETCore.App.Ref'): + framework_reference.get('TargetingPackVersion'), + 'Microsoft.NET.Sdk.WebAssembly.Pack': + webassembly.get('WebAssemblySdkPackVersion'), + f'Microsoft.NETCore.App.Crossgen2.{host_rid}': + crossgen.get('Crossgen2PackVersion'), + 'Microsoft.NET.ILLink.Tasks': + illink.get('ILLinkPackVersion'), + } + + missing_versions = [ + package_id for package_id, version in package_versions.items() if not version + ] + if missing_versions: + raise ValueError( + f'SDK {sdk_version} has no version for required WASM packages: ' + f'{", ".join(missing_versions)}') + + versions = set(package_versions.values()) + if len(versions) != 1: + formatted_versions = ', '.join( + f'{package_id}={version}' for package_id, version in package_versions.items()) + raise ValueError( + f'SDK {sdk_version} does not define one coherent CoreCLR browser-WASM ' + f'package version: {formatted_versions}') + + product_version = next(iter(versions)) + return WasmWorkloadCohort( + sdk_version=sdk_version, + product_version=product_version, + host_rid=host_rid, + workload_id='wasm-tools', + packages=tuple( + WasmPackage(package_id, version) + for package_id, version in package_versions.items())) + + +def get_wasm_workload_commands( + dotnet_executable: str, + cohort: WasmWorkloadCohort, + config_file: str, + restore_project: str, + package_root: str) -> list[list[str]]: + return [ + [ + dotnet_executable, + 'restore', + restore_project, + '--packages', + package_root, + '--configfile', + config_file, + '--no-http-cache', + ], + [ + dotnet_executable, + 'workload', + 'install', + cohort.workload_id, + '--skip-manifest-update', + '--configfile', + config_file, + '--no-cache', + ], + ] + + +def install_wasm_workload( + architecture: str, + target_framework_monikers: list[str], + package_source: str, + sdk_versions: list[str], + verbose: bool) -> WasmWorkloadCohort: + if len(target_framework_monikers) != 1: + raise ValueError( + '--wasm-workload-source requires exactly one target framework') + if len(sdk_versions) > 1: + raise ValueError( + '--wasm-workload-source supports at most one SDK version') + + dotnet_root = environ.get('DOTNET_ROOT') + if not dotnet_root: + raise ValueError('DOTNET_ROOT is not configured') + dotnet_executable = path.join( + dotnet_root, 'dotnet.exe' if platform == 'win32' else 'dotnet') + + if sdk_versions: + sdk_version = sdk_versions[0] + else: + sdk_version = check_output( + [dotnet_executable, '--version'], + cwd=dotnet_root, + text=True).strip() + + host_rid = get_host_rid(architecture) + cohort = get_wasm_workload_cohort( + dotnet_root, + sdk_version, + target_framework_monikers[0], + host_rid) + + parsed_source = urlparse(package_source) + if parsed_source.scheme not in ('http', 'https', 'file'): + package_source = path.abspath(package_source) + if not path.isdir(package_source): + raise ValueError( + f'WASM workload package source does not exist: {package_source}') + + available_packages = { + path.basename(package).casefold() + for package in iglob( + path.join(package_source, '**', '*.nupkg'), + recursive=True) + } + missing_packages = [ + f'{package.package_id}.{package.version}.nupkg' + for package in cohort.packages + if f'{package.package_id}.{package.version}.nupkg'.casefold() + not in available_packages + ] + if missing_packages: + raise ValueError( + f'WASM workload source {package_source} is missing required coherent ' + f'cohort packages: {", ".join(missing_packages)}') + + with tempfile.TemporaryDirectory(prefix='perflab-wasm-workload-') as temp_dir: + config_path = path.join(temp_dir, 'NuGet.Config') + config_root = ET.Element('configuration') + package_sources = ET.SubElement(config_root, 'packageSources') + ET.SubElement(package_sources, 'clear') + ET.SubElement( + package_sources, + 'add', + {'key': 'wasm-cohort', 'value': package_source}) + ET.ElementTree(config_root).write( + config_path, encoding='utf-8', xml_declaration=True) + + global_json_path = path.join(temp_dir, 'global.json') + with open(global_json_path, 'w', encoding='utf-8') as global_json: + json.dump({ + 'sdk': { + 'version': sdk_version, + 'rollForward': 'disable', + 'allowPrerelease': True, + } + }, global_json) + + restore_project_path = path.join(temp_dir, 'WasmCohort.csproj') + project = ET.Element('Project', {'Sdk': 'Microsoft.NET.Sdk'}) + properties = ET.SubElement(project, 'PropertyGroup') + ET.SubElement(properties, 'TargetFramework').text = target_framework_monikers[0] + ET.SubElement(properties, 'RestoreProjectStyle').text = 'PackageReference' + package_downloads = ET.SubElement(project, 'ItemGroup') + for package in cohort.packages: + ET.SubElement( + package_downloads, + 'PackageDownload', + { + 'Include': package.package_id, + 'Version': f'[{package.version}]', + }) + ET.ElementTree(project).write( + restore_project_path, encoding='utf-8', xml_declaration=True) + + package_root = path.join(temp_dir, 'packages') + commands = get_wasm_workload_commands( + dotnet_executable, + cohort, + config_path, + restore_project_path, + package_root) + try: + for command in commands: + RunCommand(command, verbose=verbose).run(temp_dir) + except CalledProcessError as error: + raise ValueError( + f'Failed to install the exact CoreCLR browser-WASM cohort ' + f'{cohort.product_version} from {package_source}') from error + + for package in cohort.packages: + restored_package = path.join( + package_root, + package.package_id.casefold(), + package.version.casefold()) + if not path.isdir(restored_package): + raise ValueError( + f'NuGet did not restore {package.package_id} {package.version} ' + f'from {package_source}') + installed_package = path.join( + dotnet_root, 'packs', package.package_id, package.version) + shutil.copytree(restored_package, installed_package, dirs_exist_ok=True) + + missing_installed_packs = [ + f'{package.package_id}/{package.version}' + for package in cohort.packages + if not path.isdir(path.join( + dotnet_root, 'packs', package.package_id, package.version)) + ] + if missing_installed_packs: + raise ValueError( + f'CoreCLR browser-WASM workload installation is incomplete: ' + f'{", ".join(missing_installed_packs)}') + + getLogger().info( + 'Installed coherent CoreCLR browser-WASM cohort %s for %s', + cohort.product_version, + cohort.host_rid) + return cohort + _VERSION_RE = re.compile(r'^\d+\.\d+\.\d+') def version_type(value: str) -> str: if not _VERSION_RE.search(value): diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index db14b4511a4..018cbbf2bed 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -12,6 +12,7 @@ import tempfile import time from traceback import format_exc +import urllib.parse import urllib.request import xml.etree.ElementTree as ET from typing import Any, Optional @@ -30,6 +31,20 @@ def apt_command(arguments: str, *, executable: str = "apt-get") -> str: return f"sudo {executable} {APT_LOCK_TIMEOUT_OPTION} {arguments}" +def normalize_wasm_workload_source(source: Optional[str]) -> Optional[str]: + normalized_source = source.strip() if source else "" + return normalized_source or None + + +def set_shell_environment_variable( + os_group: str, + name: str, + value: str) -> str: + if os_group == "windows": + return f'set "{name}={value}"' + return f"export {name}={value}" + + def output_counters_for_crank(reports: list[Any]): print("#StartJobStatistics") @@ -123,6 +138,7 @@ class RunPerformanceJobArgs: pdn_path: Optional[str] = None os_version: Optional[str] = None dotnet_version_link: Optional[str] = None + wasm_workload_source: Optional[str] = None target_csproj: Optional[str] = None build_config: str = DEFAULT_BUILD_CONFIG live_libraries_build_config: Optional[str] = None @@ -209,7 +225,12 @@ def get_pre_commands( codegen_type: str, build_config: str, v8_version: str, - wasm_local_package_version: Optional[str] = None): + wasm_local_package_version: Optional[str] = None, + wasm_workload_source: Optional[str] = None): + if os_group == "windows" and runtime_type in ("wasm", "wasm_coreclr"): + raise ValueError( + "WASM performance job prerequisite setup is not supported on Windows") + helix_pre_commands: list[str] = [] # Remember the previous PYTHONPATH that was set so it can be restored in the post commands @@ -286,17 +307,24 @@ def get_pre_commands( # Set up everything needed for WASM runs (both Mono and CoreCLR) if runtime_type in ("wasm", "wasm_coreclr"): + use_workload_source = ( + runtime_type == "wasm_coreclr" and bool(wasm_workload_source)) if runtime_type == "wasm_coreclr": - if not wasm_local_package_version: - raise ValueError("CoreCLR WASM requires a local WebAssembly toolchain package version") - install_prerequisites += [ - f"export PERFLAB_WASM_PACKAGE_VERSION={wasm_local_package_version}" - ] + if wasm_local_package_version: + install_prerequisites += [ + set_shell_environment_variable( + os_group, + "PERFLAB_WASM_PACKAGE_VERSION", + wasm_local_package_version) + ] + elif not use_workload_source: + raise ValueError( + "CoreCLR WASM requires either a private runtime payload or " + "a coherent WASM workload source") if os_distro == "azurelinux": # Azure Linux uses tdnf package manager install_prerequisites += [ - "export RestoreAdditionalProjectSources=$HELIX_CORRELATION_PAYLOAD/built-nugets", "sudo tdnf -y update", "sudo tdnf -y remove nodejs", "sudo tdnf -y install ca-certificates curl gnupg nodejs npm", @@ -308,7 +336,6 @@ def get_pre_commands( ] else: install_prerequisites += [ - "export RestoreAdditionalProjectSources=$HELIX_CORRELATION_PAYLOAD/built-nugets", apt_command("-y remove nodejs"), apt_command("update"), apt_command("install -y ca-certificates curl gnupg"), @@ -327,6 +354,18 @@ def get_pre_commands( "${V8_ENGINE_PATH} -e 'console.log(`V8 version: ${this.version()}`)'" ] + if not use_workload_source: + correlation_payload = ( + "%HELIX_CORRELATION_PAYLOAD%\\built-nugets" + if os_group == "windows" + else "$HELIX_CORRELATION_PAYLOAD/built-nugets") + install_prerequisites.insert( + 0, + set_shell_environment_variable( + os_group, + "RestoreAdditionalProjectSources", + correlation_payload)) + # Add the install_prerequisites to the pre_commands if os_group == "windows": # Chain Windows commands with error checking using && to ensure each command succeeds @@ -710,7 +749,8 @@ def get_work_item_command( bdn_artifacts_dir: str, wasm_coreclr: bool = False, wasm_ready_to_run: bool = False, - only_sanity_check: bool = False): + only_sanity_check: bool = False, + wasm_workload_source: Optional[str] = None): if os_group == "windows": work_item_command = [ "python", @@ -737,11 +777,25 @@ def get_work_item_command( work_item_command += ["--dotnet-versions", "$DOTNET_VERSION"] if wasm: - work_item_command += ["--run-isolated", "--wasm", "--dotnet-path", "$HELIX_CORRELATION_PAYLOAD/dotnet/"] + dotnet_path = ( + "%HELIX_CORRELATION_PAYLOAD%\\dotnet\\" + if os_group == "windows" + else "$HELIX_CORRELATION_PAYLOAD/dotnet/") + work_item_command += [ + "--run-isolated", + "--wasm", + "--dotnet-path", + dotnet_path, + ] if wasm_coreclr: work_item_command += ["--wasm-runtime-flavor", "CoreCLR"] if wasm_ready_to_run: work_item_command += ["--wasm-ready-to-run"] + if wasm_ready_to_run and wasm_workload_source: + work_item_command += [ + "--wasm-workload-source", + wasm_workload_source, + ] work_item_command += ["--bdn-artifacts", bdn_artifacts_dir] @@ -963,7 +1017,30 @@ def run_performance_job(args: RunPerformanceJobArgs): v8_version = "" wasm_local_package_version = None - if wasm_coreclr: + wasm_workload_source = normalize_wasm_workload_source( + args.wasm_workload_source) + wasm_sdk_cohort = ( + wasm_coreclr + and args.r2r_run_type == "r2r" + and wasm_workload_source is not None) + helix_wasm_workload_source = ( + wasm_workload_source if wasm_sdk_cohort else None) + if wasm_sdk_cohort and wasm_workload_source: + parsed_workload_source = urllib.parse.urlparse(wasm_workload_source) + local_workload_source = ( + urllib.request.url2pathname(parsed_workload_source.path) + if parsed_workload_source.scheme == "file" + else os.path.abspath(wasm_workload_source)) + if os.path.isdir(local_workload_source): + payload_workload_source = os.path.join( + payload_dir, "wasm-workload-source") + shutil.copytree(local_workload_source, payload_workload_source) + helix_wasm_workload_source = ( + "%HELIX_CORRELATION_PAYLOAD%\\wasm-workload-source" + if args.os_group == "windows" + else "$HELIX_CORRELATION_PAYLOAD/wasm-workload-source") + + if wasm_coreclr and not wasm_sdk_cohort: if args.libraries_download_dir is None: raise Exception("Libraries not downloaded for wasm_coreclr runs") @@ -974,7 +1051,7 @@ def run_performance_job(args: RunPerformanceJobArgs): payload_dir, ) - elif wasm: + elif wasm and not wasm_coreclr: if args.libraries_download_dir is None: raise Exception("Libraries not downloaded for wasm runs") @@ -1008,7 +1085,8 @@ def run_performance_job(args: RunPerformanceJobArgs): if args.javascript_engine_path is None: args.javascript_engine_path = f"/home/helixbot/.jsvu/bin/{args.javascript_engine}" - ci_setup_arguments.dotnet_path = f"{payload_dir}/dotnet" + if not wasm_sdk_cohort: + ci_setup_arguments.dotnet_path = f"{payload_dir}/dotnet" if args.dotnet_version_link is not None: if args.dotnet_version_link.startswith("https"): # Version link is a proper url @@ -1164,7 +1242,8 @@ def run_performance_job(args: RunPerformanceJobArgs): args.codegen_type, args.build_config, v8_version, - wasm_local_package_version) + wasm_local_package_version, + helix_wasm_workload_source) helix_post_commands = get_post_commands(args.os_group, args.internal, args.runtime_type) # Point ML.NET at the SSWE model that was pre-downloaded into the correlation payload above, so it @@ -1200,7 +1279,7 @@ def run_performance_job(args: RunPerformanceJobArgs): ci_setup_arguments.target_windows = args.os_group == "windows" ci_setup_arguments.output_file = os.path.join(root_payload_dir, "machine-setup") - if args.is_scenario: + if args.is_scenario or wasm_sdk_cohort: ci_setup_arguments.install_dir = os.path.join(payload_dir, "dotnet") elif wasm_coreclr: # For wasm_coreclr, we already have the SDK in the payload - skip downloading @@ -1413,7 +1492,8 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): artifact_dir, wasm_coreclr, wasm_coreclr and args.r2r_run_type == "r2r", - args.only_sanity_check) + args.only_sanity_check, + helix_wasm_workload_source) work_item_command = get_work_item_command_for_artifact_dir(bdn_artifacts_directory) baseline_work_item_command = get_work_item_command_for_artifact_dir(bdn_baseline_artifacts_dir) @@ -1570,6 +1650,7 @@ def main(argv: list[str]): "--perf-hash": "perf_hash", "--os-version": "os_version", "--dotnet-version-link": "dotnet_version_link", + "--wasm-workload-source": "wasm_workload_source", "--target-csproj": "target_csproj", "--pdn-path": "pdn_path", "--runtime-repo-dir": "runtime_repo_dir", diff --git a/scripts/tests/test_wasm_coreclr_r2r.py b/scripts/tests/test_wasm_coreclr_r2r.py index 6f88e06adac..7114434d8d3 100644 --- a/scripts/tests/test_wasm_coreclr_r2r.py +++ b/scripts/tests/test_wasm_coreclr_r2r.py @@ -3,6 +3,7 @@ import xml.etree.ElementTree as ET from argparse import Namespace from pathlib import Path +from typing import Optional import pytest @@ -10,8 +11,15 @@ sys.path.insert(0, str(scripts_dir)) import micro_benchmarks +import dotnet from build_runtime_payload import build_wasm_coreclr_payload -from run_performance_job import get_pre_commands, get_run_configurations, get_work_item_command +from run_performance_job import ( + get_pre_commands, + get_run_configurations, + get_work_item_command, + normalize_wasm_workload_source, + set_shell_environment_variable, +) def test_ready_to_run_requires_coreclr_wasm(): @@ -52,6 +60,48 @@ def test_ready_to_run_argument_is_forwarded_to_helix_work_item(): assert "--wasm-ready-to-run" in command +def test_workload_source_is_forwarded_to_helix_work_item(): + command = get_work_item_command( + os_group="linux", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="/tmp/artifacts", + wasm_coreclr=True, + wasm_ready_to_run=True, + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + source_index = command.index("--wasm-workload-source") + assert command[source_index + 1] == "https://example.test/cohort/v3/index.json" + + +def test_windows_wasm_work_item_uses_windows_payload_paths(): + command = get_work_item_command( + os_group="windows", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="%HELIX_WORKITEM_UPLOAD_ROOT%\\artifacts", + wasm_coreclr=True, + wasm_ready_to_run=True, + wasm_workload_source=( + "%HELIX_CORRELATION_PAYLOAD%\\wasm-workload-source"), + ) + + dotnet_path_index = command.index("--dotnet-path") + workload_source_index = command.index("--wasm-workload-source") + assert command[dotnet_path_index + 1] == ( + "%HELIX_CORRELATION_PAYLOAD%\\dotnet\\") + assert command[workload_source_index + 1] == ( + "%HELIX_CORRELATION_PAYLOAD%\\wasm-workload-source") + assert all("$HELIX_CORRELATION_PAYLOAD" not in argument for argument in command) + + def test_ready_to_run_has_distinct_result_configuration(): configurations = get_run_configurations( run_kind="micro", @@ -142,3 +192,264 @@ def test_coreclr_pre_commands_export_local_toolchain_package_version(): ) assert any("PERFLAB_WASM_PACKAGE_VERSION=11.0.0-ci" in command for command in commands) + assert any("RestoreAdditionalProjectSources" in command for command in commands) + + +@pytest.mark.parametrize("runtime_type", ["wasm", "wasm_coreclr"]) +def test_windows_wasm_pre_commands_fail_clearly(runtime_type): + with pytest.raises(ValueError, match="not supported on Windows"): + get_pre_commands( + os_group="windows", + os_distro=None, + internal=False, + runtime_type=runtime_type, + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_local_package_version=( + "11.0.0-ci" if runtime_type == "wasm_coreclr" else None), + ) + + +@pytest.mark.parametrize( + ("os_group", "expected"), + [ + ("windows", 'set "NAME=value"'), + ("linux", "export NAME=value"), + ], +) +def test_shell_environment_variable_command(os_group, expected): + assert set_shell_environment_variable(os_group, "NAME", "value") == expected + + +def test_coreclr_sdk_pre_commands_do_not_enable_private_package_overrides(): + commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=False, + runtime_type="wasm_coreclr", + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + assert not any("PERFLAB_WASM_PACKAGE_VERSION" in command for command in commands) + assert not any("RestoreAdditionalProjectSources" in command for command in commands) + + +def test_mono_wasm_pre_commands_preserve_private_package_source(): + commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=False, + runtime_type="wasm", + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + assert any("RestoreAdditionalProjectSources" in command for command in commands) + + +def test_non_r2r_coreclr_command_ignores_shared_workload_source(): + command = get_work_item_command( + os_group="linux", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="/tmp/artifacts", + wasm_coreclr=True, + wasm_ready_to_run=False, + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + assert "--wasm-workload-source" not in command + + +def test_pipeline_scopes_workload_source_to_coreclr_r2r(): + template = ( + scripts_dir.parent + / "eng" + / "pipelines" + / "templates" + / "run-performance-job.yml" + ).read_text(encoding="utf-8") + + condition = ( + "and(ne(parameters.wasmWorkloadSource, ''), " + "eq(parameters.runtimeType, 'wasm_coreclr'), " + "eq(parameters.r2rRunType, 'r2r'))" + ) + assert condition in template + + +@pytest.mark.parametrize("source", [None, "", " "]) +def test_empty_workload_source_is_not_enabled(source): + assert normalize_wasm_workload_source(source) is None + + +def test_workload_source_is_trimmed(): + assert normalize_wasm_workload_source( + " https://example.test/cohort/v3/index.json " + ) == "https://example.test/cohort/v3/index.json" + + +@pytest.mark.parametrize( + ("system", "architecture", "libc", "expected"), + [ + ("Darwin", "arm64", "", "osx-arm64"), + ("Darwin", "x64", "", "osx-x64"), + ("Linux", "x64", "glibc", "linux-x64"), + ("Linux", "x64", "musl", "linux-musl-x64"), + ("Windows", "x64", "", "win-x64"), + ], +) +def test_crossgen2_host_rid(system, architecture, libc, expected): + assert dotnet.get_host_rid(architecture, system=system, libc=libc) == expected + + +def _write_bundled_versions( + dotnet_root: Path, + sdk_version: str = "11.0.100-preview.1.12345.1", + product_version: str = "11.0.0-preview.1.12345.1", + illink_version: Optional[str] = None) -> None: + sdk_dir = dotnet_root / "sdk" / sdk_version + sdk_dir.mkdir(parents=True) + (sdk_dir / "Microsoft.NETCoreSdk.BundledVersions.props").write_text( + f""" + + + + + + + +""", + encoding="utf-8", + ) + + +def test_cohort_uses_exact_sdk_product_version_and_host_crossgen2(tmp_path): + sdk_version = "11.0.100-preview.1.12345.1" + product_version = "11.0.0-preview.1.12345.1" + _write_bundled_versions(tmp_path, sdk_version, product_version) + + cohort = dotnet.get_wasm_workload_cohort( + str(tmp_path), sdk_version, "net11.0", "osx-arm64") + + assert cohort.product_version == product_version + assert cohort.workload_id == "wasm-tools" + assert dotnet.WasmPackage( + "Microsoft.NETCore.App.Crossgen2.osx-arm64", + product_version, + ) in cohort.packages + assert {package.version for package in cohort.packages} == {product_version} + + +def test_cohort_rejects_mismatched_sdk_package_versions(tmp_path): + _write_bundled_versions( + tmp_path, + illink_version="11.0.0-preview.1.12345.2", + ) + + with pytest.raises(ValueError, match="does not define one coherent"): + dotnet.get_wasm_workload_cohort( + str(tmp_path), + "11.0.100-preview.1.12345.1", + "net11.0", + "linux-x64", + ) + + +def test_local_cohort_source_requires_all_exact_packages(tmp_path, monkeypatch): + sdk_version = "11.0.100-preview.1.12345.1" + product_version = "11.0.0-preview.1.12345.1" + dotnet_root = tmp_path / "dotnet" + package_source = tmp_path / "packages" + package_source.mkdir() + _write_bundled_versions(dotnet_root, sdk_version, product_version) + monkeypatch.setenv("DOTNET_ROOT", str(dotnet_root)) + monkeypatch.setattr(dotnet, "get_host_rid", lambda architecture: "linux-x64") + + for package_id in ( + "Microsoft.NETCore.App.Runtime.browser-wasm", + "Microsoft.NETCore.App.Ref", + "Microsoft.NET.Sdk.WebAssembly.Pack", + "Microsoft.NETCore.App.Crossgen2.linux-x64", + ): + (package_source / f"{package_id}.{product_version}.nupkg").touch() + + with pytest.raises(ValueError, match="Microsoft.NET.ILLink.Tasks"): + dotnet.install_wasm_workload( + architecture="x64", + target_framework_monikers=["net11.0"], + package_source=str(package_source), + sdk_versions=[sdk_version], + verbose=False, + ) + + +def test_cohort_source_rejects_multiple_sdk_versions(): + with pytest.raises(ValueError, match="at most one SDK version"): + dotnet.install_wasm_workload( + architecture="x64", + target_framework_monikers=["net11.0"], + package_source="https://example.test/cohort/v3/index.json", + sdk_versions=["11.0.100-preview.1", "11.0.100-preview.2"], + verbose=False, + ) + + +def test_generated_workload_commands_pin_workload_and_coreclr_r2r_cohort(): + cohort = dotnet.WasmWorkloadCohort( + sdk_version="11.0.100-preview.1.12345.1", + product_version="11.0.0-preview.1.12345.1", + host_rid="linux-x64", + workload_id="wasm-tools", + packages=(), + ) + + commands = dotnet.get_wasm_workload_commands( + "/dotnet/dotnet", + cohort, + "/tmp/NuGet.Config", + "/tmp/WasmCohort.csproj", + "/tmp/packages", + ) + + assert commands[0] == [ + "/dotnet/dotnet", + "restore", + "/tmp/WasmCohort.csproj", + "--packages", + "/tmp/packages", + "--configfile", + "/tmp/NuGet.Config", + "--no-http-cache", + ] + assert commands[1] == [ + "/dotnet/dotnet", + "workload", + "install", + "wasm-tools", + "--skip-manifest-update", + "--configfile", + "/tmp/NuGet.Config", + "--no-cache", + ]