diff --git a/NuGet.config b/NuGet.config index 2c233f9c47f..84541553c4b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -7,13 +7,10 @@ - - + - - diff --git a/docs/benchmarkdotnet.md b/docs/benchmarkdotnet.md index f8c9e222849..be429f57ad7 100644 --- a/docs/benchmarkdotnet.md +++ b/docs/benchmarkdotnet.md @@ -289,7 +289,7 @@ M00_L00: The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**. -Available options are: Mono, wasmnet70, CoreRT, net462, net47, net471, net472, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. +Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. Example: run the benchmarks for .NET 7.0 and 8.0: @@ -361,18 +361,6 @@ dotnet run -c Release -f net9.0 --cli "C:\Projects\performance\.dotnet\dotnet.ex This is very useful when you want to compare different builds of .NET. -### Private CLR Build - -It's possible to benchmark a private build of .NET Runtime. You just need to pass the value of `COMPLUS_Version` to BenchmarkDotNet. You can do that by either using `--clrVersion $theVersion` as an argument or `Job.ShortRun.With(new ClrRuntime(version: "$theVersion"))` in the code. - -So if you made a change in CLR and want to measure the difference, you can run the benchmarks with: - -```cmd -dotnet run -c Release -f net48 -- --clrVersion $theVersion -``` - -More info can be found in [BenchmarkDotNet issue #706](https://github.com/dotnet/BenchmarkDotNet/issues/706). - ### Private CoreRT Build To run benchmarks with private CoreRT build you need to provide the `IlcPath`. Example: diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6edc23c1cef..c345e47db95 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,39 +1,39 @@ - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 - + https://github.com/dotnet/dotnet - 5bed4499b04cbbaec57ac4209ae993acca3648cc + 547c640d5626b2976499cb3433abc741a63d67c5 https://github.com/dotnet/xharness @@ -55,17 +55,17 @@ 6e563dcf3cbf4853316eb4724e49ec92caeabb07 https://github.com/dotnet/maui - - 5bed4499b04cbbaec57ac4209ae993acca3648cc + + 547c640d5626b2976499cb3433abc741a63d67c5 https://github.com/dotnet/dotnet - - 5bed4499b04cbbaec57ac4209ae993acca3648cc + + 547c640d5626b2976499cb3433abc741a63d67c5 https://github.com/dotnet/dotnet - + https://github.com/dotnet/android - 9f89a27a70ce6828cffc2e66c967dd047b98656a + 6b255ec42cef9f9f757400f6edec7af239cacd58 4c9d1b112c16716c2479e054e9ad4db8b5b8c70c diff --git a/eng/Versions.props b/eng/Versions.props index c8cece31b71..07ade91729c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,12 +7,12 @@ - 11.0.0-preview.4.26210.111 - 11.0.0-preview.4.26210.111 - 11.0.0-preview.4.26210.111 - 11.0.0-preview.4.26210.111 - 0.16.0-nightly.20260320.467 - 11.0.0-preview.4.26210.111 + 11.0.0-preview.5.26261.101 + 11.0.0-preview.5.26261.101 + 11.0.0-preview.5.26261.101 + 11.0.0-preview.5.26261.101 + 0.16.0-nightly.20260518.1249 + 11.0.0-preview.5.26261.101 11.0.0-prerelease.26204.1 diff --git a/eng/common/AGENTS.md b/eng/common/AGENTS.md new file mode 100644 index 00000000000..a5ed8f72926 --- /dev/null +++ b/eng/common/AGENTS.md @@ -0,0 +1,5 @@ +# `eng/common` + +Files under `eng/common` come from [Arcade](https://github.com/dotnet/arcade). +Edits in `eng/common` will be overwritten by automation unless the changes are made directly in the Arcade repository. +For more information, see the [Arcade documentation](https://github.com/dotnet/arcade/tree/main/Documentation). diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 748c4f07a64..66c7988f222 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -26,12 +26,12 @@ parameters: enablePublishBuildArtifacts: false enablePublishBuildAssets: false enablePublishTestResults: false + enablePublishing: false enableBuildRetry: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' name: '' - componentGovernanceSteps: [] preSteps: [] artifactPublishSteps: [] runAsPublic: false @@ -152,9 +152,6 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - - ${{ each step in parameters.componentGovernanceSteps }}: - - ${{ step }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/core-templates/steps/cleanup-microbuild.yml parameters: diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index eefed3b667a..86ea9f63504 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -22,6 +22,7 @@ parameters: GitHubOrg: dotnet MirrorRepo: '' MirrorBranch: main + xLocCustomPowerShellScript: '' condition: '' JobNameSuffix: '' is1ESPipeline: '' @@ -97,6 +98,8 @@ jobs: gitHubOrganization: ${{ parameters.GitHubOrg }} mirrorRepo: ${{ parameters.MirrorRepo }} mirrorBranch: ${{ parameters.MirrorBranch }} + ${{ if ne(parameters.xLocCustomPowerShellScript, '') }}: + xLocCustomPowerShellScript: ${{ parameters.xLocCustomPowerShellScript }} condition: ${{ parameters.condition }} # Copy the locProject.json to the root of the Loc directory, then publish a pipeline artifact diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 9d7490518c4..700f7711465 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -172,17 +172,18 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml' artifactName: AssetManifests displayName: 'Publish Merged Manifest' - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish ReleaseConfigs Artifact - pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - publishLocation: Container + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' artifactName: ReleaseConfigs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # just metadata for publishing - ${{ if or(eq(parameters.publishAssetsImmediately, 'true'), eq(parameters.isAssetlessBuild, 'true')) }}: - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -218,4 +219,5 @@ jobs: - template: /eng/common/core-templates/steps/publish-logs.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'BuildAssetRegistry' JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/renovate.yml b/eng/common/core-templates/job/renovate.yml index ab233539b5d..ff86c80b468 100644 --- a/eng/common/core-templates/job/renovate.yml +++ b/eng/common/core-templates/job/renovate.yml @@ -135,7 +135,7 @@ jobs: condition: succeededOrFailed() targetPath: $(Build.ArtifactStagingDirectory) artifactName: $(Agent.JobName)_Logs_Attempt$(System.JobAttempt) - sbomEnabled: false + isProduction: false # logs are non-production artifacts steps: - checkout: self diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml index 01ada747665..cc8cce45278 100644 --- a/eng/common/core-templates/jobs/jobs.yml +++ b/eng/common/core-templates/jobs/jobs.yml @@ -43,6 +43,10 @@ parameters: artifacts: {} is1ESPipeline: '' + + # Publishing version w/default. + publishingVersion: 3 + repositoryAlias: self officialBuildId: '' @@ -102,6 +106,7 @@ jobs: parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} continueOnError: ${{ parameters.continueOnError }} + publishingVersion: ${{ parameters.publishingVersion }} dependsOn: - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.publishBuildAssetsDependsOn }}: diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index d5627a994ae..db298ae16ba 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -11,8 +11,6 @@ variables: - name: MaestroApiVersion value: "2020-02-20" - - name: SourceLinkCLIVersion - value: 3.0.0 - name: SymbolToolVersion value: 1.0.1 - name: BinlogToolVersion diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index 0994189969f..8aa86e30491 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -9,6 +9,7 @@ parameters: default: 3 values: - 3 + - 4 - name: BARBuildId displayName: BAR Build Id @@ -130,16 +131,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true - task: PowerShell@2 displayName: Validate @@ -173,16 +188,30 @@ stages: PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true + - ${{ if ne(parameters.publishingInfraVersion, 4) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + - ${{ if eq(parameters.publishingInfraVersion, 4) }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts (V4) + inputs: + itemPattern: '*/packages/**/*.nupkg' + targetPath: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + - task: CopyFiles@2 + displayName: Flatten packages to PackageArtifacts + inputs: + SourceFolder: '$(Build.ArtifactStagingDirectory)/PipelineArtifactsDownload' + Contents: '**/*.nupkg' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + flattenFolders: true # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -196,7 +225,7 @@ stages: displayName: Validate inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore + arguments: -task SigningValidation -restore -msbuildEngine dotnet /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' ${{ parameters.signingValidationAdditionalParameters }} @@ -208,53 +237,20 @@ stages: JobLabel: 'Signing' BinlogToolVersion: $(BinlogToolVersion) - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2025 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2026.amd64 - os: windows - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2026.amd64 - steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true + # SourceLink validation has been removed — the underlying CLI tool + # (targeting netcoreapp2.1) has not functioned for years. + # The enableSourceLinkValidation parameter is kept but ignored so + # existing pipelines that pass it are not broken. + # See https://github.com/dotnet/arcade/issues/16647 + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: 'SourceLink Validation Removed - please remove enableSourceLinkValidation from your pipeline' + pool: server + steps: + - task: Delay@1 + displayName: 'Warning: SourceLink validation removed (see https://github.com/dotnet/arcade/issues/16647)' + inputs: + delayForMinutes: '0' - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc @@ -317,7 +313,7 @@ stages: scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: > -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -PublishingInfraVersion 3 -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -RequireDefaultChannels ${{ parameters.requireDefaultChannels }} diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml index a7abd58c4bb..6dfa99ec5e3 100644 --- a/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -8,12 +8,11 @@ steps: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@2 displayName: Download Release Configs inputs: - buildType: current artifactName: ReleaseConfigs - checkDownloadedFiles: true + targetPath: '$(Build.StagingDirectory)/ReleaseConfigs' - task: AzureCLI@2 name: setReleaseVars diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml deleted file mode 100644 index cf0649aa956..00000000000 --- a/eng/common/core-templates/steps/component-governance.yml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - is1ESPipeline: false - displayName: 'Component Detection' - -steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - displayName: ${{ parameters.displayName }} - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml index 003f7eae0fa..aad0a8aeda3 100644 --- a/eng/common/core-templates/steps/generate-sbom.yml +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -1,54 +1,14 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - parameters: - PackageVersion: 11.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - is1ESPipeline: false - # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). - publishArtifacts: true + PackageVersion: unused + BuildDropPath: unused + PackageName: unused + ManifestDirPath: unused + IgnoreDirectories: unused + sbomContinueOnError: unused + is1ESPipeline: unused + publishArtifacts: unused steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 - script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- ${{ if eq(parameters.publishArtifacts, 'true')}}: - - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - targetPath: '${{ parameters.manifestDirPath }}' - artifactName: $(ARTIFACT_NAME) - + echo "##vso[task.logissue type=warning]Including generate-sbom.yml is deprecated, SBOM generation is handled 1ES PT now. Remove this include." + displayName: Issue generate-sbom.yml deprecation warning diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index a9ea99ba6aa..84a1922c73f 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -50,13 +50,15 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' condition: always() -- template: /eng/common/core-templates/steps/publish-build-artifacts.yml +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} args: displayName: Publish Logs - pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' - publishLocation: Container - artifactName: PostBuildLogs + targetPath: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + artifactName: PostBuildLogs_${{ parameters.StageLabel }}_${{ parameters.JobLabel }}_Attempt$(System.JobAttempt) continueOnError: true condition: always() + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts + diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index acf16ed3496..b75f59c428d 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -62,4 +62,4 @@ steps: artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() - sbomEnabled: false # we don't need SBOM for logs + isProduction: false # logs are non-production artifacts diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ff2dfdb4a5b..99d6dfe82dd 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -225,13 +225,19 @@ elseif(ILLUMOS) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + if ($ENV{CCC_CC} MATCHES ".*gcc.*") + set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + else() + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") + endif() # let CMake set up the correct search paths include(Platform/Haiku) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index e3374310563..a5be41db690 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -29,11 +29,11 @@ function InstallDarcCli ($darcVersion, $toolpath) { Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" - & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 9f5ad6b763b..b56d40e5706 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -73,9 +73,9 @@ function InstallDarcCli { echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity -g) else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") fi } diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 811f0f717f7..50ae6273768 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -10,7 +10,11 @@ Param( . $PSScriptRoot\tools.ps1 -$dotnetRoot = Join-Path $RepoRoot '.dotnet' +if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR +} else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' +} $installdir = $dotnetRoot try { diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 61f302bb677..1cb3f5abac2 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -80,7 +80,11 @@ case $cpuname in ;; esac -dotnetRoot="${repo_root}.dotnet" +if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnetRoot="$DOTNET_GLOBAL_INSTALL_DIR" +else + dotnetRoot="${repo_root}.dotnet" +fi if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 deleted file mode 100644 index a0c7d792a76..00000000000 --- a/eng/common/generate-sbom-prep.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed -) - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' -$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName - -Write-Host "Artifact name before : $ArtifactName" -Write-Host "Artifact name after : $SafeArtifactName" - -Write-Host "Creating dir $ManifestDirPath" - -# create directory for sbom manifest to be placed -if (!(Test-Path -path $SbomGenerationDir)) -{ - New-Item -ItemType Directory -path $SbomGenerationDir - Write-Host "Successfully created directory $SbomGenerationDir" -} -else{ - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -} - -Write-Host "Updating artifact name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh deleted file mode 100644 index b8ecca72bbf..00000000000 --- a/eng/common/generate-sbom-prep.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -. $scriptroot/pipeline-logging-functions.sh - - -# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. -artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" -safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" -manifest_dir=$1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -sbom_generation_dir="$manifest_dir/$safe_artifact_name" - -if [ ! -d "$sbom_generation_dir" ] ; then - mkdir -p "$sbom_generation_dir" - echo "Sbom directory created." $sbom_generation_dir -else - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -fi - -echo "Artifact name before : "$artifact_name -echo "Artifact name after : "$safe_artifact_name -export ARTIFACT_NAME=$safe_artifact_name -echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" - -exit 0 diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index fc0218a013d..672f4e2652e 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -49,8 +49,8 @@ try { Write-Host "Installing Binlog redactor CLI..." Write-Host "'$dotnet' new tool-manifest" & "$dotnet" new tool-manifest - Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" - & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + Write-Host "'$dotnet' tool install $packageName --local --source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --source "$PackageFeed" -v $verbosity --version $BinlogToolVersion if (Test-Path $TokensFilePath) { Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 deleted file mode 100644 index 1976ef70fb8..00000000000 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ /dev/null @@ -1,327 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages - [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -$disableConfigureToolsetImport = $true -. $PSScriptRoot\..\tools.ps1 - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -# Maximum number of jobs to run in parallel -$MaxParallelJobs = 16 - -$MaxRetries = 5 -$RetryWaitTimeInSeconds = 30 - -# Wait time between check for system load -$SecondsBetweenLoadChecks = 10 - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No files to validate." - ExitWithExitCode 0 -} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - . $using:PSScriptRoot\..\tools.ps1 - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-Host "Input file does not exist: $PackagePath" - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @('.dll', '.exe', '.pdb') - - Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith('.resources.dll')) { - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" - $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" - $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" - - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - $attempts = 0 - - while ($attempts -lt $using:MaxRetries) { - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - if ($Link -match "submodules") { - # Skip submodule links until sourcelink properly handles submodules - $Status = 200 - } - elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - # Only GitHub links are valid - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - # If it's not a github link, we want to break out of the loop and not retry. - $Status = 0 - $attempts = $using:MaxRetries - } - } - catch { - Write-Host $_ - $Status = 0 - } - } - - if ($Status -ne 200) { - $attempts++ - - if ($attempts -lt $using:MaxRetries) - { - $attemptsLeft = $using:MaxRetries - $attempts - Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $using:RetryWaitTimeInSeconds - } - else { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - else { - break - } - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - } - catch { - Write-Host $_ - } - finally { - $zip.Dispose() - } - - if ($FailedFiles -eq 0) { - Write-Host 'Passed.' - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - else { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } -} - -function CheckJobResult( - $result, - $packagePath, - [ref]$ValidationFailures, - [switch]$logErrors) { - if ($result -ne '0') { - if ($logErrors) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." - } - $ValidationFailures.Value++ - } -} - -function ValidateSourceLinkLinks { - if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) { - if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'" - ExitWithExitCode 1 - } - else { - $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; - } - } - - if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" - ExitWithExitCode 1 - } - - if ($GHRepoName -ne '' -and $GHCommit -ne '') { - $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1') - $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript') - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." - } - } - elseif ($GHRepoName -ne '' -or $GHCommit -ne '') { - Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.' - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - $ValidationFailures = 0 - - # Process each NuGet package in parallel - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - Write-Host "Starting $($_.FullName)" - Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null - $NumJobs = @(Get-Job -State 'Running').Count - - while ($NumJobs -ge $MaxParallelJobs) { - Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." - sleep $SecondsBetweenLoadChecks - $NumJobs = @(Get-Job -State 'Running').Count - } - - foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors - Remove-Job -Id $Job.Id - } - } - - foreach ($Job in @(Get-Job)) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) - Remove-Job -Id $Job.Id - } - if ($ValidationFailures -gt 0) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." - ExitWithExitCode 1 - } -} - -function InstallSourcelinkCli { - $sourcelinkCliPackageName = 'sourcelink' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { - Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." - } - else { - Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global - } -} - -try { - InstallSourcelinkCli - - foreach ($Job in @(Get-Job)) { - Remove-Job -Id $Job.Id - } - - ValidateSourceLinkLinks -} -catch { - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 4017ff15ebf..68119de603e 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -66,20 +66,7 @@ try { if( $msbuildEngine -eq "vs") { # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty - } - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty - } - if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true - } - if ($xcopyMSBuildToolsFolder -eq $null) { - throw 'Unable to get xcopy downloadable version of msbuild' - } - - $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" + $global:_MSBuildExe = InitializeVisualStudioMSBuild } $taskProject = GetSdkTaskProject $task diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index cdc62e72b07..f772aa3d78f 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -81,7 +81,6 @@ eng\common\ publish-build-artifacts.yml (logic) publish-pipeline-artifacts.yml (logic) component-governance.yml (shim) - generate-sbom.yml (shim) publish-logs.yml (shim) retain-build.yml (shim) send-to-helix.yml (shim) @@ -104,7 +103,6 @@ eng\common\ setup-maestro-vars.yml (logic) steps\ component-governance.yml (logic) - generate-sbom.yml (logic) publish-build-artifacts.yml (redirect) publish-logs.yml (logic) publish-pipeline-artifacts.yml (redirect) diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index f70224eaa45..d68e9fbc265 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,24 +1,15 @@ parameters: -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml parameters: is1ESPipeline: true - componentGovernanceSteps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion }} - BuildDropPath: ${{ parameters.buildDropPath }} - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - publishArtifacts: false - # publish artifacts # for 1ES managed templates, use the templateContext.output to handle multiple outputs. templateContext: @@ -26,12 +17,19 @@ jobs: outputs: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - output: buildArtifacts + - output: pipelineArtifact displayName: Publish pipeline artifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + continueOnError: true + - output: pipelineArtifact + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked continueOnError: true - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - output: pipelineArtifact @@ -40,8 +38,8 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: - output: pipelineArtifact @@ -50,7 +48,8 @@ jobs: artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # logs are non-production artifacts - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - output: pipelineArtifact @@ -58,14 +57,20 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked + isProduction: false # BuildConfiguration is a non-production artifact - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + # V4 publishing: automatically publish staged artifacts as a pipeline artifact. + # The artifact name matches the SDK's FutureArtifactName ($(System.PhaseName)_Artifacts), + # which is encoded in the asset manifest for downstream publishing to discover. + # Jobs can opt in by setting enablePublishing: true. + - ${{ if and(eq(parameters.publishingVersion, 4), eq(parameters.enablePublishing, 'true')) }}: - output: pipelineArtifact - displayName: Publish SBOM manifest + displayName: 'Publish V4 pipeline artifacts' + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: '$(System.PhaseName)_Artifacts' continueOnError: true - targetPath: $(Build.ArtifactStagingDirectory)/sbom - artifactName: $(ARTIFACT_NAME) + retryCountOnTaskFailure: 10 # for any files being locked # add any outputs provided via root yaml - ${{ if ne(parameters.templateContext.outputs, '') }}: diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml deleted file mode 100644 index 30bb3985ca2..00000000000 --- a/eng/common/templates-official/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml index 172f9f0fdc9..9e5981365e5 100644 --- a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -24,5 +24,7 @@ steps: artifactName: ${{ parameters.args.artifactName }} ${{ if parameters.args.properties }}: properties: ${{ parameters.args.properties }} - ${{ if parameters.args.sbomEnabled }}: + ${{ if ne(parameters.args.sbomEnabled, '') }}: sbomEnabled: ${{ parameters.args.sbomEnabled }} + ${{ if ne(parameters.args.isProduction, '') }}: + isProduction: ${{ parameters.args.isProduction }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 7f1b5d97d1a..5e261f34db4 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,12 +1,12 @@ parameters: enablePublishBuildArtifacts: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' -# Sbom related params - enableSbom: true runAsPublic: false - PackageVersion: 9.0.0 - BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' +# CG related params, unused now and can eventually be removed + disableComponentGovernance: unused +# Sbom related params, unused now and can eventually be removed + enableSbom: unused + PackageVersion: unused + BuildDropPath: unused jobs: - template: /eng/common/core-templates/job/job.yml @@ -21,32 +21,34 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - componentGovernanceSteps: - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + # we don't run CG in public + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable artifactPublishSteps: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: is1ESPipeline: false args: displayName: Publish pipeline artifacts - pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - publishLocation: Container + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true - condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked + condition: succeeded() + retryCountOnTaskFailure: 10 # for any files being locked + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: false + args: + displayName: Publish pipeline artifacts + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts' + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: not(succeeded()) + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml parameters: @@ -57,8 +59,7 @@ jobs: displayName: 'Publish logs' continueOnError: true condition: always() - retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -70,7 +71,7 @@ jobs: artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }} continueOnError: true condition: always() - sbomEnabled: false + retryCountOnTaskFailure: 10 # for any files being locked - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -81,4 +82,4 @@ jobs: artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true - sbomEnabled: false # we don't need SBOM for BuildConfiguration + retryCountOnTaskFailure: 10 # for any files being locked diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml deleted file mode 100644 index c12a5f8d21d..00000000000 --- a/eng/common/templates/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 6710ffb884b..0e281df8cae 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -168,6 +168,12 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 } + # Keep repo builds isolated from machine-installed SDK state and workload advertising. + # This avoids preview SDK builds picking up mismatched workloads on CI images. + $env:DOTNET_MULTILEVEL_LOOKUP = '0' + $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = '1' + $env:DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE = '1' + # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetExecutable = GetExecutableFileName 'dotnet' @@ -185,7 +191,11 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { - $dotnetRoot = Join-Path $RepoRoot '.dotnet' + if (-not [string]::IsNullOrEmpty($env:DOTNET_GLOBAL_INSTALL_DIR)) { + $dotnetRoot = $env:DOTNET_GLOBAL_INSTALL_DIR + } else { + $dotnetRoot = Join-Path $RepoRoot '.dotnet' + } if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { @@ -226,6 +236,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { Write-PipelinePrependPath -Path $dotnetRoot Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' + Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' + Write-PipelineSetVariable -Name 'DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -375,12 +388,11 @@ function InstallDotNet([string] $dotnetRoot, # # 1. MSBuild from an active VS command prompt # 2. MSBuild from a compatible VS installation -# 3. MSBuild from the xcopy tool package # # Returns full path to msbuild.exe. # Throws on failure. # -function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { +function InitializeVisualStudioMSBuild([object]$vsRequirements = $null) { if (-not (IsWindowsPlatform)) { throw "Cannot initialize Visual Studio on non-Windows" } @@ -390,13 +402,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } # Minimum VS version to require. - $vsMinVersionReqdStr = '17.7' - $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) - - # If the version of msbuild is going to be xcopied, - # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/18.0.0 - $defaultXCopyMSBuildVersion = '18.0.0' + $vsMinVersionReqdStr = '18.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -426,46 +432,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } } - # Locate Visual Studio installation or download x-copy msbuild. + # Locate Visual Studio installation. $vsInfo = LocateVisualStudio $vsRequirements - if ($vsInfo -ne $null -and $env:ForceUseXCopyMSBuild -eq $null) { + if ($vsInfo -ne $null) { # Ensure vsInstallDir has a trailing slash $vsInstallDir = Join-Path $vsInfo.installationPath "\" $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { - $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } else { - #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download - if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" - $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion - $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] - } - else{ - # If the VS version IS compatible, look for an xcopy msbuild package - # with a version matching VS. - # Note: If this version does not exist, then an explicit version of xcopy msbuild - # can be specified in global.json. This will be required for pre-release versions of msbuild. - $vsMajorVersion = $vsMinVersion.Major - $vsMinorVersion = $vsMinVersion.Minor - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" - } - } - - $vsInstallDir = $null - if ($xcopyMSBuildVersion.Trim() -ine "none") { - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install - if ($vsInstallDir -eq $null) { - throw "Could not xcopy msbuild. Please check that package 'Microsoft.DotNet.Arcade.MSBuild.Xcopy @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." - } - } - if ($vsInstallDir -eq $null) { - throw 'Unable to find Visual Studio that has required version and components installed' - } + throw 'Unable to find Visual Studio that has required version and components installed' } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } @@ -492,38 +468,6 @@ function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [str } } -function InstallXCopyMSBuild([string]$packageVersion) { - return InitializeXCopyMSBuild $packageVersion -install $true -} - -function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'Microsoft.DotNet.Arcade.MSBuild.Xcopy' - $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" - $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" - - if (!(Test-Path $packageDir)) { - if (!$install) { - return $null - } - - Create-Directory $packageDir - - Write-Host "Downloading $packageName $packageVersion" - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Retry({ - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -UseBasicParsing -OutFile $packagePath - }) - - if (!(Test-Path $packagePath)) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "See https://dev.azure.com/dnceng/internal/_wiki/wikis/DNCEng%20Services%20Wiki/1074/Updating-Microsoft.DotNet.Arcade.MSBuild.Xcopy-WAS-RoslynTools.MSBuild-(xcopy-msbuild)-generation?anchor=troubleshooting for help troubleshooting issues with XCopy MSBuild" - throw - } - Unzip $packagePath $packageDir - } - - return Join-Path $packageDir 'tools' -} - # # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # @@ -632,7 +576,7 @@ function InitializeBuildTool() { $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' } } elseif ($msbuildEngine -eq "vs") { try { - $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + $msbuildPath = InitializeVisualStudioMSBuild } catch { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ ExitWithExitCode 1 @@ -684,11 +628,7 @@ function GetSdkTaskProject([string]$taskName) { if (Test-Path $proj) { return $proj } - # TODO: Remove this fallback once all supported versions use the new layout. - $legacyProj = Join-Path $toolsetDir "SdkTasks\$taskName.proj" - if (Test-Path $legacyProj) { - return $legacyProj - } + throw "Unable to find $taskName.proj in toolset at: $toolsetDir" } @@ -745,32 +685,33 @@ function InitializeToolset() { ExitWithExitCode 1 } - $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--prerelease", "--output", "$nugetCache") - if ($env:NUGET_CONFIG) { + $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetCache") + $nugetConfig = $env:NUGET_CONFIG + if (-not $nugetConfig) { + # Search for any variation of nuget.config in the RepoRoot + $configFile = Get-ChildItem -Path $RepoRoot -File | Where-Object { $_.Name -ieq "nuget.config" } | Select-Object -First 1 + + if ($configFile) { + $nugetConfig = $configFile.FullName + } + } + + if ($nugetConfig) { $downloadArgs += "--configfile" - $downloadArgs += $env:NUGET_CONFIG + $downloadArgs += $nugetConfig } DotNet @downloadArgs $packageDir = Join-Path $nugetCache (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) $packageToolsetDir = Join-Path $packageDir 'toolset' - $packageToolsDir = Join-Path $packageDir 'tools' - # TODO: Remove the tools/ check once all supported versions have the toolset folder. - if (!(Test-Path $packageToolsetDir) -and !(Test-Path $packageToolsDir)) { + if (!(Test-Path $packageToolsetDir)) { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Arcade SDK package does not contain a toolset or tools folder: $packageDir" ExitWithExitCode 3 } New-Item -ItemType Directory -Path $toolsetToolsDir -Force | Out-Null - - # Copy toolset if present at the package root (new layout), otherwise fall back to tools - if (Test-Path $packageToolsetDir) { - Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force - } else { - # TODO: Remove this fallback once all supported versions have the toolset folder. - Copy-Item -Path "$packageToolsDir\*" -Destination $toolsetToolsDir -Recurse -Force - } + Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force if (Test-Path $buildProjPath) { $toolsetBuildProj = $buildProjPath @@ -897,6 +838,10 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" + if ($ci -and $buildTool.Tool -eq 'dotnet') { + $cmdArgs += ' /p:MSBuildEnableWorkloadResolver=false' + } + # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable if ($env:MSBUILD_MT_ENABLED -eq "1") { $cmdArgs += ' -mt' @@ -1007,6 +952,12 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if (-not $env:MSBUILDDEBUGPATH) { + $env:MSBUILDDEBUGPATH = Join-Path $LogDir 'MsbuildDebugLogs' +} + Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index d2339eb21d5..5ff37cfb700 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -126,6 +126,12 @@ function InitializeDotNetCli { export DOTNET_CLI_TELEMETRY_OPTOUT=1 fi + # Keep repo builds isolated from machine-installed SDK state and workload advertising. + # This avoids preview SDK builds picking up mismatched workloads on CI images. + export DOTNET_MULTILEVEL_LOOKUP=0 + export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + export DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=1 + # LTTNG is the logging infrastructure used by Core CLR. Need this variable set # so it doesn't output warnings to the console. export LTTNG_HOME="$HOME" @@ -148,7 +154,11 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="${repo_root}.dotnet" + if [[ -n "${DOTNET_GLOBAL_INSTALL_DIR:-}" ]]; then + dotnet_root="$DOTNET_GLOBAL_INSTALL_DIR" + else + dotnet_root="${repo_root}.dotnet" + fi export DOTNET_INSTALL_DIR="$dotnet_root" @@ -167,6 +177,9 @@ function InitializeDotNetCli { Write-PipelinePrependPath -path "$dotnet_root" Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" + Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" + Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" + Write-PipelineSetVariable -name "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE" -value "1" # return value _InitializeDotNetCli="$dotnet_root" @@ -426,29 +439,32 @@ function InitializeToolset { ExitWithExitCode 2 fi - local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--prerelease" "--output" "$_GetNuGetPackageCachePath") - if [[ -n "${NUGET_CONFIG:-}" ]]; then - download_args+=("--configfile" "$NUGET_CONFIG") + local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_GetNuGetPackageCachePath") + local nuget_config="${NUGET_CONFIG:-}" + if [[ -z "$nuget_config" ]]; then + # Search for any variation of nuget.config in the RepoRoot + local found_config + found_config=$(find "$repo_root" -maxdepth 1 -type f -iname "nuget.config" -print -quit) + + if [[ -n "$found_config" ]]; then + nuget_config="$found_config" + fi + fi + + if [[ -n "$nuget_config" ]]; then + download_args+=("--configfile" "$nuget_config") fi DotNet "${download_args[@]}" local package_dir="$_GetNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" - # TODO: Remove the tools/ check once all supported versions have the toolset folder. - if [[ ! -d "$package_dir/toolset" && ! -d "$package_dir/tools" ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset or tools folder: $package_dir" + if [[ ! -d "$package_dir/toolset" ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset folder: $package_dir" ExitWithExitCode 3 fi mkdir -p "$toolset_tools_dir" - - # Copy toolset if present at the package root (new layout), otherwise fall back to tools - if [[ -d "$package_dir/toolset" ]]; then - cp -r "$package_dir/toolset/." "$toolset_tools_dir" - else - # TODO: Remove this fallback once all supported versions have the toolset folder. - cp -r "$package_dir/tools/." "$toolset_tools_dir" - fi + cp -r "$package_dir/toolset/." "$toolset_tools_dir" if [[ -a "$toolset_tools_dir/Build.proj" ]]; then toolset_build_proj="$toolset_tools_dir/Build.proj" @@ -575,7 +591,12 @@ function MSBuild-Core { warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=$warn_not_as_error" fi - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + local workload_resolver_switch="" + if [[ "$ci" == true && -n "${_InitializeBuildToolCommand:-}" ]]; then + workload_resolver_switch="/p:MSBuildEnableWorkloadResolver=false" + fi + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch $workload_resolver_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { @@ -600,12 +621,7 @@ function GetSdkTaskProject { echo "$proj" return fi - # TODO: Remove this fallback once all supported versions use the new layout. - local legacyProj="$toolsetDir/SdkTasks/$taskName.proj" - if [[ -a "$legacyProj" ]]; then - echo "$legacyProj" - return - fi + Write-PipelineTelemetryError -category 'Build' "Unable to find $taskName.proj in toolset at: $toolsetDir" ExitWithExitCode 3 } @@ -645,6 +661,12 @@ mkdir -p "$toolset_dir" mkdir -p "$temp_dir" mkdir -p "$log_dir" +# Direct MSBuild crash diagnostics (MSB4166 failure.txt files) to a known location +# under artifacts/log so they are captured as build artifacts in CI. +if [[ -z "${MSBUILDDEBUGPATH:-}" ]]; then + export MSBUILDDEBUGPATH="$log_dir/MsbuildDebugLogs" +fi + Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir" Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" diff --git a/eng/pipelines/runtime-perf-jobs.yml b/eng/pipelines/runtime-perf-jobs.yml index a37eef06f18..3b9493cc0ee 100644 --- a/eng/pipelines/runtime-perf-jobs.yml +++ b/eng/pipelines/runtime-perf-jobs.yml @@ -56,23 +56,22 @@ parameters: type: object default: enabled: true - # Temporarily disabling cobalt jobs because the cobalt machines are offline - name: cobaltMicro type: object default: - enabled: false + enabled: true configs: - linux_arm64 - name: cobaltSveMicro type: object default: - enabled: false + enabled: true configs: - linux_arm64 - name: cobaltMicroR2RInterpreter type: object default: - enabled: false + enabled: true configs: - linux_arm64 - name: androidCoreclrR2r diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 4c3a62d1a72..c3c87fca015 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -17,6 +17,43 @@ jobs: # Public correctness jobs ###################################################### +- ${{ if parameters.runPublicJobs }}: + - job: Tooling_Tests_Windows + displayName: Tooling Tests (Windows) + timeoutInMinutes: 30 + condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main')) + pool: + vmImage: windows-2022 + + steps: + - task: UseDotNet@2 + displayName: Install .NET 8.0 + inputs: + version: 8.0.x + + - task: UseDotNet@2 + displayName: Install .NET 10.0 + inputs: + version: 10.0.x + includePreviewVersions: true + + - task: UseDotNet@2 + displayName: Install .NET 11.0 + inputs: + version: 11.0.x + includePreviewVersions: true + + - pwsh: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + + dotnet test src\tools\ResultsComparer.Tests\ResultsComparer.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal + dotnet test src\tools\Reporting\Reporting.Tests\Reporting.Tests.csproj --configuration Release --framework net11.0 --nologo --verbosity minimal + dotnet test src\tools\CertHelperTests\CertRotatorTests.csproj --configuration Release --framework net10.0 --nologo --verbosity minimal + dotnet test src\tests\harness\BenchmarkDotNet.Extensions.Tests\BenchmarkDotNet.Extensions.Tests.csproj --configuration Release --framework net11.0 -p:PERFLAB_TARGET_FRAMEWORKS=net10.0 --nologo --verbosity minimal + dotnet test src\tools\ScenarioMeasurement\Startup.Tests\Startup.Tests.csproj --configuration Release --nologo --verbosity minimal + displayName: Run tooling tests + - ${{ if parameters.runPublicJobs }}: # Scenario benchmarks diff --git a/gc-azure-pipelines.yml b/gc-azure-pipelines.yml index cf6b208a8ee..1d1fd31fb48 100644 --- a/gc-azure-pipelines.yml +++ b/gc-azure-pipelines.yml @@ -1,5 +1,5 @@ -# Project: src\benchmarks\gc\GC.Infrastructure\GC.Infrastructure.NotebookTests\GC.Infrastructure.NotebookTests.csproj. -# Output: artifacts\bin\GC.Infrastructure.NotebookTests\Debug\net8.0\GC.Infrastructure.NotebookTests.dll +# Project: src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core.UnitTests/GC.Infrastructure.Core.UnitTests.csproj. +# Output: artifacts\bin\GC.Infrastructure.Core.UnitTests\Debug\net8.0\GC.Infrastructure.Core.UnitTests.dll resources: containers: @@ -26,15 +26,15 @@ pr: jobs: # TODO: Add back once we have validated windows. - # - job: GCNotebookValidation_Ubuntu + # - job: GCInfrastructureValidation_Ubuntu # pool: # vmImage: ubuntu-latest # container: ubuntu_x64_build_container # TODO: Add. - - job: GCNotebookValidation_Windows + - job: GCInfrastructureValidation_Windows pool: - vmImage: windows-2019 + vmImage: windows-2025 steps: # Install dotnet. We temporarily need both .NET 8 (for building and running the projects) and .NET 9 (for installing deps). @@ -52,10 +52,17 @@ jobs: inputs: version: 10.0.x includePreviewVersions: true + - task: UseDotNet@2 + displayName: Install .NET 11.0 + inputs: + version: 11.0.x + includePreviewVersions: true - script: dotnet tool restore - script: dotnet tool install --global dotnet-repl - - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.NotebookTests/GC.Infrastructure.NotebookTests.csproj --configuration Debug --framework net8.0 - - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/GC.Analysis.API.csproj --configuration Release --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/GC.Infrastructure.Core.csproj --configuration Debug --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core.UnitTests/GC.Infrastructure.Core.UnitTests.csproj --configuration Debug --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/GC.Analysis.API.csproj --configuration Debug --framework net8.0 + - script: dotnet build src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API.UnitTests/GC.Analysis.API.UnitTests.csproj --configuration Debug --framework net8.0 # Run tests. Template installed from: https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/vstest-v3?view=azure-pipelines - task: VSTest@3 @@ -63,16 +70,36 @@ jobs: # -------------- # # Test selection # # -------------- # - testAssemblyVer2: GC.Infrastructure.NotebookTests.dll - searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/bin/GC.Infrastructure.NotebookTests/Debug/net8.0' + testAssemblyVer2: GC.Infrastructure.Core.UnitTests.dll + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/bin/GC.Infrastructure.Core.UnitTests/Debug/net8.0' + otherConsoleOptions: '--Framework:net8.0' # Uncomment to save results in the future. # resultsFolder: '$(Agent.TempDirectory)\TestResults' # string. Test results folder. Default: $(Agent.TempDirectory)\TestResults. # ----------------- # # Reporting options # # ----------------- # - testRunTitle: 'GC Notebook Test' + testRunTitle: 'GC Infrastructure Core Unit Test' # TODO: Double Check. platform: 'Windows' ## string. Build platform. # TODO: Double Check. - configuration: 'Test Notebooks' # string. Build configuration. \ No newline at end of file + configuration: 'Test GC Infrastructure Core' # string. Build configuration. + - task: VSTest@3 + inputs: + # -------------- # + # Test selection # + # -------------- # + testAssemblyVer2: GC.Analysis.API.UnitTests.dll + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/bin/GC.Analysis.API.UnitTests/Debug/net8.0' + otherConsoleOptions: '--Framework:net8.0' + # Uncomment to save results in the future. + # resultsFolder: '$(Agent.TempDirectory)\TestResults' # string. Test results folder. Default: $(Agent.TempDirectory)\TestResults. + + # ----------------- # + # Reporting options # + # ----------------- # + testRunTitle: 'GC Analysis API Unit Test' + # TODO: Double Check. + platform: 'Windows' ## string. Build platform. + # TODO: Double Check. + configuration: 'Test GC Analysis API' # string. Build configuration. diff --git a/global.json b/global.json index 79f2b114c10..148bd94594e 100644 --- a/global.json +++ b/global.json @@ -1,15 +1,15 @@ { "sdk": { - "version": "11.0.100-preview.3.26170.106", + "version": "11.0.100-preview.4.26210.111", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "11.0.100-preview.3.26170.106" + "dotnet": "11.0.100-preview.4.26210.111" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26210.111", - "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26210.111" + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26261.101", + "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26261.101" }, "native-tools": { "python3": "3.7.1" diff --git a/requirements.txt b/requirements.txt index 41c1983ebf9..4b121a529d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -cryptography==46.0.3 +cryptography==46.0.7 azure.storage.blob==12.13.0 azure.storage.queue==12.4.0 azure.identity==1.16.1 gitpython<=3.1.41 -urllib3==2.6.3 +urllib3==2.7.0 opentelemetry-api==1.23.0 opentelemetry-sdk==1.23.0 six==1.17.0 \ No newline at end of file diff --git a/scripts/channel_map.py b/scripts/channel_map.py index 0ca011576d5..118ad072022 100644 --- a/scripts/channel_map.py +++ b/scripts/channel_map.py @@ -9,13 +9,11 @@ class ChannelMap(): }, '10.0': { 'tfm': 'net10.0', - 'branch': '10.0', - 'quality': 'daily' + 'branch': '10.0' }, 'release/10.0': { 'tfm': 'net10.0', - 'branch': '10.0', - 'quality': 'daily' + 'branch': '10.0' }, 'nativeaot10.0': { 'tfm': 'nativeaot10.0', diff --git a/scripts/ci_setup.py b/scripts/ci_setup.py index e622a9fe69b..07c0a95733f 100644 --- a/scripts/ci_setup.py +++ b/scripts/ci_setup.py @@ -424,6 +424,11 @@ def main(args: CiSetupArgs): if args.experiment_name == "jitoptrepeat": experiment_config = variable_format % ('DOTNET_JitOptRepeat', '*') + if args.experiment_name == "runtimeasync": + # Surfaced to MSBuild as the $(EnableRuntimeAsync) property; gates the + # runtime-async Features flag in src/Directory.Build.targets. + experiment_config = variable_format % ('EnableRuntimeAsync', 'true') + output = '' with push_dir(get_repo_root_path()): diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 72f006bac80..fc57103c5f1 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -5,6 +5,7 @@ """ import re +import json import datetime from argparse import ArgumentParser, ArgumentTypeError from glob import iglob @@ -862,6 +863,36 @@ def install( common_cmdline_args += ['-AzureFeed', azure_feed_url] common_cmdline_args += ['-FeedCredential', internal_build_key] + # Shield subsequent `dotnet` invocations (e.g. `dotnet --info` in ci_setup.py) + # from picking up an unrelated repo's global.json `paths` entry during the + # SDK resolver's upward walk. In particular, dotnet/runtime's global.json + # contains a `paths` entry that points the resolver at a `.dotnet` directory + # installed by arcade -- which can hold an SDK older than the one the perf + # scripts just installed into install_dir. We rewrite the upstream + # global.json with the `sdk.paths` field removed, preserving all other + # keys (notably `tools.dotnet`, which arcade's bootstrap requires to be + # present), letting the perf-installed SDK win via standard `$host$` + # resolution while keeping unrelated tooling functional. + workspace_root = path.abspath(path.join(get_repo_root_path(), '..')) + shield_global_json = path.join(workspace_root, 'global.json') + try: + if path.exists(shield_global_json): + with open(shield_global_json, 'r') as f: + shield_data = json.load(f) + sdk_section = shield_data.get('sdk') + if isinstance(sdk_section, dict) and 'paths' in sdk_section: + removed_paths = sdk_section.pop('paths') + with open(shield_global_json, 'w') as f: + json.dump(shield_data, f, indent=2) + f.write('\n') + getLogger().info( + "Stripped sdk.paths=%s from %s to shield SDK resolver " + "from an unrelated repo's global.json", + removed_paths, shield_global_json) + except (OSError, ValueError) as ex: + getLogger().warning( + "Could not shield global.json at %s: %s", shield_global_json, ex) + # Install Runtime/SDKs if versions: for version in versions: diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 96647e856eb..b168fcecc65 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,27 +5,27 @@ false - + latest - + False - + $(NoWarn);NU1507 $(NoWarn);NETSDK1138 $(NoWarn);CS9057 True 4 - + True false false - + false diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets new file mode 100644 index 00000000000..63d3b91f501 --- /dev/null +++ b/src/Directory.Build.targets @@ -0,0 +1,11 @@ + + + + + + $(Features);runtime-async=on + + \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml index 8dac83d30a2..17be7e9ab35 100644 --- a/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/Configurations/Run.yaml @@ -12,6 +12,10 @@ coreruns: environment_variables: DOTNET_GCName: clrgc.dll +iterations: + gcperfsim: 1 + microbenchmarks: 1 + trace_configuration_type: gc # Choose between: none, gc, verbose, cpu, cpu_managed, threadtime, join. # Optional fields: the contents of both the symbol_path and the source_path will be copied over to the output path. diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs index 85c8821a8ec..5f0309b21dd 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Analysis.API/Statistics.cs @@ -31,5 +31,38 @@ public static double StandardDeviation(this IEnumerable doubleList) double sumOfDerivationAverage = sumOfDerivation / (doubleList.Count() - 1); return Math.Sqrt(sumOfDerivationAverage - (average * average)); } + + public static IEnumerable RemoveOutliers(IEnumerable collection) + { + List validCollection = new(); + if (!collection.Any()) + { + return Array.Empty(); + } + foreach (double value in collection) + { + if (!double.IsNaN(value) && !double.IsInfinity(value)) + { + validCollection.Add(value); + } + } + if (validCollection.Count == 0) + { + return Array.Empty(); + } + // Calculate Q1 (25th percentile) and Q3 (75th percentile) + double q1 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.25); + double q3 = GC.Analysis.API.Statistics.Percentile(validCollection, 0.75); + + // Calculate IQR (Interquartile Range) + double iqr = q3 - q1; + + // Calculate bounds: [Q1 - 1.5*IQR, Q3 + 1.5*IQR] + double lowerBound = q1 - 1.5 * iqr; + double upperBound = q3 + 1.5 * iqr; + + // Filter out outliers + return GoodLinq.Where(validCollection, x => x >= lowerBound && x <= upperBound); + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs similarity index 53% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs index 2af39abd257..a64f11000a4 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/BdnJsonResult.cs @@ -1,103 +1,10 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.GCPerfSim; -using Newtonsoft.Json; - -namespace GC.Infrastructure.Core.Analysis +namespace GC.Infrastructure.Core.Analysis { - public sealed class MicrobenchmarkResult - { - public Statistics Statistics { get; set; } - - [JsonIgnore] - public GCProcessData? GCData { get; set; } - - public ResultItem ResultItem { get; set; } - - [JsonIgnore] - public CPUProcessData? CPUData { get; set; } - public Run Parent { get; set; } - public string MicrobenchmarkName { get; set; } - public Dictionary OtherMetrics { get; set; } = new(); - - private static readonly IReadOnlyDictionary> _customStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) - { - { "number of iterations", (Statistics stats) => stats.N }, - { "min", (Statistics stats) => stats.Min }, - { "max", (Statistics stats) => stats.Max }, - { "median", (Statistics stats) => stats.Median }, - { "q1", (Statistics stats) => stats.Q1 }, - { "q3", (Statistics stats) => stats.Q3 }, - { "variance", (Statistics stats) => stats.Variance }, - { "standard deviation", (Statistics stats) => stats.StandardDeviation }, - { "skewness", (Statistics stats) => stats.Skewness }, - { "kurtosis", (Statistics stats) => stats.Kurtosis }, - { "standard error", (Statistics stats) => stats.StandardError }, - { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, - }; - - public static double? LookupStatisticsCalculation(string columnName, MicrobenchmarkResult result) - { - if (string.IsNullOrEmpty(columnName)) - { - return null; - } - - if (!_customStatisticsCalculationMap.TryGetValue(columnName, out var val)) - { - return null; - } - - else - { - return val.Invoke(result.Statistics); - } - } - } - - public sealed class Benchmark - { - public string DisplayInfo { get; set; } - public string Namespace { get; set; } - public string Type { get; set; } - public string Method { get; set; } - public string MethodTitle { get; set; } - public string Parameters { get; set; } - public string FullName { get; set; } - public Statistics Statistics { get; set; } - public Memory Memory { get; set; } - public List Measurements { get; set; } - public List Metrics { get; set; } - } - public sealed class ChronometerFrequency { public int Hertz { get; set; } } - public sealed class ConfidenceInterval - { - public int N { get; set; } - public double? Mean { get; set; } - public double? StandardError { get; set; } - public int? Level { get; set; } - public double? Margin { get; set; } - public double? Lower { get; set; } - public double? Upper { get; set; } - } - - public sealed class Descriptor - { - public string Id { get; set; } - public string DisplayName { get; set; } - public string Legend { get; set; } - public string NumberFormat { get; set; } - public int UnitType { get; set; } - public string Unit { get; set; } - public bool TheGreaterTheBetter { get; set; } - public int PriorityInCategory { get; set; } - } - public sealed class HostEnvironmentInfo { public string BenchmarkDotNetCaption { get; set; } @@ -117,6 +24,16 @@ public sealed class HostEnvironmentInfo public string HardwareTimerKind { get; set; } } + + public sealed class Memory + { + public int Gen0Collections { get; set; } + public int Gen1Collections { get; set; } + public int Gen2Collections { get; set; } + public int TotalOperations { get; set; } + public long BytesAllocatedPerOperation { get; set; } + } + public sealed class Measurement { public string IterationMode { get; set; } @@ -127,19 +44,27 @@ public sealed class Measurement public long Nanoseconds { get; set; } } - public sealed class Memory + public sealed class Descriptor { - public int Gen0Collections { get; set; } - public int Gen1Collections { get; set; } - public int Gen2Collections { get; set; } - public int TotalOperations { get; set; } - public long BytesAllocatedPerOperation { get; set; } + public string Id { get; set; } + public string DisplayName { get; set; } + public string Legend { get; set; } + public string NumberFormat { get; set; } + public int UnitType { get; set; } + public string Unit { get; set; } + public bool TheGreaterTheBetter { get; set; } + public int PriorityInCategory { get; set; } } - public sealed class Metric + public sealed class ConfidenceInterval { - public double Value { get; set; } - public Descriptor Descriptor { get; set; } + public int N { get; set; } + public double Mean { get; set; } + public double StandardError { get; set; } + public int Level { get; set; } + public double Margin { get; set; } + public double Lower { get; set; } + public double Upper { get; set; } } public sealed class Percentiles @@ -155,35 +80,56 @@ public sealed class Percentiles public double P100 { get; set; } } - public sealed class MicrobenchmarkResults - { - public string Title { get; set; } - public HostEnvironmentInfo HostEnvironmentInfo { get; set; } - public List Benchmarks { get; set; } - } - public sealed class Statistics { public List OriginalValues { get; set; } public int N { get; set; } - public double? Min { get; set; } - public double? LowerFence { get; set; } - public double? Q1 { get; set; } - public double? Median { get; set; } - public double? Mean { get; set; } - public double? Q3 { get; set; } - public double? UpperFence { get; set; } - public double? Max { get; set; } - public double? InterquartileRange { get; set; } - public List LowerOutliers { get; set; } + public double Min { get; set; } + public double LowerFence { get; set; } + public double Q1 { get; set; } + public double Median { get; set; } + public double Mean { get; set; } + public double Q3 { get; set; } + public double UpperFence { get; set; } + public double Max { get; set; } + public double InterquartileRange { get; set; } + public List LowerOutliers { get; set; } public List UpperOutliers { get; set; } - public List AllOutliers { get; set; } - public double? StandardError { get; set; } - public double? Variance { get; set; } - public double? StandardDeviation { get; set; } - public double? Skewness { get; set; } - public double? Kurtosis { get; set; } + public List AllOutliers { get; set; } + public double StandardError { get; set; } + public double Variance { get; set; } + public double StandardDeviation { get; set; } + public double Skewness { get; set; } + public double Kurtosis { get; set; } public ConfidenceInterval? ConfidenceInterval { get; set; } public Percentiles Percentiles { get; set; } } -} \ No newline at end of file + + public sealed class Metric + { + public double Value { get; set; } + public Descriptor Descriptor { get; set; } + } + + public sealed class Benchmark + { + public string DisplayInfo { get; set; } + public string Namespace { get; set; } + public string Type { get; set; } + public string Method { get; set; } + public string MethodTitle { get; set; } + public string Parameters { get; set; } + public string FullName { get; set; } + public Statistics Statistics { get; set; } + public Memory Memory { get; set; } + public List Measurements { get; set; } + public List Metrics { get; set; } + } + + public sealed class BdnJsonResult + { + public string Title { get; set; } + public HostEnvironmentInfo HostEnvironmentInfo { get; set; } + public List Benchmarks { get; set; } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs new file mode 100644 index 00000000000..3715fb5657a --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparison.cs @@ -0,0 +1,8 @@ +namespace GC.Infrastructure.Core.Analysis +{ + public static class GCTraceMetricComparison + { + public static GCTraceMetricComparisonResult CompareGCTraceMetric(IEnumerable baselines, IEnumerable comparands,string nameOfMetric) + => new GCTraceMetricComparisonResult(baselines, comparands, nameOfMetric); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs new file mode 100644 index 00000000000..9f749a6a0c7 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetricComparisonResult.cs @@ -0,0 +1,109 @@ +using API = GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Analysis.GC; +using System.Reflection; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetricComparisonResult + { + public GCTraceMetricComparisonResult(IEnumerable baselines, IEnumerable comparands, string metricName) + { + RunName = baselines.FirstOrDefault()?.RunName; + Key = $"{baselines.FirstOrDefault()?.ConfigurationName}_{RunName}"; + + MetricName = metricName; + PropertyInfo pInfo = typeof(GCTraceMetrics).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + + // Property found on the GCTraceMetrics. + if (pInfo != null) + { + OriginalBaselineMetricCollection = baselines.Select(baseline => (double)pInfo.GetValue(baseline)); + OriginalComparandMetricCollection = comparands.Select(comparand => (double)pInfo.GetValue(comparand)); + } + + // If property isn't found on the GCTraceMetrics, look in GCStats. + // TODO: Add the case where we look into the map. + else + { + pInfo = typeof(GCStats).GetProperty(metricName, BindingFlags.Instance | BindingFlags.Public); + if (pInfo == null) + { + FieldInfo fieldInfo = typeof(GCStats).GetField(metricName, BindingFlags.Instance | BindingFlags.Public); + if (fieldInfo == null) + { + // Out of luck! + OriginalBaselineMetricCollection = Array.Empty(); + OriginalComparandMetricCollection = Array.Empty(); + OutliersFreeBaselineMetricCollection = Array.Empty(); + OutliersFreeComparandMetricCollection = Array.Empty(); + AveragedBaselineMetric = double.NaN; + AveragedComparandMetric = double.NaN; + return; + } + + else + { + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(fieldInfo.Name)) + .Select(baseline => baseline.StatsData[fieldInfo.Name]); + + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(fieldInfo.Name)) + .Select(comparand => comparand.StatsData[fieldInfo.Name]); + } + } + + else + { + OriginalBaselineMetricCollection = baselines + .Where(baseline => baseline.StatsData.ContainsKey(pInfo.Name)) + .Select(baseline => baseline.StatsData[pInfo.Name]); + OriginalComparandMetricCollection = comparands + .Where(comparand => comparand.StatsData.ContainsKey(pInfo.Name)) + .Select(comparand => comparand.StatsData[pInfo.Name]); + } + } + + // Filter out outliers using IQR method + OutliersFreeBaselineMetricCollection = API.Statistics.RemoveOutliers(OriginalBaselineMetricCollection); + OutliersFreeComparandMetricCollection = API.Statistics.RemoveOutliers(OriginalComparandMetricCollection); + + // Calculate averaged metrics + AveragedBaselineMetric = API.GoodLinq.Average(OutliersFreeBaselineMetricCollection, r => r); + AveragedComparandMetric = API.GoodLinq.Average(OutliersFreeComparandMetricCollection, r => r); + } + + public string RunName { get; } + public string Key { get; } + public string MetricName { get; } + public IEnumerable OriginalBaselineMetricCollection { get; } + public IEnumerable OriginalComparandMetricCollection { get; } + public IEnumerable OutliersFreeBaselineMetricCollection { get; } + public IEnumerable OutliersFreeComparandMetricCollection { get; } + + public double AveragedBaselineMetric { get; } + public double AveragedComparandMetric { get; } + public double Delta => AveragedComparandMetric - AveragedBaselineMetric; + public double PercentageDelta + { + get + { + if (AveragedBaselineMetric == 0) + { + if (AveragedComparandMetric == 0) + { + return 0; + } + else + { + return double.NaN; + } + } + else + { + return Delta / AveragedBaselineMetric * 100.0; + } + } + } + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs new file mode 100644 index 00000000000..d88777bc010 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/GCTraceMetrics.cs @@ -0,0 +1,149 @@ +using GC.Analysis.API; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; + +namespace GC.Infrastructure.Core.Analysis +{ + public sealed class GCTraceMetrics + { + public static GCTraceMetrics GetNullItem(string runName, string corerun) => + new GCTraceMetrics(runName, corerun); + + private GCTraceMetrics(string runName, string corerun) + { + ConfigurationName = corerun; + RunName = runName; + PctTimePausedInGC = double.NaN; + FirstToLastGCSeconds = double.NaN; + HeapSizeBeforeMB_Mean = double.NaN; + HeapSizeAfter_Mean = double.NaN; + TotalCommittedInUse = double.NaN; + TotalBookkeepingCommitted = double.NaN; + TotalCommittedInGlobalDecommit = double.NaN; + TotalCommittedInFree = double.NaN; + TotalCommittedInGlobalFree = double.NaN; + PauseDurationMSec_95PWhereIsGen0 = double.NaN; + PauseDurationMSec_95PWhereIsGen1 = double.NaN; + PauseDurationMSec_95PWhereIsBackground = double.NaN; + PauseDurationMSec_MeanWhereIsBackground = double.NaN; + PauseDurationMSec_95PWhereIsBlockingGen2 = double.NaN; + PauseDurationMSec_MeanWhereIsBlockingGen2 = double.NaN; + CountIsBlockingGen2 = double.NaN; + PauseDurationMSec_SumWhereIsGen1 = double.NaN; + PauseDurationMSec_MeanWhereIsEphemeral = double.NaN; + PromotedMB_MeanWhereIsGen1 = double.NaN; + CountIsGen1 = double.NaN; + CountIsGen0 = double.NaN; + HeapCount = double.NaN; + PauseDurationMSec_Sum = double.NaN; + TotalAllocatedMB = double.NaN; + TotalNumberGCs = double.NaN; + Speed_MBPerMSec = double.NaN; + ExecutionTimeMSec = double.NaN; + } + + public GCTraceMetrics(GCProcessData processData, string runName, string configurationName) + { + RunName = runName; + ConfigurationName = configurationName; + ExecutionTimeMSec = processData.DurationMSec; + + PctTimePausedInGC = processData.Stats.GetGCPauseTimePercentage(); + FirstToLastGCSeconds = (processData.GCs.Last().StartRelativeMSec - processData.GCs.First().StartRelativeMSec) / 1000; + HeapSizeAfter_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeAfterMB)); + HeapSizeBeforeMB_Mean = GoodLinq.Average(processData.GCs, (gc => gc.HeapSizeBeforeMB)); + + TotalCommittedInUse = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInUse)); + TotalBookkeepingCommitted = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalBookkeepingCommitted)); + TotalCommittedInGlobalDecommit = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalDecommit)); + TotalCommittedInFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInFree)); + TotalCommittedInGlobalFree = GoodLinq.Average(processData.GCs, (gc => gc.CommittedUsageBefore.TotalCommittedInGlobalFree)); + + var properties = processData.Stats.GetType().GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var property in properties) + { + if (property.PropertyType != typeof(double) && property.PropertyType != typeof(int)) + { + continue; + } + + string propertyName = property.Name; + object? value = property.GetValue(processData.Stats); + double propertyValue = value != null ? Convert.ToDouble(value) : double.NaN; + StatsData[propertyName] = propertyValue; + } + + var fields = processData.Stats.GetType().GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance); + foreach (var field in fields) + { + if (field.FieldType != typeof(double) && field.FieldType != typeof(int)) + { + continue; + } + + string name = field.Name; + object? value = field.GetValue(processData.Stats); + double doubleValue = value != null ? Convert.ToDouble(value) : double.NaN; + StatsData[name] = doubleValue; + } + + // 95P + PauseDurationMSec_95PWhereIsGen0 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 0)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_95PWhereIsGen1 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)), 0.95); + + PauseDurationMSec_95PWhereIsBackground = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBackground = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type == GCType.BackgroundGC)), (gc => gc.PauseDurationMSec)), (p => p)); + + PauseDurationMSec_95PWhereIsBlockingGen2 = GC.Analysis.API.Statistics.Percentile(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), 0.95); + PauseDurationMSec_MeanWhereIsBlockingGen2 = GoodLinq.Average(GoodLinq.Select(GoodLinq.Where(processData.GCs, (gc => gc.Type != GCType.BackgroundGC && gc.Generation == 2)), (gc => gc.PauseDurationMSec)), (p => p)); + + CountIsBlockingGen2 = processData.GCs.Count(gc => gc.Generation == 2 && gc.Type != GCType.BackgroundGC); + + HeapCount = processData.Stats.HeapCount; + TotalNumberGCs = processData.Stats.Count; + TotalAllocatedMB = processData.Stats.TotalAllocatedMB; + + Speed_MBPerMSec = processData.Stats.TotalPromotedMB / processData.Stats.TotalPauseTimeMSec; + + PauseDurationMSec_MeanWhereIsEphemeral = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1 || gc.Generation == 0)), (gc => gc.PauseDurationMSec)); + PauseDurationMSec_SumWhereIsGen1 = + GoodLinq.Sum(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PauseDurationMSec)); + PromotedMB_MeanWhereIsGen1 = + GoodLinq.Average(GoodLinq.Where(processData.GCs, (gc => gc.Generation == 1)), (gc => gc.PromotedMB)); + PauseDurationMSec_Sum = GoodLinq.Sum(processData.GCs, (gc => gc.PauseDurationMSec)); + CountIsGen1 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 1).Count; + CountIsGen0 = GoodLinq.Where(processData.GCs, gc => gc.Generation == 0).Count; + } + + public double PctTimePausedInGC { get; } + public double FirstToLastGCSeconds { get; } + public double HeapSizeBeforeMB_Mean { get; } + public double HeapSizeAfter_Mean { get; } + public double TotalCommittedInUse { get; set; } + public double TotalCommittedInGlobalDecommit { get; set; } + public double TotalCommittedInFree { get; set; } + public double TotalCommittedInGlobalFree { get; set; } + public double TotalBookkeepingCommitted { get; set; } + public double PauseDurationMSec_95PWhereIsGen0 { get; } + public double PauseDurationMSec_95PWhereIsGen1 { get; } + public double PauseDurationMSec_95PWhereIsBackground { get; } + public double PauseDurationMSec_MeanWhereIsBackground { get; } + public double PauseDurationMSec_95PWhereIsBlockingGen2 { get; } + public double PauseDurationMSec_MeanWhereIsBlockingGen2 { get; } + public double CountIsBlockingGen2 { get; } + public double PauseDurationMSec_SumWhereIsGen1 { get; } + public double PauseDurationMSec_MeanWhereIsEphemeral { get; } + public double PromotedMB_MeanWhereIsGen1 { get; } + public double CountIsGen1 { get; } + public double CountIsGen0 { get; } + public double HeapCount { get; } + public double PauseDurationMSec_Sum { get; } + public double TotalAllocatedMB { get; set; } + public double TotalNumberGCs { get; } + public double Speed_MBPerMSec { get; } + public string RunName { get; } + public string ConfigurationName { get; } + public double ExecutionTimeMSec { get; } + public Dictionary StatsData { get; } = new(); + } +} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs index 60150f4d4c9..760a042e6ff 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResult.cs @@ -1,60 +1,167 @@ -using GC.Infrastructure.Core.Presentation.GCPerfSim; +using API = GC.Analysis.API; namespace GC.Infrastructure.Core.Analysis.Microbenchmarks { // Per Microbenchmark result. public sealed class MicrobenchmarkComparisonResult { - public MicrobenchmarkComparisonResult() { } + public static readonly string[] RequiredMetrics = new string[] + { + "PctTimePausedInGC", + "ExecutionTimeMSec", + "PauseDurationMSec_MeanWhereIsEphemeral", + "PauseDurationMSec_MeanWhereIsBackground", + "PauseDurationMSec_MeanWhereIsBlockingGen2" + }; - public MicrobenchmarkComparisonResult(MicrobenchmarkResult baseline, MicrobenchmarkResult comparand) + public MicrobenchmarkComparisonResult(IEnumerable baselines, IEnumerable comparands, bool includeTraces = true) { - Baseline = baseline; - Comparand = comparand; - var result = new ResultItemComparison(baseline.ResultItem, comparand.ResultItem); ComparisonResults = new(); - ComparisonResults.Add(result.GetComparison("PctTimePausedInGC")); - ComparisonResults.Add(result.GetComparison("ExecutionTimeMSec")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsEphemeral")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBackground")); - ComparisonResults.Add(result.GetComparison("PauseDurationMSec_MeanWhereIsBlockingGen2")); + if (includeTraces) + { + var baselineGCTraceMetricsCollection = baselines + .Where(baseline => baseline != null) + .Where(baseline => baseline.GCTraceMetrics != null) + .Select(baseline => baseline.GCTraceMetrics) + .ToArray(); + + var comparandGCTraceMetricsCollection = comparands + .Where(comparand => comparand != null) + .Where(comparand => comparand.GCTraceMetrics != null) + .Select(comparand => comparand.GCTraceMetrics) + .ToArray(); + + if (baselineGCTraceMetricsCollection.Length > 0 && comparandGCTraceMetricsCollection.Length > 0) + { + foreach (var metricName in RequiredMetrics) + { + ComparisonResults.Add( + GCTraceMetricComparison.CompareGCTraceMetric( + baselineGCTraceMetricsCollection!, comparandGCTraceMetricsCollection!, metricName)); + } + + } + } + + var firstBaseline = baselines?.FirstOrDefault(); + var firstComparand = comparands?.FirstOrDefault(); + + BaselineRunName = firstBaseline?.Parent?.Name ?? string.Empty; + ComparandRunName = firstComparand?.Parent?.Name ?? string.Empty; + MicrobenchmarkName = firstBaseline?.MicrobenchmarkName ?? string.Empty; + + Baselines = baselines ?? new List(); + Comparands = comparands ?? new List(); + + OriginalBaselineOtherMetrics = Baselines + .Select(baseline => baseline.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); + + OriginalComparandOtherMetrics = Comparands + .Select(comparand => comparand.OtherMetrics) + .SelectMany(kvp => kvp) + .GroupBy(kvp => kvp.Key) + .ToDictionary(g => g.Key, g => g.Select(kvp => kvp.Value).ToArray()); } - public MicrobenchmarkResult Baseline { get; set; } - public MicrobenchmarkResult Comparand { get; set; } - public List ComparisonResults { get; set; } + public List ComparisonResults { get; set; } + public string BaselineRunName { get; } + public string ComparandRunName { get; } + public string ComparisonName => $"{ComparandRunName} vs {BaselineRunName}"; + public string MicrobenchmarkName { get; } + public IEnumerable Baselines { get; } + public IEnumerable Comparands { get; } - // TODO: Nullable double check. - public string BaselineRunName => Baseline?.Parent?.Name; - public string ComparandRunName => Comparand?.Parent?.Name; - public string MicrobenchmarkName => Baseline.MicrobenchmarkName; + public double[] OutliersFreeBaselineMeanValueCollection => + API.Statistics.RemoveOutliers(Baselines + .Select(baseline => baseline.Statistics?.Mean ?? double.NaN)) + .ToArray(); + public double[] OutliersFreeComparandMeanValueCollection => + API.Statistics.RemoveOutliers(Comparands + .Select(comparand => comparand.Statistics?.Mean ?? double.NaN)) + .ToArray(); - public double MeanDiff => (Comparand.Statistics?.Mean.Value - Baseline.Statistics?.Mean.Value) ?? double.NaN; - public double MeanDiffPerc => (MeanDiff / Baseline.Statistics?.Mean.Value) * 100 ?? double.NaN; + public double AveragedBaselineMeanValue => API.GoodLinq.Average(OutliersFreeBaselineMeanValueCollection, r => r); + public double AveragedComparandMeanValue => API.GoodLinq.Average(OutliersFreeComparandMeanValueCollection, r => r); - public double? GetDiffPercentFromOtherMetrics(string metric) - { - if (!Baseline.OtherMetrics.TryGetValue(metric, out var baselineMetric)) + public double MeanDiff => AveragedComparandMeanValue - AveragedBaselineMeanValue; + public double MeanDiffPerc{ + get { - return null; + if (AveragedBaselineMeanValue == 0) + { + if (AveragedComparandMeanValue == 0) + { + return 0; + } + else + { + return double.NaN; + } + } + return (MeanDiff / AveragedBaselineMeanValue) * 100; } + } - if (!baselineMetric.HasValue) - { - return null; - } + public Dictionary OriginalBaselineOtherMetrics { get; } = new(); + public Dictionary OriginalComparandOtherMetrics { get; } = new(); + public Dictionary OutliersFreeBaselineOtherMetrics => OriginalBaselineOtherMetrics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(x => x.Item1, x => x.Item2); + public Dictionary OutliersFreeComparandOtherMetrics => OriginalComparandOtherMetrics + .Select(kvp => (kvp.Key, API.Statistics.RemoveOutliers(kvp.Value).ToArray())) + .ToDictionary(x => x.Item1, x => x.Item2); + public Dictionary AveragedBaselineOtherMetrics => OutliersFreeBaselineOtherMetrics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(x => x.Item1, x => x.Item2); + public Dictionary AveragedComparandOtherMetrics => OutliersFreeComparandOtherMetrics + .Select(kvp => (kvp.Key, API.GoodLinq.Average(kvp.Value, v => v))) + .ToDictionary(x => x.Item1, x => x.Item2); - if (!Comparand.OtherMetrics.TryGetValue(metric, out var comparandMetric)) + public Dictionary OtherMetricsDiff => AveragedBaselineOtherMetrics + .Select(kvp => { - return null; - } + if (AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, AveragedComparandOtherMetrics[kvp.Key] - AveragedBaselineOtherMetrics[kvp.Key]); + } + return (kvp.Key, double.NaN); + }) + .ToDictionary(x => x.Item1, x => x.Item2); - if (!comparandMetric.HasValue) + public Dictionary OtherMetricsDiffPerc => AveragedBaselineOtherMetrics + .Select(kvp => { - return null; - } + if (!AveragedComparandOtherMetrics.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } + if (AveragedBaselineOtherMetrics[kvp.Key] == 0) + { + if (AveragedComparandOtherMetrics[kvp.Key] == 0) + { + return (kvp.Key, 0); + } + else + { + return (kvp.Key, double.NaN); + } + } - return (comparandMetric.Value - baselineMetric.Value) / baselineMetric.Value; - } + if (!OtherMetricsDiff.ContainsKey(kvp.Key)) + { + return (kvp.Key, double.NaN); + } + + if (OtherMetricsDiff[kvp.Key] == double.NaN) + { + return (kvp.Key, double.NaN); + } + + return (kvp.Key, 100 * OtherMetricsDiff[kvp.Key] / AveragedBaselineOtherMetrics[kvp.Key]); + }) + .ToDictionary(x => x.Item1, x => x.Item2); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs index 79d2f8eccee..cbba03262ec 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkComparisonResults.cs @@ -17,11 +17,11 @@ public MicrobenchmarkComparisonResults(string baselineName, string runName, IEnu public IEnumerable Comparisons { get; } public IEnumerable Ordered => Comparisons.Where(o => !double.IsNaN(o.MeanDiff)).OrderByDescending(m => m.MeanDiffPerc); - public IEnumerable LargeRegressions => Ordered.Where(o => o.MeanDiffPerc > 20); - public IEnumerable LargeImprovements => Ordered.Where(o => o.MeanDiffPerc < -20).OrderBy(g => g.MeanDiffPerc); - public IEnumerable Regressions => Ordered.Where(o => o.MeanDiffPerc < 20 && o.MeanDiffPerc > 5); - public IEnumerable Improvements => Ordered.Where(o => o.MeanDiffPerc > -20 && o.MeanDiffPerc < -5).OrderBy(g => g.MeanDiffPerc); - public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)).OrderByDescending(g => g.MeanDiffPerc); - public IEnumerable StaleImprovements => Ordered.Where((o => o.MeanDiffPerc < 0 && o.MeanDiffPerc > -5)).OrderBy(g => g.MeanDiffPerc); + public IEnumerable LargeRegressions => Ordered.Where(o => o.MeanDiffPerc >= 20); + public IEnumerable LargeImprovements => Ordered.Where(o => o.MeanDiffPerc <= -20).OrderBy(g => g.MeanDiffPerc); + public IEnumerable Regressions => Ordered.Where(o => o.MeanDiffPerc < 20 && o.MeanDiffPerc >= 5); + public IEnumerable Improvements => Ordered.Where(o => o.MeanDiffPerc > -20 && o.MeanDiffPerc <= -5).OrderBy(g => g.MeanDiffPerc); + public IEnumerable StaleRegressions => Ordered.Where((o => o.MeanDiffPerc > 0 && o.MeanDiffPerc < 5)); + public IEnumerable StaleImprovements => Ordered.Where((o => o.MeanDiffPerc <= 0 && o.MeanDiffPerc > -5)).OrderBy(g => g.MeanDiffPerc); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs new file mode 100644 index 00000000000..f5fb0f898b3 --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResult.cs @@ -0,0 +1,88 @@ +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Microsoft.Diagnostics.Tracing.Parsers.Clr; +using API = GC.Analysis.API; + +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks +{ + public sealed class MicrobenchmarkResult + { + public static readonly IReadOnlyDictionary> CustomStatisticsCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "number of iterations", (Statistics stats) => stats.N }, + { "min", (Statistics stats) => stats.Min }, + { "max", (Statistics stats) => stats.Max }, + { "median", (Statistics stats) => stats.Median }, + { "q1", (Statistics stats) => stats.Q1 }, + { "q3", (Statistics stats) => stats.Q3 }, + { "variance", (Statistics stats) => stats.Variance }, + { "standard deviation", (Statistics stats) => stats.StandardDeviation }, + { "skewness", (Statistics stats) => stats.Skewness }, + { "kurtosis", (Statistics stats) => stats.Kurtosis }, + { "standard error", (Statistics stats) => stats.StandardError }, + { "standard error / mean", (Statistics stats) => stats.StandardError / stats.Mean }, + }; + + public static readonly Dictionary> CustomAggregateCalculationMap = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + { "gc count", (gc) => gc.Stats.Count }, + { "non induced gc count", (gc) => gc.Stats.Count - gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "induced gc count", (gc) => gc.GCs.Count(g => g.Reason == GCReason.Induced)}, + { "total allocated (mb)", (gc) => gc.Stats.TotalAllocatedMB }, + { "max size peak (mb)", (gc) => gc.Stats.MaxSizePeakMB }, + { "total pause time (msec)", (gc) => gc.Stats.TotalPauseTimeMSec }, + { "gc pause time %", (gc) => gc.Stats.GetGCPauseTimePercentage() }, + { "avg. heap size (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeBeforeMB) }, + { "avg. heap size after (mb)", (gc) => API.GoodLinq.Average(gc.GCs, g => g.HeapSizeAfterMB) }, + }; + + public MicrobenchmarkResult(string benchmarkFullName, + Run parent, + Benchmark benchmark, + API.GCProcessData? gcData = null, + GCTraceMetrics? gcTraceMetrics = null, + API.CPUProcessData? cpuData = null, + IEnumerable? additionalReportMetrics = null, + IEnumerable? columns = null, + IEnumerable? cpuColumns = null) + { + MicrobenchmarkName = benchmarkFullName; + Parent = parent; + Statistics = benchmark.Statistics; + GCTraceMetrics = gcTraceMetrics; + CPUData = cpuData; + + if (additionalReportMetrics != null) + { + OtherMetrics = benchmark.Metrics + .Where(metric => additionalReportMetrics.Contains(metric.Descriptor.Id)) + .ToDictionary(metric => metric.Descriptor.Id, metric => metric.Value); + } + + if (columns != null) + { + var customStatistics = columns + .Where(column => CustomStatisticsCalculationMap.Keys.Contains(column)) + .Select(column => (column, CustomStatisticsCalculationMap[column](benchmark.Statistics))) + .ToDictionary(x => x.column, x => x.Item2); + + OtherMetrics = OtherMetrics.Concat(customStatistics).ToDictionary(x => x.Key, x => x.Value); + + if (gcData != null) + { + var customGCData = columns + .Where(column => CustomAggregateCalculationMap.Keys.Contains(column)) + .Select(column => (column, CustomAggregateCalculationMap[column](gcData))) + .ToDictionary(x => x.column, x => x.Item2); + + OtherMetrics = OtherMetrics.Concat(customGCData).ToDictionary(x => x.Key, x => x.Value); + } + } + } + public string MicrobenchmarkName { get; set; } + public Run Parent { get; set; } + public Statistics Statistics { get; set; } + public GCTraceMetrics? GCTraceMetrics { get; set; } + public Dictionary OtherMetrics { get; set; } = new(); + public API.CPUProcessData? CPUData { get; set; } + } +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs new file mode 100644 index 00000000000..061a68db2eb --- /dev/null +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultComparison.cs @@ -0,0 +1,345 @@ +using GC.Analysis.API; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using Newtonsoft.Json; +using System.Collections.Concurrent; +using System.Text.RegularExpressions; + +namespace GC.Infrastructure.Core.Analysis.Microbenchmarks +{ + public static class MicrobenchmarkResultComparison + { + private static readonly int _CPUCount = System.Environment.ProcessorCount; + private static readonly Dictionary _benchmarkNameToTraceFilePatternMap = new() + { + { "ByteMark.BenchBitOps", "ByteMark.BenchBitOps"}, + { "System.Collections.CtorGivenSize.Array(Size: 512)", "System.Collections.CtorGivenSize_String_.Array_size_512_"}, + { "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor(Size: 512)", "System.Collections.Tests.Perf_BitArray.BitArrayByteArrayCtor_size_512_"}, + { "System.IO.Tests.Perf_File.ReadAllBytes(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytes_size_104857600_"}, + { "System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)", "System.IO.Tests.Perf_File.ReadAllBytesAsync_size_104857600_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: ICollection)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Linq.Tests.Perf_Enumerable.ToArray(input: IEnumerable)", "System.Linq.Tests.Perf_Enumerable.ToArray_"}, + { "System.Numerics.Tests.Perf_BigInteger.Add(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Add_arguments_65536_"}, + { "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536,65536 bits)", "System.Numerics.Tests.Perf_BigInteger.Subtract_arguments_65536_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Byte_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Byte_.NewOperator_Array_length_10000_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 1000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_1000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: False)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.AllocateUninitializedArray(length: 10000, pinned: True)", "System.Tests.Perf_GC_Char_.AllocateUninitializedArray_length_10000,_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 1000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_1000_"}, + { "System.Tests.Perf_GC.NewOperator_Array(length: 10000)", "System.Tests.Perf_GC_Char_.NewOperator_Array_length_10000_"}, + }; + + public static ConcurrentBag> LoadBdnJsonResults(MicrobenchmarkConfiguration configuration) + { + ConcurrentBag> bdnJsonResults = new(); + Parallel.ForEach(configuration.Runs, (run) => + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); + string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); + Parallel.ForEach(jsonFiles, jsonPath => + { + run.Value.Name ??= run.Key; + BdnJsonResult? results = JsonConvert.DeserializeObject(File.ReadAllText(jsonPath)); + if (results != null) + { + bdnJsonResults.Add(new(run.Value, results, jsonPath)); + } + }); + }); + return bdnJsonResults; + } + + // TODO: We should specify relationship between json files and trace files before running benchmarks instead of relying on file name patterns. + // This will make the mapping more robust and less prone to errors due to file naming. + public static Dictionary MapJsonToTrace(string outputPath, ConcurrentBag> bdnJsonResults) + { + Dictionary jsonToTrace = new(); + foreach (var groupForRun in bdnJsonResults.GroupBy(t => t.Item1)) + { + var run = groupForRun.Key; + + // GroupBy(t => t.Item2.Benchmarks.First().FullName) is not a bug: + // In single *full.json, multiple benchmarks stands for multiple input parameter combinations for the same benchmark + // If trace collection is enabled, process data for all those parameter combinations will be in the same trace file + foreach (var g in groupForRun.GroupBy(t => t.Item2.Benchmarks.First().FullName)) + { + var benchmarkName = g.Key; + var sortedJsonFiles = GoodLinq.Select(g, t => t.Item3) + .OrderBy(jsonFile => Path.GetFileName(Path.GetDirectoryName(jsonFile))) + .ToArray(); + + if (!_benchmarkNameToTraceFilePatternMap.ContainsKey(benchmarkName)) + { + throw new InvalidOperationException($"Benchmark name {benchmarkName} does not have a corresponding trace file pattern in the map."); + } + var traceFileNameTemplate = _benchmarkNameToTraceFilePatternMap[benchmarkName]; + string outputPathForRun = Path.Combine(outputPath, run.Name!); + var sortedTraceFiles = Directory.GetFiles(outputPathForRun, $"{traceFileNameTemplate}*.etl.zip", SearchOption.TopDirectoryOnly) + .OrderBy(traceFile => + { + var match = Regex.Match(Path.GetFileName(traceFile), @"_(\d+)\.etl\.zip$"); + return match.Success ? int.Parse(match.Groups[1].Value) : 0; + }) + .ToArray(); + + if (sortedJsonFiles.Length != sortedTraceFiles.Length) + { + throw new InvalidOperationException( + $"The number of JSON files ({sortedJsonFiles.Length}) does not match the number of trace files ({sortedTraceFiles.Length}) for benchmark: {benchmarkName}"); + } + + for (int i = 0; i < sortedJsonFiles.Length; i++) + { + jsonToTrace[sortedJsonFiles[i]] = sortedTraceFiles[i]; + } + } + } + + return jsonToTrace; + } + + public static ConcurrentBag + AnalyzeMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, + ConcurrentBag> bdnJsonResults, + bool excludeTraces = false) + { + ConcurrentBag microbenchmarkResults = new(); + + Dictionary jsonToTraceMap = new(); + if ((!excludeTraces) && (configuration.TraceConfigurations?.Type ?? "none") != "none") + { + jsonToTraceMap = MapJsonToTrace(configuration.Output.Path, bdnJsonResults); + } + + ParallelOptions options = new() + { + MaxDegreeOfParallelism = _CPUCount * 2 + }; + + int count = 0; + object _lock = new(); + + Parallel.ForEach(bdnJsonResults, options, t => + { + var run = t.Item1; + var bdnJsonResult = t.Item2; + var jsonPath = t.Item3; + + List? benchmarks = bdnJsonResult?.Benchmarks; + + if (benchmarks == null) + { + return; + } + + if ((!excludeTraces) && configuration.TraceConfigurations?.Type != "none") + { + string outputPathForRun = Path.Combine(configuration.Output.Path, run.Name!); + + if (!jsonToTraceMap.TryGetValue(jsonPath, out string? tracePath) || string.IsNullOrWhiteSpace(tracePath)) + { + throw new InvalidOperationException($"Trace collection is enabled, but no trace path mapping was found for benchmark result '{jsonPath}'."); + } + using (var analyzer = AnalyzerManager.GetAnalyzer(tracePath)) + { + List allPertinentProcesses = analyzer.GetProcessGCData("dotnet"); + List corerunProcesses = analyzer.GetProcessGCData("corerun"); + allPertinentProcesses.AddRange(corerunProcesses); + + foreach (var benchmark in benchmarks) + { + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + + MicrobenchmarkResult? microbenchmarkResult = null; + GCProcessData? benchmarkGCData = null; + foreach (var process in allPertinentProcesses) + { + string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); + string runCleaned = benchmark.FullName.Replace("\"", "").Replace("\\", ""); + if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) + { + benchmarkGCData = process; + break; + } + } + if (benchmarkGCData != null) + { + int processID = benchmarkGCData.ProcessID; + + /* + TODO: THIS NEEDS TO BE ADDED BACK. + if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) + { + // TODO: Add parameterize. + benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", + symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), + symbolPath: Path.Combine(configuration.Output.Path, run.Key)); + var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); + d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); + benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); + } + */ + + if (benchmarkGCData.GCs.Count > 0) + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + gcData: benchmarkGCData, + gcTraceMetrics: new GCTraceMetrics(benchmarkGCData, run.Name!, benchmark.FullName), + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + } + else + { + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + } + + microbenchmarkResults.Add(microbenchmarkResult!); + } + } + } + else + { + foreach (var benchmark in benchmarks) + { + Statistics statistics = benchmark.Statistics; + var benchmarkFullName = benchmark.FullName; + + MicrobenchmarkResult? microbenchmarkResult = null; + microbenchmarkResult = new(benchmarkFullName, + run, + benchmark, + additionalReportMetrics: configuration.Output.additional_report_metrics, + cpuColumns: configuration.Output.cpu_columns, + columns: configuration.Output.Columns); + microbenchmarkResults.Add(microbenchmarkResult!); + } + } + + lock (_lock) + { + count = count + 1; + Console.Write($"\r{count}/{bdnJsonResults.Count} BDN results analyzed."); + } + }); + + Console.WriteLine(); + return microbenchmarkResults; + } + + public static List CompareMicrobenchmarkResults(MicrobenchmarkConfiguration configuration, IEnumerable microbenchmarkResults, bool excludeTraces = false) + { + bool includeTraces = (!excludeTraces) && (configuration.TraceConfigurations.Type != "none"); + var microbenchmarkResultsGroupedByBenchmarkName = microbenchmarkResults + .GroupBy(microbenchmarkResult => microbenchmarkResult.MicrobenchmarkName); + + List comparisonResults = new(); + object _lock = new(); + ParallelOptions options = new() + { + MaxDegreeOfParallelism = _CPUCount + }; + Parallel.ForEach(microbenchmarkResultsGroupedByBenchmarkName, options, microbenchmarkResultsGroup => + { + if (configuration.Output.run_comparisons != null) + { + foreach (var comparison in configuration.Output.run_comparisons) + { + string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); + string baselineName = breakup[0]; + string runName = breakup[1]; + + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == baselineName); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.Name == runName); + + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults ==null) + { + continue; + } + + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + continue; + } + + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } + } + + // Default case where the run comparisons aren't specified. + else + { + var baselineMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => r.Parent.is_baseline); + var comparandMicrobenchmarkResults = GoodLinq.Where(microbenchmarkResultsGroup, r => !r.Parent.is_baseline); + + if (baselineMicrobenchmarkResults == null || comparandMicrobenchmarkResults == null) + { + return; + } + + if (baselineMicrobenchmarkResults.Count == 0 || comparandMicrobenchmarkResults.Count == 0) + { + return; + } + lock (_lock) + { + comparisonResults.Add(new(baselineMicrobenchmarkResults, comparandMicrobenchmarkResults, includeTraces)); + } + } + }); + + return comparisonResults; + } + + public static List GroupComparisonResultsByName(MicrobenchmarkConfiguration configuration, List comparisonResultForAllBenchmarks, bool excludeTraces = false) + { + List allComparisonResults = new(); + + comparisonResultForAllBenchmarks + .GroupBy(r => r.ComparisonName) + .ToList() + .ForEach(group => + { + string baselineName = group.FirstOrDefault()?.BaselineRunName ?? "Baseline"; + string runName = group.FirstOrDefault()?.ComparandRunName ?? "Comparand"; + allComparisonResults.Add(new(baselineName, runName, group.ToList())); + }); + + return allComparisonResults; + } + } +} \ No newline at end of file diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs deleted file mode 100644 index 4ad9158e74c..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Analysis/Microbenchmarks/MicrobenchmarkResultsAnalyzer.cs +++ /dev/null @@ -1,178 +0,0 @@ -using GC.Analysis.API; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; -using Newtonsoft.Json; -using System.Collections.Concurrent; - -namespace GC.Infrastructure.Core.Analysis.Microbenchmarks -{ - public static class MicrobenchmarkResultsAnalyzer - { - public static IReadOnlyDictionary> Analyze(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - ConcurrentDictionary> runsToResults = new(); - - Parallel.ForEach(configuration.Runs, (run) => - { - string outputPathForRun = Path.Combine(configuration.Output.Path, run.Key); - run.Value.Name ??= run.Key; - - // Find the json path. - string[] jsonFiles = Directory.GetFiles(outputPathForRun, "*full.json", SearchOption.AllDirectories); - - // Retrieve benchmarks from all the JSON files. - Parallel.ForEach(jsonFiles, (jsonFile) => - { - MicrobenchmarkResults results = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); - foreach (var benchmark in results?.Benchmarks) - { - string title = benchmark.FullName; - Statistics statistics = benchmark.Statistics; - - if (!runsToResults.TryGetValue(run.Value, out var perBenchmarkData)) - { - runsToResults[run.Value] = perBenchmarkData = new ConcurrentDictionary(); - } - - runsToResults[run.Value][title] = new MicrobenchmarkResult - { - Statistics = statistics, - Parent = run.Value, - MicrobenchmarkName = title, - }; - } - }); - - if (!excludeTraces) - { - Dictionary analyzers = AnalyzerManager.GetAllAnalyzers(outputPathForRun); - - foreach (var analyzer in analyzers) - { - List allPertinentProcesses = analyzer.Value.GetProcessGCData("dotnet"); - List corerunProcesses = analyzer.Value.GetProcessGCData("corerun"); - allPertinentProcesses.AddRange(corerunProcesses); - foreach (var benchmark in runsToResults[run.Value]) - { - GCProcessData? benchmarkGCData = null; - foreach (var process in allPertinentProcesses) - { - string commandLine = process.CommandLine.Replace("\"", "").Replace("\\", ""); - string runCleaned = benchmark.Key.Replace("\"", "").Replace("\\", ""); - if (commandLine.Contains(runCleaned) && commandLine.Contains("--benchmarkName")) - { - benchmarkGCData = process; - break; - } - } - - if (benchmarkGCData != null) - { - int processID = benchmarkGCData.ProcessID; - benchmark.Value.GCData = benchmarkGCData; - benchmark.Value.ResultItem = new Presentation.GCPerfSim.ResultItem(benchmarkGCData, analyzer.Key, benchmark.Key); - /* - TODO: THIS NEEDS TO BE ADDED BACK. - if (configuration.Output.cpu_columns != null && configuration.Output.cpu_columns.Count > 0) - { - // TODO: Add parameterize. - benchmark.Value.GCData.Parent.AddCPUAnalysis(yamlPath: @"C:\Users\musharm\source\repos\GC.Analysis.API\GC.Analysis.API\CPUAnalysis\DefaultMethods.yaml", - symbolLogFile: Path.Combine(configuration.Output.Path, run.Key, Guid.NewGuid() + ".txt"), - symbolPath: Path.Combine(configuration.Output.Path, run.Key)); - var d1 = benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("dotnet"); - d1.AddRange(benchmark.Value.GCData.Parent.CPUAnalyzer.GetCPUDataForProcessName("corerun")); - benchmark.Value.CPUData = d1.FirstOrDefault(p => p.ProcessID == processID); - } - */ - } - } - }; - } - }); - - return runsToResults; - } - - public static IReadOnlyList GetComparisons(MicrobenchmarkConfiguration configuration, bool excludeTraces = false) - { - IReadOnlyDictionary> runResults = Analyze(configuration, excludeTraces); - List comparisonResults = new(); - - if (configuration.Output.run_comparisons != null) - { - foreach (var comparison in configuration.Output.run_comparisons) - { - string[] breakup = comparison.Split(",", StringSplitOptions.TrimEntries); - string baselineName = breakup[0]; - string runName = breakup[1]; - - Run run = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, runName) == 0); - Run baselineRun = runResults.Keys.FirstOrDefault(k => string.CompareOrdinal(k.Name, baselineName) == 0); - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResults[run]) - { - string microbenchmarkName = r.Key; - if (runResults[baselineRun].TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - } - } - - // At this point of time, the lack thereof of either of the runs should be a non-issue. - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - // Default case where the run comparisons aren't specified. - else - { - string baselineName = configuration.Runs.FirstOrDefault(r => r.Value.is_baseline).Key; - KeyValuePair> baselineResult = baselineName != null ? runResults.First(r => r.Key.Name == baselineName) : runResults.First(); - - // For each run, we want to grab it and it's baseline and then do a per microbenchmark association. - foreach (var runResult in runResults) - { - Run run = runResult.Key; - string runName = run.Name; - - if (string.CompareOrdinal(runName, baselineName) == 0) - { - continue; - } - - List microbenchmarkResults = new(); - - // Go through all the microbenchmarks for the current run and find the corresponding runs in the baseline. - foreach (var r in runResult.Value) - { - string microbenchmarkName = r.Key; - if (baselineResult.Value.TryGetValue(microbenchmarkName, out var m)) - { - MicrobenchmarkComparisonResult microbenchmarkResult = new(m, r.Value); - microbenchmarkResults.Add(microbenchmarkResult); - } - - else - { - Console.WriteLine($"Microbenchmark: {microbenchmarkName} isn't found on the baseline: {baselineName} for run: {runName}"); - // TODO: Log the fact that we haven't found a corresponding result in the baseline. - } - } - - comparisonResults.Add(new MicrobenchmarkComparisonResults(baselineName, runName, microbenchmarkResults)); - } - } - - return comparisonResults; - } - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs index fb2546f9bb7..b0ca7613bc6 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/InputConfiguration.cs @@ -13,6 +13,8 @@ public sealed class InputConfiguration // public Dictionary? clrgcs { get; set; } // public string? debug_parameters { get; set; } + public Dictionary? iterations { get; set; } + public Dictionary? symbol_path { get; set; } public Dictionary? source_path { get; set; } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs index c44e02cd947..ab9ab01a911 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Configurations/Microbenchmarks.Configuration.cs @@ -24,7 +24,13 @@ public sealed class Run : RunBase public class Environment { public uint default_max_seconds { get; set; } = 300; - public uint iteration { get; set; } = 1; + public uint iterations { get; set; } = 1; + + [YamlMember(Alias = "iteration")] + public uint iteration + { + set => iterations = value; + } } public sealed class MicrobenchmarkConfigurations diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs index ce0415c70e0..61cfd5dcadf 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/MarkdownReportBuilder.cs @@ -103,10 +103,12 @@ public static void AddReproSection(this StreamWriter sw, Dictionary kvp.Value.CommandlineArgs); + foreach (var kvp in distinctCommandlineArgs) { - sw.WriteLine($"### {p.Key}"); - sw.WriteLine($"```{p.Value.CommandlineArgs}```\n"); + sw.WriteLine($"### {kvp.Key}"); + sw.WriteLine($"```{kvp.Value.CommandlineArgs}```\n"); } sw.WriteLine(); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs similarity index 50% rename from src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs rename to src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs index 3d547c03a0c..9bf2736ef0a 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/Json.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json.cs @@ -1,14 +1,16 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.GCPerfSim; using Newtonsoft.Json; -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json +namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Json { - public static void Generate(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, string path) + public static void Generate(MicrobenchmarkConfiguration configuration, List comparisonResultsGroupedByName, string path) { - string json = JsonConvert.SerializeObject(comparisonResults); + string json = JsonConvert.SerializeObject(comparisonResultsGroupedByName); File.WriteAllText(path, json); } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs deleted file mode 100644 index 9840432b2ba..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Json/JsonOutput.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks.Json -{ - public sealed class JsonOutput - { - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs index 76dffd7f87d..7636f46f314 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Markdown.cs @@ -1,33 +1,29 @@ -using API = GC.Analysis.API; -using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Analysis.API; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using API = GC.Analysis.API; namespace GC.Infrastructure.Core.Presentation.Microbenchmarks { public static class Markdown { - private const string baseTableString = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; - private const string baseTableRows = "| --- | --- | -- | --- | --- | --- | --- | "; - - public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResults, Dictionary executionDetails, string path) + public static void GenerateTable(MicrobenchmarkConfiguration configuration, IReadOnlyList comparisonResultsCollection, Dictionary executionDetails, string path) { using (StreamWriter sw = new StreamWriter(path)) { // Create summary. sw.WriteLine("# Summary"); - string header = $"| Criteria | {string.Join("|", GoodLinq.Select(comparisonResults, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; + string header = $"| Criteria | {string.Join("|", API.GoodLinq.Select(comparisonResultsCollection, s => $"[{s.BaselineName} {s.RunName}]({s.MarkdownIdentifier})"))}|"; sw.WriteLine(header); - sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResults.Count))} |"); - sw.WriteLine($"| Large Regressions (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeRegressions.Count())}|"); - sw.WriteLine($"| Regressions (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Regressions.Count())}|"); - sw.WriteLine($"| Stale Regressions (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleRegressions.Count())}|"); - sw.WriteLine($"| Stale Improvements (0% - 5%) | {GoodLinq.Sum(comparisonResults, s => s.StaleImprovements.Count())}|"); - sw.WriteLine($"| Improvements (5% - 20%) | {GoodLinq.Sum(comparisonResults, s => s.Improvements.Count())}|"); - sw.WriteLine($"| Large Improvements (>20%) | {GoodLinq.Sum(comparisonResults, s => s.LargeImprovements.Count())}|"); - sw.WriteLine($"| Total | {comparisonResults.Count} |"); + sw.WriteLine($"| ----- | {string.Join("|", Enumerable.Repeat(" ----- ", comparisonResultsCollection.Count))} |"); + sw.WriteLine($"| Large Regressions (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeRegressions.Count())}|"); + sw.WriteLine($"| Regressions (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Regressions.Count())}|"); + sw.WriteLine($"| Stale Regressions (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleRegressions.Count())}|"); + sw.WriteLine($"| Stale Improvements (0% - 5%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.StaleImprovements.Count())}|"); + sw.WriteLine($"| Improvements (5% - 20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.Improvements.Count())}|"); + sw.WriteLine($"| Large Improvements (>20%) | {API.GoodLinq.Sum(comparisonResultsCollection, s => s.LargeImprovements.Count())}|"); + sw.WriteLine($"| Total | {comparisonResultsCollection.Count} |"); sw.WriteLine("\n"); // Incomplete Tests. @@ -41,9 +37,9 @@ public static void GenerateTable(MicrobenchmarkConfiguration configuration, IRea sw.WriteLine("## Individual Results"); // Add details of Each Comparison. - foreach (var comparisonResult in comparisonResults) + foreach (var comparisonResults in comparisonResultsCollection) { - AddDetailsOfSingleComparison(sw, configuration, comparisonResult); + AddDetailsOfSingleComparison(sw, configuration, comparisonResults); sw.WriteLine("\n"); } } @@ -55,12 +51,12 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben sw.WriteLine("\n"); // Large Regressions - sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); + sw.WriteLine($"### Large Regressions (>=20%): {comparisonResult.LargeRegressions.Count()} \n"); sw.AddTableForSingleCriteria(configuration, comparisonResult.LargeRegressions); sw.WriteLine("\n"); // Large Improvements - sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); + sw.WriteLine($"### Large Improvements (>=20%): {comparisonResult.LargeImprovements.Count()} \n"); sw.AddTableForSingleCriteria(configuration, comparisonResult.LargeImprovements); sw.WriteLine("\n"); @@ -75,7 +71,7 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben sw.WriteLine("\n"); // Stale Regressions - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); + sw.WriteLine($"### Stale Regressions (Percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); sw.AddTableForSingleCriteria(configuration, comparisonResult.StaleRegressions); sw.WriteLine("\n"); @@ -89,52 +85,65 @@ internal static void AddDetailsOfSingleComparison(this StreamWriter sw, Microben foreach (var metric in configuration.Output.additional_report_metrics) { sw.WriteLine($"## Comparison by {metric}"); - var ordered = comparisonResult.Comparisons.OrderByDescending(c => c.GetDiffPercentFromOtherMetrics(metric)); + var ordered = comparisonResult.Comparisons + .Where(c => c.OtherMetricsDiffPerc.ContainsKey(metric)) + .OrderByDescending(c => c.OtherMetricsDiffPerc[metric]); // Large Regressions - sw.WriteLine($"### Large Regressions (>20%): {comparisonResult.LargeRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.2)); + var largeRegression = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 20); + sw.WriteLine($"### Large Regressions (>20%): {largeRegression.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeRegression, metric); sw.WriteLine("\n"); // Large Improvements - sw.WriteLine($"### Large Improvements (>20%): {comparisonResult.LargeImprovements.Count()} \n"); - var largeImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < -0.2); + var largeImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -20); largeImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, largeImprovements); + sw.WriteLine($"### Large Improvements (>20%): {largeImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, largeImprovements, metric); sw.WriteLine("\n"); // Regressions - sw.WriteLine($"### Regressions (5% - 20%): {comparisonResult.Regressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2)); + var regressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] >= 5 && o.OtherMetricsDiffPerc[metric] < 20); + sw.WriteLine($"### Regressions (5% - 20%): {regressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, regressions, metric); sw.WriteLine("\n"); // Improvements - sw.WriteLine($"### Improvements (5% - 20%): {comparisonResult.Improvements.Count()} \n"); - var improvements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > 0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.2); + var improvements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] <= -5 && o.OtherMetricsDiffPerc[metric] > -20); improvements.Reverse(); - sw.AddTableForSingleCriteria(configuration, improvements); + sw.WriteLine($"### Improvements (5% - 20%): {improvements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, improvements, metric); sw.WriteLine("\n"); // Stale Regressions - sw.WriteLine($"### Stale Regressions (Same or percent difference within 5% margin): {comparisonResult.StaleRegressions.Count()} \n"); - sw.AddTableForSingleCriteria(configuration, GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) < 0.05 && o.GetDiffPercentFromOtherMetrics(metric) >= 0.0)); + var staleRegressions = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > 0.0 && o.OtherMetricsDiffPerc[metric] < 5); + sw.WriteLine($"### Stale Regressions (Percent difference within 5% margin): {staleRegressions.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleRegressions, metric); sw.WriteLine("\n"); // Stale Improvements - sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {comparisonResult.StaleImprovements.Count()} \n"); - var staleImprovements = GoodLinq.Where(ordered, o => o.GetDiffPercentFromOtherMetrics(metric) > -0.05 && o.GetDiffPercentFromOtherMetrics(metric) < 0.0); + var staleImprovements = API.GoodLinq.Where(ordered, o => o.OtherMetricsDiffPerc[metric] > -5 && o.OtherMetricsDiffPerc[metric] <= 0.0); staleImprovements.Reverse(); - sw.AddTableForSingleCriteria(configuration, staleImprovements); + sw.WriteLine($"### Stale Improvements (Same or percent difference within 5% margin): {staleImprovements.Count()} \n"); + sw.AddTableForSingleCriteria(configuration, staleImprovements, metric); sw.WriteLine("\n"); } } } - internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons) + internal static void AddTableForSingleCriteria(this StreamWriter sw, MicrobenchmarkConfiguration configuration, IEnumerable comparisons, string? metricName = null) { // Check if all comparisons have traces. - string tableHeader0 = baseTableString; - string tableHeader1 = baseTableRows; + string tableHeader0 = ""; + if (!string.IsNullOrEmpty(metricName)) + { + tableHeader0 = $"| Benchmark Name | Baseline | Comparand | Baseline {metricName} | Comparand {metricName} | Δ {metricName} | Δ% {metricName} |"; + } + else + { + tableHeader0 = "| Benchmark Name | Baseline | Comparand | Baseline Mean Duration (MSec) | Comparand Mean Duration (MSec) | Δ Mean Duration (MSec) | Δ% Mean Duration |"; + } + string tableHeader1 = "| --- | --- | -- | --- | --- | --- | --- | "; if (configuration.Output.Columns != null) { @@ -145,14 +154,15 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; - tableHeader1 += "--- | --- | --- | --- |"; - } - } + // TODO: Add CPU columns if needed in the future. + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // tableHeader0 += $"Baseline {column} | Comparand {column} | Δ {column} | Δ% {column} |"; + // tableHeader1 += "--- | --- | --- | --- |"; + // } + //} sw.WriteLine(tableHeader0); sw.WriteLine(tableHeader1); @@ -161,24 +171,27 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm { try { - var baseRow = $"| {lr.MicrobenchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.Baseline.Statistics.Mean.Value, 2)} | {Math.Round(lr.Comparand.Statistics.Mean.Value, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + string benchmarkName = lr.MicrobenchmarkName.Replace("<", "\\<").Replace(">", "\\>"); + string baseRow = ""; + + if (!String.IsNullOrEmpty(metricName)) + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineOtherMetrics[metricName], 2)} | {Math.Round(lr.AveragedComparandOtherMetrics[metricName], 2)} | {Math.Round(lr.OtherMetricsDiff[metricName], 2)}| {Math.Round(lr.OtherMetricsDiffPerc[metricName], 2)}|"; + } + else + { + baseRow = $"| {benchmarkName} | {lr.BaselineRunName} | {lr.ComparandRunName} | {Math.Round(lr.AveragedBaselineMeanValue, 2)} | {Math.Round(lr.AveragedComparandMeanValue, 2)} | {Math.Round(lr.MeanDiff, 2)}| {Math.Round(lr.MeanDiffPerc, 2)}|"; + } if (configuration.Output.Columns != null) { foreach (var column in configuration.Output.Columns) { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) - { - lr.Baseline.OtherMetrics[column] = baselineValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Baseline.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Baseline); - } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; + double? baselineValue = lr.AveragedBaselineOtherMetrics.GetValueOrDefault(column); + double? comparandValue = lr.AveragedComparandOtherMetrics.GetValueOrDefault(column); - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) - { - lr.Comparand.OtherMetrics[column] = comparandValue = API.GCProcessData.LookupAggregateCalculation(column, lr.Comparand.GCData) ?? MicrobenchmarkResult.LookupStatisticsCalculation(column, lr.Comparand); - } + string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 4).ToString() : string.Empty; string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 4).ToString() : string.Empty; - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; string deltaResult = delta.HasValue ? Math.Round(delta.Value, 4).ToString() : string.Empty; @@ -189,31 +202,32 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm } } - if (configuration.Output.cpu_columns != null) - { - foreach (var column in configuration.Output.cpu_columns) - { - if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) - { - lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; - - if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) - { - lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; - } - string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; - - double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; - string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; - - double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; - string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; - - baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; - } - } + // TODO: Add CPU columns if needed in the future. + //if (configuration.Output.cpu_columns != null) + //{ + // foreach (var column in configuration.Output.cpu_columns) + // { + // if (!lr.Baseline.OtherMetrics.TryGetValue(column, out double? baselineValue)) + // { + // lr.Baseline.OtherMetrics[column] = baselineValue = lr.Baseline.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string baselineResult = baselineValue.HasValue ? Math.Round(baselineValue.Value, 2).ToString() : string.Empty; + + // if (!lr.Comparand.OtherMetrics.TryGetValue(column, out double? comparandValue)) + // { + // lr.Comparand.OtherMetrics[column] = comparandValue = lr.Comparand.CPUData?.GetIncCountForGCMethod(column) ?? null; + // } + // string comparandResult = comparandValue.HasValue ? Math.Round(comparandValue.Value, 2).ToString() : string.Empty; + + // double? delta = baselineValue.HasValue && comparandValue.HasValue ? comparandValue.Value - baselineValue.Value : null; + // string deltaResult = delta.HasValue ? Math.Round(delta.Value, 2).ToString() : string.Empty; + + // double? deltaPercent = delta.HasValue ? (delta / baselineValue.Value) * 100 : null; + // string deltaPercentResult = deltaPercent.HasValue ? Math.Round(deltaPercent.Value, 2).ToString() : string.Empty; + + // baseRow += $"{baselineResult} | {comparandResult} | {deltaResult} | {deltaPercentResult} |"; + // } + //} sw.WriteLine(baseRow); } @@ -224,15 +238,6 @@ internal static void AddTableForSingleCriteria(this StreamWriter sw, Microbenchm Console.WriteLine(e.StackTrace); } } - - // Dispose all the Analyzers now that we are persisting the values. - foreach (var comparison in comparisons) - { - comparison.Baseline?.GCData?.Parent?.Dispose(); - comparison.Baseline?.CPUData?.Parent?.Analyzer?.Dispose(); - comparison.Comparand?.GCData?.Parent?.Dispose(); - comparison.Comparand?.CPUData?.Parent?.Analyzer?.Dispose(); - } } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs deleted file mode 100644 index 93f4faea7f0..00000000000 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure.Core/Presentation/Microbenchmarks/Presentation.cs +++ /dev/null @@ -1,31 +0,0 @@ -using GC.Infrastructure.Core.Analysis; -using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Configurations; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; - -namespace GC.Infrastructure.Core.Presentation.Microbenchmarks -{ - public static class Presentation - { - public static IReadOnlyList Present(MicrobenchmarkConfiguration configuration, Dictionary executionDetails) - { - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); - foreach (var format in configuration.Output.Formats) - { - if (format == "markdown") - { - Markdown.GenerateTable(configuration, comparisonResults, executionDetails, Path.Combine(configuration.Output.Path, "Results.md")); - continue; - } - - if (format == "json") - { - Json.Json.Generate(configuration, comparisonResults, Path.Combine(configuration.Output.Path, "Results.json")); - continue; - } - } - - return comparisonResults; - } - } -} diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs index 35c4e50a266..b179894d524 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkAnalyzeCommand.cs @@ -1,10 +1,12 @@ -using Spectre.Console.Cli; -using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; +using GC.Infrastructure.Core.Analysis; using GC.Infrastructure.Core.Analysis.Microbenchmarks; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; using GC.Infrastructure.Core.Configurations; using GC.Infrastructure.Core.Configurations.Microbenchmarks; +using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using Spectre.Console; +using Spectre.Console.Cli; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -21,9 +23,46 @@ public override int Execute([NotNull] CommandContext context, [NotNull] Microben { ConfigurationChecker.VerifyFile(settings.ConfigurationPath, nameof(MicrobenchmarkAnalyzeCommand)); MicrobenchmarkConfiguration configuration = MicrobenchmarkConfigurationParser.Parse(settings.ConfigurationPath); - IReadOnlyList comparisonResults = MicrobenchmarkResultsAnalyzer.GetComparisons(configuration); - Presentation.Present(configuration, new()); // Execution details aren't available for the analysis-only mode. + + var comparisonResultsGroupedByName = ExecuteAnalysis(configuration); + + Present(configuration, comparisonResultsGroupedByName, new()); // Execution details aren't available for the analysis-only mode. return 0; } + + public static List ExecuteAnalysis(MicrobenchmarkConfiguration configuration) + { + var bdnJsonResults = MicrobenchmarkResultComparison.LoadBdnJsonResults(configuration); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) {bdnJsonResults.Count} BDN results loaded.[/]"); + var microbenchmarkResults = MicrobenchmarkResultComparison.AnalyzeMicrobenchmarkResults(configuration, bdnJsonResults); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Analysis completed.[/]"); + var comparisonResults = MicrobenchmarkResultComparison.CompareMicrobenchmarkResults(configuration, microbenchmarkResults); + + return MicrobenchmarkResultComparison.GroupComparisonResultsByName(configuration, comparisonResults); + } + + public static void Present(MicrobenchmarkConfiguration configuration, + List comparisonResultsGroupedByName, + Dictionary executionDetails) + { + foreach (var format in configuration.Output.Formats) + { + if (format == "markdown") + { + string outputPath = Path.Combine(configuration.Output.Path, "Results.md"); + Markdown.GenerateTable(configuration, comparisonResultsGroupedByName, executionDetails, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); + continue; + } + + if (format == "json") + { + string outputPath = Path.Combine(configuration.Output.Path, "Results.json"); + Json.Generate(configuration, comparisonResultsGroupedByName, outputPath); + AnsiConsole.MarkupLine($"[bold green] ({DateTime.Now}) Results written to {Markup.Escape(outputPath)}.[/]"); + continue; + } + } + } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs index ca063389ef5..4de521e5cc8 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/Microbenchmark/MicrobenchmarkCommand.cs @@ -1,18 +1,17 @@ -using GC.Infrastructure.Core.Analysis.Microbenchmarks; +using GC.Analysis.API; using GC.Infrastructure.Core.Analysis; +using GC.Infrastructure.Core.Analysis.Microbenchmarks; using GC.Infrastructure.Core.CommandBuilders; -using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.Configurations; -using GC.Infrastructure.Core.Presentation.Microbenchmarks; +using GC.Infrastructure.Core.Configurations.Microbenchmarks; using GC.Infrastructure.Core.TraceCollection; using Newtonsoft.Json; -using Spectre.Console.Cli; using Spectre.Console; +using Spectre.Console.Cli; using System.ComponentModel; -using System.Diagnostics.CodeAnalysis; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Text; -using System.Configuration; namespace GC.Infrastructure.Commands.Microbenchmark { @@ -121,11 +120,11 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi // Should only be one if it's a fresh run. string jsonFile = jsonFiles.First(); - MicrobenchmarkResults output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); + BdnJsonResult output = JsonConvert.DeserializeObject(File.ReadAllText(jsonFile)); // Assumption: A particular run, regardless of the parameters, will run ~the same vals. - IEnumerable operationsPerNanos = output.Benchmarks.First().Measurements.Where(m => m.IterationMode == "Workload" && m.IterationStage == "Actual") - .Select(m => m.Operations); + var operationsPerNanos = GoodLinq.Select(GoodLinq.Where(output.Benchmarks.First().Measurements, m => m.IterationMode == "Workload" && m.IterationStage == "Actual"), m => m.Operations); + // For now take the max but we will possibly be sacrificing duration for precision. invocationCountFromBaseline = operationsPerNanos.Max(); } @@ -149,9 +148,9 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi (string, string) fileNameAndCommand = MicrobenchmarkCommandBuilder.Build(configuration, run, benchmark, invocationCountFromBaseline); run.Value.Name = run.Key; - for (int index = 0; index < configuration.Environment.iteration; index++) + for (int index = 0; index < configuration.Environment.iterations; index++) { - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbechmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); + AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Running Microbenchmarks: {configuration.Name} - {run.Key} {benchmark} - iteration: {index} [/]\n"); // Run The BDN process with the trace collector. using (Process bdnProcess = new()) { @@ -199,10 +198,11 @@ public static MicrobenchmarkOutputResults RunMicrobenchmarks(MicrobenchmarkConfi } } - IReadOnlyList results = Presentation.Present(configuration, executionDetails); + var comparisonResultsGroupedName = MicrobenchmarkAnalyzeCommand.ExecuteAnalysis(configuration); + + MicrobenchmarkAnalyzeCommand.Present(configuration, comparisonResultsGroupedName, executionDetails); // Execution details aren't available for the analysis-only mode. Directory.SetCurrentDirectory(currentDirectory); - AnsiConsole.Markup($"[bold green] ({DateTime.Now}) Wrote Microbechmark Results to: {Markup.Escape(Path.Combine(configuration.Output.Path, "Results.md"))} [/]"); - return new MicrobenchmarkOutputResults(executionDetails, results); + return new MicrobenchmarkOutputResults(executionDetails, comparisonResultsGroupedName); } } } diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml index ea719bad713..257f021c024 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/Microbenchmarks.yaml @@ -6,7 +6,7 @@ microbenchmark_configurations: environment: default_max_seconds: 3000 - iteration: 1 + iterations: 1 # Configurations that involve capturing a trace. trace_configurations: diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt index 1bef991eca5..6056a312aff 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/BaseSuite/MicrobenchmarksToRun.txt @@ -11,7 +11,6 @@ "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 1000)" | "System.Tests.Perf_GC.NewOperator_Array(length: 10000)" | -"System.IO.Tests.Perf_File.ReadAllBytesAsync(size: 104857600)" | "System.Numerics.Tests.Perf_BigInteger.Subtract(arguments: 65536*" | "System.Collections.CtorGivenSize.Array(size: 512)" | "ByteMark.BenchBitOps" | diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs index 0cec453d0c3..213385812ea 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/CreateSuiteCommand.cs @@ -91,7 +91,7 @@ public static Dictionary CreateSuites(InputConfiguration configu configurationMap["GCPerfSim"] = gcPerfSimBase; string microbenchmarkBase = CreateMicrobenchmarkSuite(configuration, suitePath); - configurationMap["Microbenchmark"] = microbenchmarkBase; + configurationMap["Microbenchmarks"] = microbenchmarkBase; string aspnetBenchmarkBase = CreateASPNetBenchmarkSuite(configuration, suitePath); configurationMap["ASPNetBenchmarks"] = aspnetBenchmarkBase; @@ -158,7 +158,7 @@ internal static string CreateASPNetBenchmarkSuite(InputConfiguration inputConfig internal static string CreateMicrobenchmarkSuite(InputConfiguration inputConfiguration, string suitePath) { - string microbenchmarkSuitePath = Path.Combine(suitePath, "Microbenchmark"); + string microbenchmarkSuitePath = Path.Combine(suitePath, "Microbenchmarks"); Core.Utilities.TryCreateDirectory(microbenchmarkSuitePath); @@ -225,6 +225,12 @@ internal static MicrobenchmarkConfiguration CreateBaseMicrobenchmarkSuite(InputC }); } + // Set iterations if they exist. + if (inputConfiguration.iterations != null) + { + configuration.Environment.iterations = inputConfiguration.iterations.GetValueOrDefault("microbenchmarks", 1); + } + // The first run is always the baseline. configuration.Runs.First().Value.is_baseline = true; diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs index 0cd4fc33087..1483aa011d1 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunCommand.cs @@ -138,7 +138,7 @@ public override int Execute([NotNull] CommandContext context, [NotNull] RunComma AnsiConsole.WriteLine(); AnsiConsole.Write(new Rule("Running Microbenchmarks")); AnsiConsole.WriteLine(); - string microbenchmarkBase = configurationMap["Microbenchmark"]; + string microbenchmarkBase = configurationMap["Microbenchmarks"]; string[] microbenchmarkConfigurations = Directory.GetFiles(microbenchmarkBase, "*.yaml"); HashSet uniqueMicrobenchmarks = new(); diff --git a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs index 2fc52eb00ff..baac3f6ffca 100644 --- a/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs +++ b/src/benchmarks/gc/GC.Infrastructure/GC.Infrastructure/Commands/RunCommand/RunSuiteCommand.cs @@ -22,7 +22,7 @@ public sealed class RunSuiteCommandSettings : CommandSettings public override int Execute([NotNull] CommandContext context, [NotNull] RunSuiteCommandSettings settings) { - if (!string.IsNullOrEmpty(settings.SuiteBasePath) || !Directory.Exists(settings.SuiteBasePath)) + if (string.IsNullOrEmpty(settings.SuiteBasePath) || !Directory.Exists(settings.SuiteBasePath)) { throw new ArgumentNullException($"{nameof(RunSuiteCommandSettings)}: {nameof(settings.SuiteBasePath)} was either null or the directory doesn't exists."); } @@ -50,7 +50,7 @@ public static void RunSuite(Dictionary configuration) catch (Exception e) { - AnsiConsole.Write($"[red] GCPerfSim Configuration: {c} failed with {e.Message} [/]"); + AnsiConsole.MarkupLine($"[red] GCPerfSim Configuration: {c} failed with {e.Message} [/]"); } } @@ -66,7 +66,7 @@ public static void RunSuite(Dictionary configuration) catch (Exception e) { - AnsiConsole.Write($"[red] Microbenchmark Configuration: {c} failed with {e.Message} [/]"); + AnsiConsole.MarkupLine($"[red] Microbenchmark Configuration: {c} failed with {e.Message} [/]"); } } @@ -83,7 +83,7 @@ public static void RunSuite(Dictionary configuration) catch (Exception e) { - AnsiConsole.Write($"[red] ASPNet Configuration: {c} failed with {e.Message} [/]"); + AnsiConsole.MarkupLine($"[red] ASPNet Configuration: {c} failed with {e.Message} [/]"); } } } diff --git a/src/benchmarks/micro/MicroBenchmarks.csproj b/src/benchmarks/micro/MicroBenchmarks.csproj index cf9ab26266b..d7b14d9336f 100644 --- a/src/benchmarks/micro/MicroBenchmarks.csproj +++ b/src/benchmarks/micro/MicroBenchmarks.csproj @@ -16,6 +16,13 @@ $(NoWarn);SYSLIB0011 $(NoWarn);SYSLIB5003 + + false + $(NoWarn);NU1510 Exe AnyCPU portable @@ -94,7 +101,7 @@ - + @@ -102,13 +109,18 @@ - - - - - - - + + + + + + + diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 121d9dd9b98..8d2a590cae9 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Threading.Tasks; using BenchmarkDotNet.Running; using System.IO; using BenchmarkDotNet.Extensions; @@ -14,7 +15,7 @@ namespace MicroBenchmarks { class Program { - static int Main(string[] args) + static async Task Main(string[] args) { var argsList = new List(args); int? partitionCount; @@ -40,9 +41,14 @@ static int Main(string[] args) return 1; } - return BenchmarkSwitcher + // Use RunAsync (not Run) so BDN does not install its single-threaded + // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync + // entrypoint installs that context before benchmark discovery, which + // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource] + // callbacks (e.g. SslStreamTests.GetTls13Support). + var summaries = await BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) - .Run(argsList.ToArray(), + .RunAsync(argsList.ToArray(), RecommendedConfig.Create( artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")), mandatoryCategories: ImmutableHashSet.Create([Categories.Libraries, Categories.Runtime, Categories.ThirdParty, Categories.Sve]), @@ -52,7 +58,9 @@ static int Main(string[] args) categoryExclusionFilterValue: categoryExclusionFilterValue, getDiffableDisasm: getDiffableDisasm) .AddValidator(new NoWasmValidator(Categories.NoWASM))) - .ToExitCode(); + .ConfigureAwait(false); + + return summaries.ToExitCode(); } } } \ No newline at end of file diff --git a/src/benchmarks/micro/README.md b/src/benchmarks/micro/README.md index 590edec71af..d19ed650f5f 100644 --- a/src/benchmarks/micro/README.md +++ b/src/benchmarks/micro/README.md @@ -12,7 +12,7 @@ To learn more about designing benchmarks, please read [Microbenchmark Design Gui ## Quick Start -The first thing that you need to choose is the Target Framework. Available options are: `netcoreapp3.1|net6.0|net7.0|net8.0|net9.0|net10.0|net11.0|net472`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework. +The first thing that you need to choose is the Target Framework. Available options are: `net8.0|net9.0|net10.0|net11.0`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework. The following commands are run from the `src/benchmarks/micro` directory. diff --git a/src/benchmarks/micro/Serializers/DataGenerator.cs b/src/benchmarks/micro/Serializers/DataGenerator.cs index b78f5d9330b..55bb71917a1 100644 --- a/src/benchmarks/micro/Serializers/DataGenerator.cs +++ b/src/benchmarks/micro/Serializers/DataGenerator.cs @@ -32,7 +32,7 @@ internal static T Generate() return (T)(object)CreateIndexViewModel(); if (typeof(T) == typeof(MyEventsListerViewModel)) return (T)(object)CreateMyEventsListerViewModel(); - if (typeof(T) == typeof(BinaryData)) + if (typeof(T) == typeof(BinaryDataPayload)) return (T)(object)CreateBinaryData(1024); if (typeof(T) == typeof(CollectionsOfPrimitives)) return (T)(object)CreateCollectionsOfPrimitives(1024); // 1024 values was copied from CoreFX benchmarks @@ -163,8 +163,8 @@ private static MyEventsListerItem CreateMyEventsListerItem() }, 4).ToList() }; - private static BinaryData CreateBinaryData(int size) - => new BinaryData + private static BinaryDataPayload CreateBinaryData(int size) + => new BinaryDataPayload { ByteArray = CreateByteArray(size) }; @@ -383,7 +383,13 @@ public string FormattedDate [Serializable] [ProtoContract] [MessagePackObject] - public class BinaryData + // Renamed from BinaryData to avoid name collision with System.BinaryData + // (introduced in net6+ via System.Memory.Data, transitive via Azure.Core). + // The collision caused [GenericTypeArguments(typeof(BinaryData))] to resolve + // to System.BinaryData in some compile contexts (e.g. when the harness targets + // net8.0 instead of netstandard2.0), bypassing the data generator and producing + // NotImplementedException at runtime. + public class BinaryDataPayload { [ProtoMember(1)] [Key(0)] public byte[] ByteArray { get; set; } } @@ -496,7 +502,7 @@ public enum SystemTextJsonSerializationMode [JsonSerializable(typeof(Location))] [JsonSerializable(typeof(IndexViewModel))] [JsonSerializable(typeof(MyEventsListerViewModel))] - [JsonSerializable(typeof(BinaryData))] + [JsonSerializable(typeof(BinaryDataPayload))] [JsonSerializable(typeof(CollectionsOfPrimitives))] [JsonSerializable(typeof(XmlElement))] [JsonSerializable(typeof(SimpleStructWithProperties))] diff --git a/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs b/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs index 4622f970fd7..357052a8d39 100644 --- a/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs +++ b/src/benchmarks/micro/libraries/System.Runtime/Perf.String.cs @@ -321,6 +321,7 @@ public static string[] ReadInputFile(string name) => File.ReadAllLines(Path.Combine(AppContext.BaseDirectory, "libraries", "System.Runtime", "TestData", name)); [Benchmark] + [BenchmarkCategory(Categories.NoWASM)] // CSV files in TestData are not available in WASM filesystem [ArgumentsSource(nameof(CsvCorpus))] public string[] Split_Csv(string testName, string[] lines) { diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs index 10fe6cc0778..49ec76814a1 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/ReadJson.cs @@ -18,7 +18,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(Location))] [GenericTypeArguments(typeof(IndexViewModel))] [GenericTypeArguments(typeof(MyEventsListerViewModel))] - [GenericTypeArguments(typeof(BinaryData))] + [GenericTypeArguments(typeof(BinaryDataPayload))] [GenericTypeArguments(typeof(Dictionary))] [GenericTypeArguments(typeof(ImmutableDictionary))] [GenericTypeArguments(typeof(ImmutableSortedDictionary))] diff --git a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs index 1eb8b8a57f8..f00a4da8b26 100644 --- a/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs +++ b/src/benchmarks/micro/libraries/System.Text.Json/Serializer/WriteJson.cs @@ -20,7 +20,7 @@ namespace System.Text.Json.Serialization.Tests [GenericTypeArguments(typeof(Location))] [GenericTypeArguments(typeof(IndexViewModel))] [GenericTypeArguments(typeof(MyEventsListerViewModel))] - [GenericTypeArguments(typeof(BinaryData))] + [GenericTypeArguments(typeof(BinaryDataPayload))] [GenericTypeArguments(typeof(Dictionary))] [GenericTypeArguments(typeof(ImmutableDictionary))] [GenericTypeArguments(typeof(ImmutableSortedDictionary))] diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index ef8782db2aa..a54dc819171 100644 --- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -1,7 +1,11 @@  Library - netstandard2.0 + + net8.0 enable true @@ -13,7 +17,7 @@ - + diff --git a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs b/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs index 3c8b343fc57..b76dd07dc29 100644 --- a/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs +++ b/src/harness/BenchmarkDotNet.Extensions/CommandLineOptions.cs @@ -35,7 +35,7 @@ public static List ParseAndRemoveStringsParameter(List argsList, int parameterIndex = argsList.IndexOf(parameter); parameterValue = new List(); - if (parameterIndex + 1 < argsList.Count) + if (parameterIndex != -1 && parameterIndex + 1 < argsList.Count) { while (parameterIndex + 1 < argsList.Count && !argsList[parameterIndex + 1].StartsWith("-")) { @@ -94,4 +94,4 @@ public static void ValidatePartitionParameters(int? count, int? index) } } } -} \ No newline at end of file +} diff --git a/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs b/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs index 44f81dd29e9..3124e82c546 100644 --- a/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs @@ -61,26 +61,26 @@ internal static string BuildDisassemblyString(DisassemblyResult disassemblyResul private static Func GetElementGetter(string name) { - var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier"); + var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!; - type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic); + type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic)!; - var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic); + var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic)!; - var method = property.GetGetMethod(nonPublic: true); + var method = property.GetGetMethod(nonPublic: true)!; var generic = typeof(Func<,>).MakeGenericType(type, typeof(T)); var @delegate = method.CreateDelegate(generic); - return (obj) => (T)@delegate.DynamicInvoke(obj); // cast to (Func) throws + return (obj) => (T)@delegate.DynamicInvoke(obj)!; // cast to (Func) throws } private static Func> GetPrettifyMethod() { - var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier"); + var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!; - var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic); + var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic)!; var @delegate = method.CreateDelegate(typeof(Func>)); diff --git a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs index 7b3b2d38f3b..280e43653d9 100644 --- a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs @@ -21,7 +21,7 @@ public class MandatoryCategoryValidator : IValidator public MandatoryCategoryValidator(ImmutableHashSet categories) => _mandatoryCategories = categories; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => !benchmark.Descriptor.Categories.Any(category => _mandatoryCategories.Contains(category))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) @@ -30,6 +30,7 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: TreatsWarningsAsErrors, $"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]") - ); + ) + .ToAsyncEnumerable(); } } \ No newline at end of file diff --git a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs index 5b2cfec63ca..68fe73a4645 100644 --- a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs @@ -23,7 +23,7 @@ public class NoWasmValidator : IValidator public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) @@ -32,7 +32,8 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: TreatsWarningsAsErrors, $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]") - ); + ) + .ToAsyncEnumerable(); private bool IsAsyncMethod(MethodInfo workloadMethod) { diff --git a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs index c1ed4268e0f..d308afb27a8 100644 --- a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs @@ -4,24 +4,67 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using Reporting; using System; +using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace BenchmarkDotNet.Extensions { - public class PerfLabExporter : ExporterBase + // Implements IExporter directly (not ExporterBase) because PerfLabExporter writes + // a file with a custom name pattern ("{type}-perf-lab-report.json") via + // File.WriteAllTextAsync and manages the file lifecycle itself, rather than having + // ExporterBase open and hand us a writer for a default-named file. + public class PerfLabExporter : IExporter { - protected override string FileExtension => "json"; - protected override string FileCaption => "perf-lab-report"; + private const string FileExtension = "json"; + private const string FileCaption = "perf-lab-report"; - public PerfLabExporter() + public string Name => nameof(PerfLabExporter); + + public async ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken) + { + string? jsonOutput = BuildJson(summary); + if (jsonOutput is null) + return; + + string filePath = GetArtifactFullName(summary); + if (File.Exists(filePath)) + { + try + { + File.Delete(filePath); + } + catch (IOException) + { + string uniqueString = DateTime.Now.ToString("yyyyMMdd-HHmmss"); + string altPath = $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}-{uniqueString}.{FileExtension}"; + logger.WriteLineError($"Could not overwrite file {filePath}. Exporting to {altPath}"); + filePath = altPath; + } + } + + await File.WriteAllTextAsync(filePath, jsonOutput, cancellationToken).ConfigureAwait(false); + logger.WriteLineInfo($" {filePath}"); + } + + private string GetArtifactFullName(Summary summary) + => $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}.{FileExtension}"; + + private static string GetFileName(Summary summary) { + var targets = summary.BenchmarksCases.Select(b => b.Descriptor.Type).Distinct().ToArray(); + if (targets.Length == 1) + return FolderNameHelper.ToFolderName(targets.Single()); + return summary.Title; } - public override void ExportToLog(Summary summary, ILogger logger) + private static string? BuildJson(Summary summary) { var reporter = new Reporter(); @@ -49,7 +92,7 @@ public override void ExportToLog(Summary summary, ILogger logger) var test = new Test(); test.Name = FullNameProvider.GetBenchmarkName(report.BenchmarkCase); test.Categories = report.BenchmarkCase.Descriptor.Categories; - + if (hasCriticalErrors) { test.AdditionalData["criticalErrors"] = "true"; @@ -58,7 +101,7 @@ public override void ExportToLog(Summary summary, ILogger logger) var results = from result in report.AllMeasurements where result.IterationMode == Engines.IterationMode.Workload && result.IterationStage == Engines.IterationStage.Result orderby result.LaunchIndex, result.IterationIndex - select new { result.Nanoseconds, result.Operations}; + select new { result.Nanoseconds, result.Operations }; var overheadResults = from result in report.AllMeasurements where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jitting @@ -104,7 +147,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit HigherIsBetter = true, MetricName = "Count", Results = (from result in results - select (double)result.Operations).ToList() + select (double)result.Operations).ToList() }); foreach (var metric in report.Metrics.Keys) @@ -130,9 +173,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit reporter.AddTest(test); } - var jsonOutput = reporter.GetJson(); - if (jsonOutput is not null) - logger.WriteLine(jsonOutput); + return reporter.GetJson(); } } } diff --git a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs index aaf8a62317a..9df1b3d99cb 100644 --- a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs @@ -19,7 +19,7 @@ public class TooManyTestCasesValidator : IValidator public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) { var byDescriptor = validationParameters.Benchmarks .Where(benchmark => !SkipValidation(benchmark.Descriptor.WorkloadMethod)) @@ -29,10 +29,11 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: true, message: $"{group.Key.Descriptor.Type.Name}.{group.Key.Descriptor.WorkloadMethod.Name} has {group.Count()} test cases. It MUST NOT have more than {Limit} test cases. We don't have inifinite amount of time to run all the benchmarks!!", - benchmarkCase: group.First())); + benchmarkCase: group.First())) + .ToAsyncEnumerable(); } - private static bool SkipValidation(MemberInfo member) + private static bool SkipValidation(MemberInfo? member) { while (member != null) { diff --git a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs index 532e9b003f5..e3903769a69 100644 --- a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs @@ -14,7 +14,9 @@ public class UniqueArgumentsValidator : IValidator { public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) + // Use ToAsyncEnumerable() (not async + yield return) to avoid the AsyncIteratorMethodBuilder + // state machine deadlocking with BDN's BenchmarkSynchronizationContext (matches BDN's own validators). + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => benchmark.HasArguments || benchmark.HasParameters) .GroupBy(benchmark => (benchmark.Descriptor.Type, benchmark.Descriptor.WorkloadMethod, benchmark.Job)) @@ -25,12 +27,16 @@ public IEnumerable Validate(ValidationParameters validationPara return numberOfTestCases != numberOfUniqueTestCases; }) - .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())); + .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())) + .ToAsyncEnumerable(); private class BenchmarkArgumentsComparer : IEqualityComparer { - public bool Equals(BenchmarkCase x, BenchmarkCase y) + public bool Equals(BenchmarkCase? x, BenchmarkCase? y) { + if (x is null || y is null) + return ReferenceEquals(x, y); + if (FullNameProvider.GetBenchmarkName(x).Equals(FullNameProvider.GetBenchmarkName(y), System.StringComparison.Ordinal)) return true; diff --git a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs index 107c298e93f..e36efdf9747 100644 --- a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs @@ -130,7 +130,7 @@ public static byte[] ArrayBase64EncodingBytes(int count) /// the stored values are randomly generated. /// GenerateValue is used to generate a random value in the appropriate range for both the key and value /// - public static Dictionary Dictionary(int count) + public static Dictionary Dictionary(int count) where TKey : notnull { if (count > 2 && typeof(TKey) == typeof(bool)) throw new ArgumentOutOfRangeException("count", "Cannot exceed 2 for Dictionary"); diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj index 796d014d367..24a2526f6ec 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/BenchmarkDotNet.Extensions.Tests.csproj @@ -7,7 +7,9 @@ - + diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs index 4fbe6a7a10e..48e53620bcf 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/CommandLineOptionsTests.cs @@ -128,5 +128,89 @@ public void PartitionIndexValueGreaterThanCount() Assert.Throws(() => CommandLineOptions.ValidatePartitionParameters(count, index)); } + + [Fact] + public void ParseAndRemoveStringsParameterCollectsValuesUntilNextFlag() + { + List argsList = new List { + "--exclusion-filter", + "System.*", + "Microsoft.*", + "--partition-count", + "4" + }; + + var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List filters); + + Assert.Equal(new[] { "System.*", "Microsoft.*" }, filters); + Assert.Equal(new[] { "--partition-count", "4" }, remaining); + } + + [Fact] + public void ParseAndRemoveBooleanParameterRemovesSwitchWhenPresent() + { + List argsList = new List { + "--wasm", + "--filter", + "*" + }; + + CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled); + + Assert.True(enabled); + Assert.Equal(new[] { "--filter", "*" }, argsList); + } + + [Fact] + public void ParseAndRemoveStringsParameterLeavesArgsUntouchedWhenSwitchIsMissing() + { + List argsList = new List { + "literal-value", + "--filter", + "*" + }; + + var remaining = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out List filters); + + Assert.Empty(filters); + Assert.Equal(new[] { "literal-value", "--filter", "*" }, remaining); + } + + [Fact] + public void ParseAndRemoveBooleanParameterReturnsFalseWhenSwitchIsMissing() + { + List argsList = new List { + "--filter", + "*" + }; + + CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--wasm", out bool enabled); + + Assert.False(enabled); + Assert.Equal(new[] { "--filter", "*" }, argsList); + } + + [Theory] + [InlineData("--partition-count")] + [InlineData("--partition-index")] + public void ParseAndRemoveIntParameterThrowsWhenValueIsMissing(string parameter) + { + List argsList = new List { parameter }; + + Assert.Throws(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _)); + } + + [Theory] + [InlineData("--partition-count", "abc")] + [InlineData("--partition-index", "3.14")] + public void ParseAndRemoveIntParameterThrowsWhenValueIsNotAnInteger(string parameter, string value) + { + List argsList = new List { + parameter, + value + }; + + Assert.Throws(() => CommandLineOptions.ParseAndRemoveIntParameter(argsList, parameter, out int? _)); + } } } diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs index 9319d50c607..f2a551c3bf7 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/PartitionFilterTests.cs @@ -63,8 +63,9 @@ public void NoBenchmarksAreOmitted_RealData() IConfig recommendedConfig = RecommendedConfig.Create( artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(PartitionFilterTests).Assembly.Location)!, "BenchmarkDotNet.Artifacts")), mandatoryCategories: ImmutableHashSet.Create(Categories.Libraries, Categories.Runtime, Categories.ThirdParty)); - (bool isSuccess, IConfig parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig); + (bool isSuccess, IConfig? parsedConfig, var _) = ConfigParser.Parse(new string[] { "--filter", "*" }, nullLogger, recommendedConfig); Assert.True(isSuccess); + Assert.NotNull(parsedConfig); Assembly microbenchmarksAssembly = typeof(Categories).Assembly; (bool allTypesValid, IReadOnlyList runnable) = Running.TypeFilter.GetTypesWithRunnableBenchmarks( @@ -73,7 +74,7 @@ public void NoBenchmarksAreOmitted_RealData() nullLogger); Assert.True(allTypesValid); - BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig, runnable); + BenchmarkRunInfo[] allBenchmarks = GetAllBenchmarks(parsedConfig!, runnable); Dictionary idToPartitionIndex = new (); for (int i = 0; i < 10; i++) @@ -86,7 +87,7 @@ public void NoBenchmarksAreOmitted_RealData() { PartitionFilter filter = new(PartitionCount, partitionIndex); - foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig, runnable).SelectMany(benchmark => benchmark.BenchmarksCases)) + foreach (BenchmarkCase benchmark in GetAllBenchmarks(parsedConfig!, runnable).SelectMany(benchmark => benchmark.BenchmarksCases)) { if (filter.Predicate(benchmark)) { diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs index fdabaa78db0..5715ffcf676 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; @@ -26,7 +27,7 @@ public void DuplicatedArgumentsAreDetected(Type typeWithBenchmarks, bool shouldR var benchmarksForType = BenchmarkConverter.TypeToBenchmarks(typeWithBenchmarks); var validationParameters = new ValidationParameters(benchmarksForType.BenchmarksCases, benchmarksForType.Config); - var validationErrors = new UniqueArgumentsValidator().Validate(validationParameters); + var validationErrors = new UniqueArgumentsValidator().ValidateAsync(validationParameters).ToBlockingEnumerable(); if (shouldReportError) Assert.NotEmpty(validationErrors); diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs index ff3b8f0f6e0..58a6124938f 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs @@ -151,14 +151,14 @@ private static void SupportsNonDefaultValue() Assert.NotEqual(default, value); } - private static void SupportsDictionary(int count) + private static void SupportsDictionary(int count) where TKey : notnull { var dictionary = ValuesGenerator.Dictionary(count); Assert.NotNull(dictionary); Assert.Equal(count, dictionary.Count); } - private static void Supports(int count = 10) + private static void Supports(int count = 10) where T : notnull { SupportsArray(count); SupportsNonDefaultValue(); diff --git a/src/tools/Reporting/Directory.Packages.props b/src/tools/Reporting/Directory.Packages.props index ab6543dc86b..0fde0cf995d 100644 --- a/src/tools/Reporting/Directory.Packages.props +++ b/src/tools/Reporting/Directory.Packages.props @@ -4,7 +4,7 @@ true - + diff --git a/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs b/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs new file mode 100644 index 00000000000..8f3c5aa69b6 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ConsoleOutputCollection.cs @@ -0,0 +1,8 @@ +using Xunit; + +namespace ResultsComparer.Tests; + +[CollectionDefinition("Console output", DisableParallelization = true)] +public sealed class ConsoleOutputCollection +{ +} diff --git a/src/tools/ResultsComparer.Tests/DataTests.cs b/src/tools/ResultsComparer.Tests/DataTests.cs new file mode 100644 index 00000000000..a94592f139f --- /dev/null +++ b/src/tools/ResultsComparer.Tests/DataTests.cs @@ -0,0 +1,161 @@ +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using System.Text; +using Xunit; + +namespace ResultsComparer.Tests; + +public class DataTests +{ + [Fact] + public void DecompressExtractsJsonFilesFromNestedZipArchives() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var innerZipBytes = CreateInnerZip(("net10.0/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson())); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip"); + using var entryStream = entry.Open(); + entryStream.Write(innerZipBytes, 0, innerZipBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories); + var extractedFile = Assert.Single(extractedFiles); + var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile)); + + Assert.Contains("testuser", directoryName, System.StringComparison.OrdinalIgnoreCase); + Assert.Contains("net10.0", directoryName, System.StringComparison.OrdinalIgnoreCase); + Assert.Contains("SampleBenchmark", File.ReadAllText(extractedFile)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void DecompressExtractsJsonFilesFromTarGzArchives() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var tarGzBytes = CreateTarGzArchive( + ("payload/SampleBenchmark.full.json", ResultsComparerTestData.CreateBdnJson()), + ("payload/README.md", "ignored")); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/nativeaot10.0/testuser/arm64_win10-nativeaot10.0.tar.gz"); + using var entryStream = entry.Open(); + entryStream.Write(tarGzBytes, 0, tarGzBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFiles = Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories); + var extractedFile = Assert.Single(extractedFiles); + var directoryName = Path.GetFileName(Path.GetDirectoryName(extractedFile)); + + Assert.Contains("nativeaot10.0", directoryName, System.StringComparison.OrdinalIgnoreCase); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void DecompressPrefersNewestBenchmarkDotNetVersionWhenDuplicatesExist() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var outerZipPath = Path.Combine(tempDir.FullName, "results.zip"); + var outputDirectory = new DirectoryInfo(Path.Combine(tempDir.FullName, "output")); + outputDirectory.Create(); + + var innerZipBytes = CreateInnerZip( + ("net10.0/SampleBenchmark-a.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.9")), + ("net10.0/SampleBenchmark-b.full.json", ResultsComparerTestData.CreateBdnJson(benchmarkDotNetVersion: "0.13.10"))); + + using (var fileStream = File.Create(outerZipPath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create)) + { + var entry = archive.CreateEntry("Performance-Runs/net10.0/testuser/results.zip"); + using var entryStream = entry.Open(); + entryStream.Write(innerZipBytes, 0, innerZipBytes.Length); + } + + global::ResultsComparer.Data.Decompress(new FileInfo(outerZipPath), outputDirectory); + + var extractedFile = Assert.Single(Directory.GetFiles(outputDirectory.FullName, "*.full.json", SearchOption.AllDirectories)); + var json = File.ReadAllText(extractedFile); + + Assert.Contains("\"BenchmarkDotNetVersion\": \"0.13.10\"", json); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + private static byte[] CreateInnerZip(params (string EntryName, string Content)[] entries) + { + using var stream = new MemoryStream(); + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var (entryName, content) in entries) + { + var entry = archive.CreateEntry(entryName); + using var writer = new StreamWriter(entry.Open()); + writer.Write(content); + } + } + + return stream.ToArray(); + } + + private static byte[] CreateTarGzArchive(params (string EntryName, string Content)[] entries) + { + using var tarStream = new MemoryStream(); + using (var tarWriter = new TarWriter(tarStream, leaveOpen: true)) + { + foreach (var (entryName, content) in entries) + { + using var dataStream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + var tarEntry = new UstarTarEntry(TarEntryType.RegularFile, entryName) + { + DataStream = dataStream + }; + + tarWriter.WriteEntry(tarEntry); + } + } + + tarStream.Position = 0; + + using var gzipStream = new MemoryStream(); + using (var compressor = new GZipStream(gzipStream, CompressionLevel.SmallestSize, leaveOpen: true)) + { + tarStream.CopyTo(compressor); + } + + return gzipStream.ToArray(); + } +} diff --git a/src/tools/ResultsComparer.Tests/HelperTests.cs b/src/tools/ResultsComparer.Tests/HelperTests.cs new file mode 100644 index 00000000000..b59b713fa29 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/HelperTests.cs @@ -0,0 +1,137 @@ +using System.IO; +using System.Text; +using DataTransferContracts; +using Xunit; + +namespace ResultsComparer.Tests; + +public class HelperTests +{ + [Fact] + public void GetFilesToParseReturnsAllFullJsonFilesFromDirectory() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var nestedDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "nested")); + var expectedA = Path.Combine(tempDir.FullName, "first.full.json"); + var expectedB = Path.Combine(nestedDirectory.FullName, "second.full.json"); + + File.WriteAllText(expectedA, "{}"); + File.WriteAllText(expectedB, "{}"); + File.WriteAllText(Path.Combine(tempDir.FullName, "ignored.json"), "{}"); + + var result = global::ResultsComparer.Helper.GetFilesToParse(tempDir.FullName); + + Assert.Equal(2, result.Length); + Assert.Contains(expectedA, result); + Assert.Contains(expectedB, result); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetFilesToParseThrowsForMissingFullJsonPath() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var missingPath = Path.Combine(tempDir.FullName, "missing.full.json"); + + Assert.Throws(() => global::ResultsComparer.Helper.GetFilesToParse(missingPath)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void ReadFromStreamDeserializesBenchmarkResults() + { + var json = ResultsComparerTestData.CreateBdnJson(originalValues: [1.0, 1.1, 1.2], median: 1.1); + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + + BdnResult result = global::ResultsComparer.Helper.ReadFromStream(stream); + + Assert.Equal("SampleBenchmark-20240504-182513", result.Title); + Assert.Equal("X64", result.HostEnvironmentInfo.Architecture); + Assert.Single(result.Benchmarks); + Assert.Equal("Demo.Namespace.SampleBenchmark", result.Benchmarks[0].FullName); + } + + [Fact] + public void ReadFromFileDeserializesBenchmarkResults() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var filePath = Path.Combine(tempDir.FullName, "sample.full.json"); + File.WriteAllText(filePath, ResultsComparerTestData.CreateBdnJson()); + + BdnResult result = global::ResultsComparer.Helper.ReadFromFile(filePath); + + Assert.Equal("SampleBenchmark-20240504-182513", result.Title); + Assert.Single(result.Benchmarks); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetFilesToParseReturnsExplicitFilePath() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var filePath = Path.Combine(tempDir.FullName, "custom-name.json"); + File.WriteAllText(filePath, ResultsComparerTestData.CreateBdnJson()); + + var result = global::ResultsComparer.Helper.GetFilesToParse(filePath); + + Assert.Equal([filePath], result); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void GetModalInfoReturnsNullForSmallSampleSets() + { + var benchmark = new Benchmark + { + Statistics = new Statistics + { + N = 3, + OriginalValues = [1.0, 1.1, 1.2] + } + }; + + Assert.Null(global::ResultsComparer.Helper.GetModalInfo(benchmark)); + } + + [Fact] + public void GetModalInfoDetectsMultiClusterData() + { + var benchmark = new Benchmark + { + Statistics = new Statistics + { + N = 16, + OriginalValues = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + } + }; + + var modality = global::ResultsComparer.Helper.GetModalInfo(benchmark); + + Assert.Equal("bimodal", modality); + } +} diff --git a/src/tools/ResultsComparer.Tests/ProgramTests.cs b/src/tools/ResultsComparer.Tests/ProgramTests.cs new file mode 100644 index 00000000000..786f05209fc --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ProgramTests.cs @@ -0,0 +1,158 @@ +using System; +using System.Globalization; +using System.IO; +using Xunit; + +namespace ResultsComparer.Tests; + +[Collection("Console output")] +public class ProgramTests +{ + [Fact] + public void MainReportsInvalidThreshold() + { + var output = InvokeProgram(["--base", "base.json", "--diff", "diff.json", "--threshold", "not-a-threshold"]); + + Assert.Contains("Invalid Threshold", output); + } + + [Fact] + public void MainReportsMissingMatrixInputDirectory() + { + var missingDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + + var output = InvokeProgram(["matrix", "--input", missingDirectory, "--base", "base", "--diff", "diff", "--threshold", "5%"]); + + Assert.Contains("does NOT exist", output); + } + + [Fact] + public void MainPrintsNoDifferencesForEquivalentInputs() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var baseFile = Path.Combine(tempDir.FullName, "base.full.json"); + var diffFile = Path.Combine(tempDir.FullName, "diff.full.json"); + var json = ResultsComparerTestData.CreateBdnJson(); + + File.WriteAllText(baseFile, json); + File.WriteAllText(diffFile, json); + + var output = InvokeProgram(["--base", baseFile, "--diff", diffFile, "--threshold", "5%"]); + + Assert.Contains("No differences found", output); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void MatrixCommandPrintsLegendAndBenchmarkTable() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var inputDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "input")); + var baseDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-base")); + var diffDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-diff")); + + File.WriteAllText( + Path.Combine(baseDirectory.FullName, "SampleBenchmark.full.json"), + ResultsComparerTestData.CreateBdnJson( + originalValues: [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111], + median: 106, + allocatedBytes: 10)); + File.WriteAllText( + Path.Combine(diffDirectory.FullName, "SampleBenchmark.full.json"), + ResultsComparerTestData.CreateBdnJson( + originalValues: [200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211], + median: 206, + allocatedBytes: 30)); + + var output = InvokeProgram(["matrix", "--input", inputDirectory.FullName, "--base", "base", "--diff", "diff", "--threshold", "5%", "--ratio-only"]); + + Assert.Contains("# Legend", output); + Assert.Contains("Demo.Namespace.SampleBenchmark", output); + Assert.Contains("Slower", output); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void MatrixCommandTreatsEquivalentInputsAsSameNotNoise() + { + var tempDir = Directory.CreateTempSubdirectory(); + try + { + var inputDirectory = Directory.CreateDirectory(Path.Combine(tempDir.FullName, "input")); + var baseDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-base")); + var diffDirectory = Directory.CreateDirectory(Path.Combine(inputDirectory.FullName, "run-diff")); + var identicalJson = ResultsComparerTestData.CreateBdnJson(); + + File.WriteAllText(Path.Combine(baseDirectory.FullName, "SampleBenchmark.full.json"), identicalJson); + File.WriteAllText(Path.Combine(diffDirectory.FullName, "SampleBenchmark.full.json"), identicalJson); + + var output = InvokeProgram(["matrix", "--input", inputDirectory.FullName, "--base", "base", "--diff", "diff", "--threshold", "5%", "--ratio-only"]); + var lines = output.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + + Assert.Contains(lines, line => line.TrimStart().StartsWith("| Same", StringComparison.Ordinal)); + Assert.DoesNotContain(lines, line => line.TrimStart().StartsWith("| Noise", StringComparison.Ordinal)); + } + finally + { + tempDir.Delete(recursive: true); + } + } + + [Fact] + public void InvokeProgramRestoresCurrentCulture() + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; + + try + { + var expectedCulture = new CultureInfo("fr-FR"); + var expectedUICulture = new CultureInfo("de-DE"); + + CultureInfo.CurrentCulture = expectedCulture; + CultureInfo.CurrentUICulture = expectedUICulture; + + _ = InvokeProgram(["--base", "base.json", "--diff", "diff.json", "--threshold", "not-a-threshold"]); + + Assert.Equal(expectedCulture.Name, CultureInfo.CurrentCulture.Name); + Assert.Equal(expectedUICulture.Name, CultureInfo.CurrentUICulture.Name); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; + } + } + + private static string InvokeProgram(string[] args) + { + using var writer = new StringWriter(); + var originalOut = Console.Out; + var originalCulture = CultureInfo.CurrentCulture; + var originalUICulture = CultureInfo.CurrentUICulture; + Console.SetOut(writer); + try + { + Program.Main(args); + return writer.ToString(); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUICulture; + Console.SetOut(originalOut); + } + } +} diff --git a/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj new file mode 100644 index 00000000000..5611c64bee9 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ResultsComparer.Tests.csproj @@ -0,0 +1,13 @@ + + + net11.0 + enable + enable + false + true + + + + + + diff --git a/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs b/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs new file mode 100644 index 00000000000..323f66a9455 --- /dev/null +++ b/src/tools/ResultsComparer.Tests/ResultsComparerTestData.cs @@ -0,0 +1,52 @@ +using System.Globalization; +using System.Linq; + +namespace ResultsComparer.Tests; + +internal static class ResultsComparerTestData +{ + internal static string CreateBdnJson( + string title = "SampleBenchmark-20240504-182513", + string fullName = "Demo.Namespace.SampleBenchmark", + string? benchmarkNamespace = "Demo.Namespace", + double[]? originalValues = null, + double? median = null, + long? allocatedBytes = 42, + string benchmarkDotNetVersion = "0.13.10", + string osVersion = "Windows 11 (10.0.26100)", + string processorName = "Intel Core i7-8700", + string architecture = "X64") + { + originalValues ??= [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111]; + median ??= originalValues.OrderBy(v => v).ElementAt(originalValues.Length / 2); + string values = string.Join(", ", originalValues.Select(v => v.ToString(CultureInfo.InvariantCulture))); + string namespaceField = benchmarkNamespace is null ? "null" : $"\"{benchmarkNamespace}\""; + string memoryField = allocatedBytes is null + ? "\"Memory\": {}" + : $"\"Memory\": {{ \"BytesAllocatedPerOperation\": {allocatedBytes.Value.ToString(CultureInfo.InvariantCulture)} }}"; + + return $$""" + { + "Title": "{{title}}", + "HostEnvironmentInfo": { + "BenchmarkDotNetVersion": "{{benchmarkDotNetVersion}}", + "OsVersion": "{{osVersion}}", + "ProcessorName": "{{processorName}}", + "Architecture": "{{architecture}}" + }, + "Benchmarks": [ + { + "Namespace": {{namespaceField}}, + "FullName": "{{fullName}}", + "Statistics": { + "OriginalValues": [{{values}}], + "N": {{originalValues.Length}}, + "Median": {{median.Value.ToString(CultureInfo.InvariantCulture)}} + }, + {{memoryField}} + } + ] + } + """; + } +} diff --git a/src/tools/ResultsComparer.Tests/StatsTests.cs b/src/tools/ResultsComparer.Tests/StatsTests.cs new file mode 100644 index 00000000000..a8b8cfeb38e --- /dev/null +++ b/src/tools/ResultsComparer.Tests/StatsTests.cs @@ -0,0 +1,81 @@ +using System; +using System.IO; +using DataTransferContracts; +using Perfolizer.Mathematics.SignificanceTesting; +using Xunit; + +namespace ResultsComparer.Tests; + +[Collection("Console output")] +public class StatsTests +{ + [Fact] + public void GetSimplifiedOSNameRemovesParentheticalSuffix() + { + Assert.Equal("Windows 11", global::ResultsComparer.Stats.GetSimplifiedOSName("Windows 11 (10.0.26100)")); + } + + [Fact] + public void PrintAggregatesTotalsAndEmitsSectionsOnce() + { + var stats = new global::ResultsComparer.Stats(); + var environment = new HostEnvironmentInfo + { + Architecture = "X64", + OsVersion = "Windows 11 (10.0.26100)", + ProcessorName = "Intel Core i7-8700" + }; + var benchmark = new Benchmark + { + Namespace = "Demo.Namespace", + Statistics = new Statistics + { + OriginalValues = new[] { 1.0, 1.1, 1.2 }, + N = 3, + Median = 1.1 + }, + Memory = new Memory() + }; + + stats.Record(EquivalenceTestConclusion.Same, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Faster, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Slower, environment, benchmark); + stats.Record(EquivalenceTestConclusion.Unknown, environment, benchmark); + stats.Record(global::ResultsComparer.Stats.Noise, environment, benchmark); + + using var writer = new StringWriter(); + var originalOut = Console.Out; + Console.SetOut(writer); + try + { + stats.Print(); + var firstOutput = writer.ToString(); + + Assert.Contains("## Statistics", firstOutput); + Assert.Contains("Total: 5", firstOutput); + Assert.Contains("## Statistics per Architecture", firstOutput); + Assert.Contains("## Statistics per Operating System", firstOutput); + Assert.Contains("## Statistics per Namespace", firstOutput); + Assert.Contains("Demo.Namespace", firstOutput); + + writer.GetStringBuilder().Clear(); + stats.Print(); + + Assert.Equal(string.Empty, writer.ToString()); + } + finally + { + Console.SetOut(originalOut); + } + } + + [Fact] + public void RecordThrowsForUnsupportedConclusion() + { + var stats = new global::ResultsComparer.Stats(); + var environment = new HostEnvironmentInfo { Architecture = "X64", OsVersion = "Windows 11 (10.0.26100)" }; + var benchmark = new Benchmark { Statistics = new Statistics { OriginalValues = [1.0], N = 1, Median = 1.0 }, Memory = new Memory() }; + + Assert.Throws(() => stats.Record((EquivalenceTestConclusion)999, environment, benchmark)); + } +} diff --git a/src/tools/ResultsComparer/Data.cs b/src/tools/ResultsComparer/Data.cs index 3a61ad01a03..a663f1e8f63 100644 --- a/src/tools/ResultsComparer/Data.cs +++ b/src/tools/ResultsComparer/Data.cs @@ -130,6 +130,9 @@ static Version GetVersion(BdnResult bdnResult) private static string GetMoniker(string key) { + if (string.IsNullOrEmpty(key)) + return null; + if (key.Contains("net6")) // some files are net6.0, some are missing the dot (net60) return "net6.0"; if (key.Contains("nativeaot6")) @@ -158,14 +161,14 @@ private static string GetMoniker(string key) return "nativeaot9.0-preview" + key[key.IndexOf("nativeaot9.0-preview") + "nativeaot9.0-preview".Length]; if (key.Contains("net9.0")) return "net9.0"; - if (key.StartsWith("net10.0")) + if (key.Contains("net10.0")) return "net10.0"; - if (key.StartsWith("nativeaot10.0")) - return key; - if (key.StartsWith("net11.0")) + if (key.Contains("nativeaot10.0")) + return "nativeaot10.0"; + if (key.Contains("net11.0")) return "net11.0"; - if (key.StartsWith("nativeaot11.0")) - return key; + if (key.Contains("nativeaot11.0")) + return "nativeaot11.0"; return null; } diff --git a/src/tools/ResultsComparer/Directory.Packages.props b/src/tools/ResultsComparer/Directory.Packages.props index 28ef11f8a62..d910ce303ad 100644 --- a/src/tools/ResultsComparer/Directory.Packages.props +++ b/src/tools/ResultsComparer/Directory.Packages.props @@ -5,7 +5,7 @@ - + diff --git a/src/tools/ResultsComparer/MultipleInputsComparer.cs b/src/tools/ResultsComparer/MultipleInputsComparer.cs index 986a879e2b8..ba3ee7656ed 100644 --- a/src/tools/ResultsComparer/MultipleInputsComparer.cs +++ b/src/tools/ResultsComparer/MultipleInputsComparer.cs @@ -141,13 +141,15 @@ static long GetMetricValue(Benchmark result) var baseValues = info.baseResult.Statistics.OriginalValues; var diffValues = info.diffResult.Statistics.OriginalValues; - var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); + var userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); + var userConclusion = userThresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userThresholdResult.Conclusion; + var noiseConclusion = noiseResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : noiseResult.Conclusion; // filter noise (0.20 ns vs 0.25ns is 25% difference) - var conclusion = userTresholdResult.Conclusion != EquivalenceTestConclusion.Same && noiseResult.Conclusion == EquivalenceTestConclusion.Same + var conclusion = userConclusion != EquivalenceTestConclusion.Same && noiseConclusion == EquivalenceTestConclusion.Same ? Stats.Noise - : userTresholdResult.Conclusion == EquivalenceTestConclusion.Base ? EquivalenceTestConclusion.Same : userTresholdResult.Conclusion; + : userConclusion; stats.Record(conclusion, info.baseEnv, info.baseResult); diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs index 8bb8d1294cf..92620cb3470 100644 --- a/src/tools/ResultsComparer/Program.cs +++ b/src/tools/ResultsComparer/Program.cs @@ -172,7 +172,9 @@ private static bool TryGetPaths(DirectoryInfo input, string basePattern, string } private static Regex[] GetFilters(string[] filters) - => filters.Select(pattern => new Regex(WildcardToRegex(pattern), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); + => (filters ?? Array.Empty()) + .Select(pattern => new Regex(WildcardToRegex(pattern), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + .ToArray(); // https://stackoverflow.com/a/6907849/5852046 not perfect but should work for all we need private static string WildcardToRegex(string pattern) => $"^{Regex.Escape(pattern).Replace(@"\*", ".*").Replace(@"\?", ".")}$"; diff --git a/src/tools/ResultsComparer/Properties/AssemblyInfo.cs b/src/tools/ResultsComparer/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..11fac6ab430 --- /dev/null +++ b/src/tools/ResultsComparer/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ResultsComparer.Tests")] diff --git a/src/tools/ResultsComparer/ResultsComparer.sln b/src/tools/ResultsComparer/ResultsComparer.sln index 951a4d0fb5d..5985914bae2 100644 --- a/src/tools/ResultsComparer/ResultsComparer.sln +++ b/src/tools/ResultsComparer/ResultsComparer.sln @@ -2,15 +2,44 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer", "ResultsComparer.csproj", "{00859394-44F8-466B-8624-41578CA94009}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ResultsComparer.Tests", "..\ResultsComparer.Tests\ResultsComparer.Tests.csproj", "{F1CA7160-F99B-484F-975E-04C4638BCFAF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Debug|Any CPU.Build.0 = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x64.ActiveCfg = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x64.Build.0 = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x86.ActiveCfg = Debug|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Debug|x86.Build.0 = Debug|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.ActiveCfg = Release|Any CPU {00859394-44F8-466B-8624-41578CA94009}.Release|Any CPU.Build.0 = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x64.ActiveCfg = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x64.Build.0 = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x86.ActiveCfg = Release|Any CPU + {00859394-44F8-466B-8624-41578CA94009}.Release|x86.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x64.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x64.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x86.ActiveCfg = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Debug|x86.Build.0 = Debug|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|Any CPU.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x64.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x64.Build.0 = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x86.ActiveCfg = Release|Any CPU + {F1CA7160-F99B-484F-975E-04C4638BCFAF}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection EndGlobal diff --git a/src/tools/ResultsComparer/Stats.cs b/src/tools/ResultsComparer/Stats.cs index e230c0478a4..4e0a472c33f 100644 --- a/src/tools/ResultsComparer/Stats.cs +++ b/src/tools/ResultsComparer/Stats.cs @@ -77,7 +77,7 @@ static void Print(Dictionary dictionary, string name) } } - internal static string GetSimplifiedOSName(string text) => text.Split('(')[0]; + internal static string GetSimplifiedOSName(string text) => text.Split('(')[0].TrimEnd(); private class PerConclusion { diff --git a/src/tools/ResultsComparer/TwoInputsComparer.cs b/src/tools/ResultsComparer/TwoInputsComparer.cs index 89209aefb58..a929e937d27 100644 --- a/src/tools/ResultsComparer/TwoInputsComparer.cs +++ b/src/tools/ResultsComparer/TwoInputsComparer.cs @@ -35,15 +35,17 @@ internal static void Compare(TwoInputsOptions args) var baseValues = baseResult.Statistics.OriginalValues.ToArray(); var diffValues = diffResult.Statistics.OriginalValues.ToArray(); - var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); - if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same) + var userThresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.StatisticalTestThreshold); + if (userThresholdResult.Conclusion == EquivalenceTestConclusion.Same + || userThresholdResult.Conclusion == EquivalenceTestConclusion.Base) continue; var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, args.NoiseThreshold); - if (noiseResult.Conclusion == EquivalenceTestConclusion.Same) + if (noiseResult.Conclusion == EquivalenceTestConclusion.Same + || noiseResult.Conclusion == EquivalenceTestConclusion.Base) continue; - yield return (id, baseResult, diffResult, userTresholdResult.Conclusion); + yield return (id, baseResult, diffResult, userThresholdResult.Conclusion); } } diff --git a/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs b/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs index 83ceb5c79c0..e94ceaa3b0d 100644 --- a/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs +++ b/src/tools/ScenarioMeasurement/Startup.Tests/StartupTests.cs @@ -3,7 +3,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Security.Principal; using System.Threading; +using System.Runtime.Versioning; using Xunit; @@ -131,10 +133,27 @@ public sealed class WindowsOnly : FactAttribute { public WindowsOnly() { - if (Environment.OSVersion.Platform != PlatformID.Win32NT) + if (!OperatingSystem.IsWindows()) { Skip = "Skip on non-windows platform"; } + else if (!IsRunningAsAdministrator()) + { + Skip = "Requires administrator privileges to start ETW sessions"; + } + } + + [SupportedOSPlatform("windows")] + private static bool IsRunningAsAdministrator() + { + if (!OperatingSystem.IsWindows()) + { + return false; + } + + using WindowsIdentity identity = WindowsIdentity.GetCurrent(); + WindowsPrincipal principal = new(identity); + return principal.IsInRole(WindowsBuiltInRole.Administrator); } } @@ -142,7 +161,7 @@ public sealed class LinuxOnly : FactAttribute { public LinuxOnly() { - if(Environment.OSVersion.Platform != PlatformID.Unix) + if (!OperatingSystem.IsLinux()) { Skip = "Skip on non-linux platform"; }