diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..8ca517d6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "nuget" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/labeler.yml b/.github/labeler.yml index 9904e87d..97d707a4 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,19 +1,30 @@ +root: +- changed-files: + - any-glob-to-any-file: '*' + Configuration File: - - configurationFiles/** +- changed-files: + - any-glob-to-any-file: configurationFiles/** Executable: - - src/** +- changed-files: + - any-glob-to-any-file: src/** Kusto Query Language: - - kusto/** +- changed-files: + - any-glob-to-any-file: kusto/** WorkFlow: - - .github/** +- changed-files: + - any-glob-to-any-file: .github/** documentation: - - docs/*.* - - README.md - - CHANGELOG.md +- changed-files: + - any-glob-to-any-file: + - docs/*.* + - README.md + - CHANGELOG.md scripts: - - scripts/*.* \ No newline at end of file +- changed-files: + - any-glob-to-any-file: scripts/*.* \ No newline at end of file diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 445dccd9..e13ddf05 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,8 +6,11 @@ on: [pull_request_target] jobs: label: + permissions: + contents: read + pull-requests: write runs-on: windows-latest steps: - - uses: actions/labeler@v2 + - uses: actions/labeler@v5 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 175d3ba4..c771fcc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Change log +## 03/11/2024 + +- update kusto quickstart doc with modified image and minimum sas permissions +- update kusto quickstart doc with new exception message +- update kusto functions + +## 03/07/2024 + +- upgrade git action labeler@v5 + +## 02/07/2024 + +- add updates for SDL +- add dependabot config +- remove stale links in documentation +- update workflow for git for deprecated env variables +- update prod and dev pipelines azure pipelines for ado + ## 11/19/2023 - add support for .net8.0 diff --git a/azure-pipelines-dev.yml b/azure-pipelines-dev.yml index fd5f5a54..8db8aa99 100644 --- a/azure-pipelines-dev.yml +++ b/azure-pipelines-dev.yml @@ -1,6 +1,6 @@ # used for internal ado builds -name: ADO CI CollectSFData +name: ADO CI CollectSFData-Dev trigger: branches: @@ -15,16 +15,9 @@ variables: project_root: .\src start_time: $[format('{0}-{1:yyyy}{1:MM}{1:dd}-{1:HH}{1:mm}{1:ss}', variables['project_name'], pipeline.startTime)] artifacts_drive: Z - artifacts_share_target: Z:\$(System.TeamProject)/$(System.DefinitionName)/$(System.JobId) - artifacts_storage_uri_fileshare: /artifacts - System.Debug: true - buildConfiguration: Release - Codeql.PublishDatabaseLog: true - # testing - Codeql.Cadence: 0 - Codeql.LogLevel: 5 - Codeql.Language: csharp,powershell - Codeql.PublishDatabase: true + artifacts_share_target: 'Z:\$(System.DefinitionName)\$(System.JobId)\$(start_time)' + system.debug: true + buildConfiguration: debug steps: - task: PowerShell@2 @@ -32,7 +25,7 @@ steps: inputs: targetType: 'inline' script: | - [environment]::getenvironmentvariables().getenumerator()|sort Name + [environment]::getEnvironmentVariables().getEnumerator()|sort Name dotnet --info dotnet nuget locals all --clear errorActionPreference: 'continue' @@ -51,7 +44,12 @@ steps: - task: CodeQL3000Init@0 inputs: Enabled: true - AnalyzeInPipeline: true + AnalyzeInPipeline: false + PublishDatabase: true + PublishDatabaseLog: true + Language: csharp,powershell + Cadence: 72 # 72 hours default, use 0 for debug + LogLevel: 4 - task: PowerShell@2 displayName: 'dotnet build' @@ -112,25 +110,31 @@ steps: targetPath: $(System.ArtifactsDirectory) artifactName: artifacts-$(start_time)-$(system.JobId) -## not working - task: PowerShell@2 - displayName: 'map artifacts drive' + displayName: 'copy artifacts' inputs: targetType: 'inline' script: | - [environment]::getenvironmentvariables().getenumerator()|sort Name - write-host "Test-NetConnection -ComputerName $env:mapped_artifacts_storage_uri -Port 445 -informationLevel Detailed" - $connectTestResult = Test-NetConnection -ComputerName $env:mapped_artifacts_storage_uri -Port 445 -informationLevel Detailed - if ($connectTestResult.TcpTestSucceeded) { - # Save the password so the drive will persist on reboot - write-host "cmd.exe /C cmdkey /add:`"$env:mapped_artifacts_storage_uri/$env:artifacts_storage_uri_fileshare`" /user:`"$env:mapped_artifacts_user`" /pass:`"$env:mapped_artifacts_pass`"" - cmd.exe /C "cmdkey /add:`"$env:mapped_artifacts_storage_uri/$env:artifacts_storage_uri_fileshare`" /user:`"$env:mapped_artifacts_user`" /pass:`"$env:mapped_artifacts_pass`"" - # Mount the drive - New-PSDrive -Name $env:artifacts_drive -PSProvider FileSystem -Root "$env:mapped_artifacts_share" -Persist + [environment]::getEnvironmentVariables().getEnumerator()|sort Name + write-host "test-netConnection -computerName $env:mapped_artifacts_storage_uri -Port 445 -informationLevel detailed" + $connectTestResult = test-netConnection -computerName $env:mapped_artifacts_storage_uri -Port 445 -informationLevel detailed + if ($connectTestResult.tcpTestSucceeded) { + $securePassword = ConvertTo-SecureString -String $env:mapped_artifacts_pass -Force -AsPlainText + $credentials = [psCredential]::new($env:mapped_artifacts_user, $securePassword) + write-host "new-psDrive -name $drive -psProvider fileSystem -root "$env:mapped_artifacts_share" -credential $credentials" + new-psDrive -name $env:artifacts_drive -psProvider fileSystem -root "$env:mapped_artifacts_share" -credential $credentials -scope global } else { - Write-Error -Message "Unable to reach the Azure storage account via port 445. Check to make sure your organization or ISP is not blocking port 445, or use Azure P2S VPN, Azure S2S VPN, or Express Route to tunnel SMB traffic over a different port." + write-error -message "Unable to reach the azure storage account via port 445." } - mkdir "$env:artifacts_share_target" + if(!(test-path $env:artifacts_share_target)) + { + write-host "mkdir "$env:artifacts_share_target"" + mkdir "$env:artifacts_share_target" + } + write-host "copy $(System.DefaultWorkingDirectory)/src/bin/$(buildConfiguration) "$env:artifacts_share_target" -recurse" + copy $(System.DefaultWorkingDirectory)/src/bin/$(buildConfiguration) "$env:artifacts_share_target" -recurse + write-host "copy $(System.ArtifactsDirectory) "$env:artifacts_share_target" -recurse" + copy $(System.ArtifactsDirectory) "$env:artifacts_share_target" -recurse errorActionPreference: 'continue' verbosePreference: 'continue' debugPreference: 'continue' @@ -140,8 +144,10 @@ steps: mapped_artifacts_pass: $(artifacts_pass) mapped_artifacts_share: $(artifacts_share) -- task: DownloadPipelineArtifact@2 - inputs: - buildType: 'current' - artifactName: 'artifacts-$(start_time)-$(system.JobId)' - targetPath: '$(artifacts_share_target)' +# - task: DownloadPipelineArtifact@2 +# inputs: +# buildType: 'current' +# artifactName: 'artifacts-$(start_time)-$(system.JobId)' +# targetPath: '$(artifacts_share_target)' +# allowPartiallySucceededBuilds: true +# allowFailedBuilds: true diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bebeceaf..4221ed33 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,6 +6,7 @@ trigger: branches: include: - 'master' + - 'release*' pool: vmImage: 'windows-latest' @@ -14,25 +15,16 @@ variables: project_name: CollectSFData project_root: .\src start_time: $[format('{0}-{1:yyyy}{1:MM}{1:dd}-{1:HH}{1:mm}{1:ss}', variables['project_name'], pipeline.startTime)] - artifacts_drive: Z - artifacts_share_target: Z:\$(System.TeamProject)/$(System.DefinitionName)/$(System.JobId) - artifacts_storage_uri_fileshare: /artifacts - System.Debug: true - buildConfiguration: Release - Codeql.PublishDatabaseLog: true - # testing - Codeql.Cadence: 0 - Codeql.LogLevel: 5 - Codeql.Language: csharp,powershell - Codeql.PublishDatabase: true - + system.debug: false + buildConfiguration: release + steps: - task: PowerShell@2 displayName: 'agent environment' inputs: targetType: 'inline' script: | - [environment]::getenvironmentvariables().getenumerator()|sort Name + [environment]::getEnvironmentVariables().getEnumerator()|sort Name dotnet --info dotnet nuget locals all --clear errorActionPreference: 'continue' @@ -51,7 +43,12 @@ steps: - task: CodeQL3000Init@0 inputs: Enabled: true - AnalyzeInPipeline: true + AnalyzeInPipeline: false + PublishDatabase: false + PublishDatabaseLog: false + Language: csharp,powershell + Cadence: 72 # 72 hours default, use 0 for debug + LogLevel: 4 - task: PowerShell@2 displayName: 'dotnet build' diff --git a/docs/kustoQuickStart.md b/docs/kustoQuickStart.md index 66b8a02d..edddf4b8 100644 --- a/docs/kustoQuickStart.md +++ b/docs/kustoQuickStart.md @@ -53,6 +53,21 @@ From Azure portal https://portal.azure.com navigate to service fabric cluster re Once correct storage account is identified, select 'Shared access signature' and then 'Generate SAS and connection string'. Copy 'Blob service SAS URL' or 'Connection Sting'. This is the value that will be used for CollectSFData argument 'SasKey'. +Ensure at least the following permissions are selected: + +- Allowed Services + - Blob + - Table + +- Allowed Resource Types + - Service + - Container + - Object + +- Allowed Permissions + - Read + - List + - ![](media/azure.portal.3.png) - ![](media/azure.portal.4.png) @@ -459,6 +474,27 @@ pause ``` +4. Microsoft.Identity.Client.MsalServiceException. + Verify configuration settings are correct: + - azureClientId + - azureTenantId + - azureClientSecret + - azureClientCertificate + + .net framework will use ADAL and .net core will use MSAL. + If using .net framework (net462), use cmd.exe or powershell.exe to execute collectsfdata.exe. + If using .net core (net6.0+), use pwsh.exe (powershell core) to execute collectsfdata.exe. + +```text +Authenticate:exception: AggregateException:System.AggregateException: One or more errors occurred. ---> Microsoft.Identity.Client.MsalServiceException: +AADSTS50194: Application 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'(collectsfdata service fabric data collection) is not configured as a multi-tenant application. +Usage of the /common endpoint is not supported for such applications created after '10/15/2018'. +Use a tenant-specific endpoint or configure the application to be multi-tenant. +Trace ID: 952e918a-1414-4206-91bd-ad74eb1cbc00 +Correlation ID: 56e0075d-cc4f-4974-a651-0b25e82faaee +Timestamp: 2024-03-08 22:05:00Z +``` + ## reference ### kusto reference diff --git a/docs/media/azure.portal.3.png b/docs/media/azure.portal.3.png index ae7f7753..3f293a15 100644 Binary files a/docs/media/azure.portal.3.png and b/docs/media/azure.portal.3.png differ diff --git a/kusto/functions/sflogs/application/TraceApplicationUpgrade.csl b/kusto/functions/sflogs/application/TraceApplicationUpgrade.csl index 035ee0bf..852a100e 100644 --- a/kusto/functions/sflogs/application/TraceApplicationUpgrade.csl +++ b/kusto/functions/sflogs/application/TraceApplicationUpgrade.csl @@ -1,4 +1,4 @@ -.create-or-alter function with (docstring = "[T:string] where T=table name, [A:string] where App=application table name. function to search service fabric sflogs for application related entries", folder = "sflogs/application") +.create-or-alter function with (docstring = "[T:string] where T=table name, [A:string] where App=application name. function to search service fabric sflogs for application related entries", folder = "sflogs/application") TraceApplicationUpgrade(T:string, A:string) { let extractPattern_Context1 = @'(ApplicationUpgradeContext\()([^\)]*)\)\[([^,]*),\s([^,]*),\s([^,]*),\s([^,]*),\s([^\s]*)\s=\s([^,]*),\s([^\s]*)\s=\s'; let extractPattern_Context2 = @'(Monitoring = action:)([^,]*),\swait:([^,]*),\sstable:([^,]*),\sretry:([^,]*),\sud:([^,]*),\soverall:([^,]*)'; diff --git a/kusto/functions/sflogs/errors/TraceFabric_E_ErrorsDistinct.csl b/kusto/functions/sflogs/errors/TraceFabric_E_ErrorsDistinct.csl index deb0f275..efee75bf 100644 --- a/kusto/functions/sflogs/errors/TraceFabric_E_ErrorsDistinct.csl +++ b/kusto/functions/sflogs/errors/TraceFabric_E_ErrorsDistinct.csl @@ -7,7 +7,7 @@ | extend fabric_error = tostring(extract_all(@"(?PFABRIC_E\w+)", dynamic(['fabric_error']), Text)) | summarize count(fabric_error), First = arg_min(Timestamp, *), Last = arg_max(Timestamp, *) by fabric_error | order by fabric_error asc - | extend exception_pattern = replace("'","",replace(",",@".*",trim('\"|\\[|\\]',fabric_error))) + | extend exception_pattern = replace("'","",replace(",",@".*",trim(@'[\"\[\]]*',fabric_error))) | extend encoded_pattern = url_encode(strcat(T,'| where Text matches regex "',exception_pattern,'"')) | extend link = strcat(baseUri, encoded_pattern) | project fabric_error, Level, Count = count_fabric_error, ['First occurence'] = tostring(First), ['Last occurrence'] = iif(First == Last, "", tostring(Last)), link