Skip to content

Commit d23bf8f

Browse files
mikaelweavemikaelweave
authored
[Pipelines] Add Integration/E2E Retry (#3642)
* Added trx logging and test publishing * Fixed test reporting path * changed to vstest * rollback build change * Fixed newline issue * Fixed export tests to use VSTest * Changed over last dotnet test * Fixed export tests * pipeline test fix * formatting fix * Fixed test filter typo * pinning vstest to fix issue * trying different version of vstest * Testing batch size to fix data test issues * Testing custom batch size fix * removed vstest batching * Testing changing the TestClassWithFixtureArgumentsTypeInfo name * testing blame timeout * Testing new blame params * Changing SQL exception number * Removed restriction for testing message only in Win * Added publishing of test files on failure on error to help debugging * Updated upload test failure task * Fixed integration test artifact name * Added fix for sql test * Fox normalize in SqlServerSchemaUpgradeTests * Fixed SQL int test newline integration * Updated dump type / timeout for tests * Reduced blame back down to 7 * Added pool name to windows jobs * Changing to correct windows pool
1 parent 56abac0 commit d23bf8f

File tree

7 files changed

+102
-41
lines changed

7 files changed

+102
-41
lines changed

‎build/build-variables.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ variables:
3131
LinuxVmImage: 'ubuntu-latest'
3232
TestApplicationResource: 'https://$(DeploymentEnvironmentName).$(tenantDomain)'
3333
# The following is set by a build Pipeline variable:
34-
# DefaultLinuxPool: 'Azure Pipelines'
35-
# SharedLinuxPool: 'Azure Pipelines'
34+
# DefaultLinuxPool: 'DevOps1ESSharedPool'
35+
# SharedLinuxPool: 'DevOps1ESSharedPool'
36+
# DefaultWindowsPool: 'mshealthagents'
3637
#-----------------------------------------------------------------------------------------

‎build/jobs/e2e-tests.yml

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,13 +94,18 @@ steps:
9494

9595
dotnet dev-certs https
9696

97-
- task: DotNetCoreCLI@2
97+
- task: VSTest@3
9898
displayName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}}'
9999
inputs:
100-
command: test
101-
arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --blame-hang-timeout 7m --filter "FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning"'
102-
workingDirectory: "$(System.ArtifactsDirectory)"
100+
testSelector: 'testAssemblies'
101+
testAssemblyVer2: '**/*${{ parameters.version }}.Tests.E2E*.dll'
102+
searchFolder: '$(Agent.TempDirectory)/E2ETests/'
103+
testFiltercriteria: 'FullyQualifiedName~${{parameters.appServiceType}}&Category!=ExportLongRunning'
103104
testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}'
105+
otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
106+
rerunFailedTests: true
107+
rerunType: 'basedOnTestFailurePercentage'
108+
rerunFailedThreshold: 10
104109
env:
105110
'TestEnvironmentUrl': $(TestEnvironmentUrl)
106111
'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
@@ -137,3 +142,12 @@ steps:
137142
'user_globalReaderUser_secret': $(user_globalReaderUser_secret)
138143
'user_globalWriterUser_id': $(user_globalWriterUser_id)
139144
'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
145+
146+
# Publish the crash dump and XML files as build artifacts on failure only
147+
- task: PublishPipelineArtifact@1
148+
displayName: 'Publish E2E ${{ parameters.version }} ${{parameters.appServiceType}} Failures'
149+
condition: failed()
150+
continueOnError: true
151+
inputs:
152+
targetPath: '$(Agent.TempDirectory)/TestResults'
153+
artifactName: 'E2E ${{ parameters.version }} ${{parameters.appServiceType}} Test Failures'

‎build/jobs/run-export-tests.yml

Lines changed: 40 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ jobs:
88
- job: 'cosmosE2eTests'
99
dependsOn: []
1010
pool:
11-
name: '$(DefaultLinuxPool)'
12-
vmImage: '$(LinuxVmImage)'
11+
name: '$(DefaultWindowsPool)'
1312
steps:
1413
- template: e2e-setup.yml
1514
- template: e2e-tests-extract.yml
@@ -72,13 +71,18 @@ jobs:
7271

7372
dotnet dev-certs https
7473

75-
- task: DotNetCoreCLI@2
74+
- task: VSTest@3
7675
displayName: 'Export E2E ${{ parameters.version }} CosmosDB'
7776
inputs:
78-
command: test
79-
arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --filter "FullyQualifiedName~CosmosDb&Category=ExportLongRunning"'
80-
workingDirectory: "$(System.ArtifactsDirectory)"
81-
testRunTitle: 'Export ${{ parameters.version }} CosmosDB'
77+
testSelector: 'testAssemblies'
78+
testAssemblyVer2: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll"'
79+
searchFolder: '$(System.ArtifactsDirectory)'
80+
testFiltercriteria: 'FullyQualifiedName~CosmosDb&Category=ExportLongRunning'
81+
testRunTitle: '${{ parameters.version }} CosmosDB Export'
82+
otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
83+
rerunFailedTests: true
84+
rerunType: 'basedOnTestFailurePercentage'
85+
rerunFailedThreshold: 10
8286
env:
8387
'TestEnvironmentUrl': $(TestEnvironmentUrl)
8488
'TestEnvironmentUrl_${{ parameters.version }}': $(TestEnvironmentUrl_${{ parameters.version }})
@@ -106,11 +110,19 @@ jobs:
106110
'user_globalWriterUser_id': $(user_globalWriterUser_id)
107111
'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
108112

113+
# Publish the crash dump and XML files as build artifacts on failure only
114+
- task: PublishPipelineArtifact@1
115+
displayName: 'Publish Export E2E Cosmos Test Failures'
116+
condition: failed()
117+
continueOnError: true
118+
inputs:
119+
targetPath: '$(Agent.TempDirectory)/TestResults'
120+
artifactName: 'Export E2E Cosmos Test Failures'
121+
109122
- job: 'sqlE2eTests'
110123
dependsOn: []
111124
pool:
112-
name: '$(DefaultLinuxPool)'
113-
vmImage: '$(LinuxVmImage)'
125+
name: '$(DefaultWindowsPool)'
114126
steps:
115127
- template: e2e-setup.yml
116128
- template: e2e-tests-extract.yml
@@ -173,13 +185,18 @@ jobs:
173185

174186
dotnet dev-certs https
175187

176-
- task: DotNetCoreCLI@2
188+
- task: VSTest@3
177189
displayName: 'Export E2E ${{ parameters.version }} SQL'
178190
inputs:
179-
command: test
180-
arguments: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll" --filter "FullyQualifiedName~SqlServer&Category=ExportLongRunning"'
181-
workingDirectory: "$(System.ArtifactsDirectory)"
182-
testRunTitle: 'Export ${{ parameters.version }} SQL'
191+
testSelector: 'testAssemblies'
192+
testAssemblyVer2: '"$(Agent.TempDirectory)/E2ETests/**/*${{ parameters.version }}.Tests.E2E*.dll"'
193+
searchFolder: '$(System.ArtifactsDirectory)'
194+
testFiltercriteria: 'FullyQualifiedName~SqlServer&Category=ExportLongRunning'
195+
testRunTitle: '${{ parameters.version }} SQL Export'
196+
otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
197+
rerunFailedTests: true
198+
rerunType: 'basedOnTestFailurePercentage'
199+
rerunFailedThreshold: 10
183200
env:
184201
'TestEnvironmentUrl_Sql': $(TestEnvironmentUrl_Sql)
185202
'TestEnvironmentUrl_${{ parameters.version }}_Sql': $(TestEnvironmentUrl_${{ parameters.version }}_Sql)
@@ -206,4 +223,12 @@ jobs:
206223
'user_globalReaderUser_secret': $(user_globalReaderUser_secret)
207224
'user_globalWriterUser_id': $(user_globalWriterUser_id)
208225
'user_globalWriterUser_secret': $(user_globalWriterUser_secret)
209-
226+
227+
# Publish the crash dump and XML files as build artifacts on failure only
228+
- task: PublishPipelineArtifact@1
229+
displayName: 'Publish Export E2E SQL Test Failures'
230+
condition: failed()
231+
continueOnError: true
232+
inputs:
233+
targetPath: '$(Agent.TempDirectory)/TestResults'
234+
artifactName: 'Export E2E SQL Test Failures'

‎build/jobs/run-tests.yml

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ parameters:
88
jobs:
99
- job: "integrationTests"
1010
pool:
11-
name: '$(SharedLinuxPool)'
12-
vmImage: '$(LinuxVmImage)'
11+
name: '$(DefaultWindowsPool)'
1312
steps:
1413
- task: DownloadBuildArtifacts@0
1514
inputs:
@@ -40,23 +39,35 @@ jobs:
4039
azureSubscription: $(ConnectedServiceName)
4140
KeyVaultName: '${{ parameters.keyVaultName }}-sql'
4241

43-
- task: DotNetCoreCLI@2
42+
- task: VSTest@3
4443
displayName: 'Run Integration Tests'
4544
inputs:
46-
command: test
47-
arguments: '"$(Agent.TempDirectory)/IntegrationTests/**/*${{ parameters.version }}.Tests.Integration*.dll" --blame-hang-timeout 15m'
48-
workingDirectory: "$(System.ArtifactsDirectory)"
45+
testSelector: 'testAssemblies'
46+
testAssemblyVer2: '**/*${{ parameters.version }}.Tests.Integration*.dll'
47+
searchFolder: '$(Agent.TempDirectory)/IntegrationTests/'
4948
testRunTitle: '${{ parameters.version }} Integration'
49+
otherConsoleOptions: '/Blame:"CollectDump;DumpType=Mini;CollectHangDump;TestTimeout=7min;HangDumpType=Mini"'
50+
rerunFailedTests: true
51+
rerunType: 'basedOnTestFailurePercentage'
52+
rerunFailedThreshold: 10
5053
env:
5154
'CosmosDb:Host': $(CosmosDb--Host)
5255
'CosmosDb:Key': $(CosmosDb--Key)
5356
'SqlServer:ConnectionString': $(SqlServer--ConnectionString)
5457

58+
# Publish the crash dump and XML files as build artifacts on failure only
59+
- task: PublishPipelineArtifact@1
60+
displayName: 'Publish Integration Test Failures'
61+
condition: failed()
62+
continueOnError: true
63+
inputs:
64+
targetPath: '$(Agent.TempDirectory)/TestResults'
65+
artifactName: '${{ parameters.version }} Integration Test Failures'
66+
5567
- job: 'cosmosE2eTests'
5668
dependsOn: []
5769
pool:
58-
name: '$(SharedLinuxPool)'
59-
vmImage: '$(LinuxVmImage)'
70+
name: '$(DefaultWindowsPool)'
6071
steps:
6172
- template: e2e-setup.yml
6273
- template: e2e-tests.yml
@@ -68,8 +79,7 @@ jobs:
6879
- job: 'sqlE2eTests'
6980
dependsOn: []
7081
pool:
71-
name: '$(SharedLinuxPool)'
72-
vmImage: '$(LinuxVmImage)'
82+
name: '$(DefaultWindowsPool)'
7383
steps:
7484
- template: e2e-setup.yml
7585
- template: e2e-tests.yml

‎src/Microsoft.Health.Extensions.Xunit/TestClassWithFixtureArgumentsTypeInfo.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public TestClassWithFixtureArgumentsTypeInfo(ITypeInfo typeInfoImplementation, I
2727

2828
_typeInfoImplementation = typeInfoImplementation;
2929
FixtureArguments = fixtureArguments;
30-
Name = $"{typeInfoImplementation.Name}({string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))})";
30+
Name = $"{typeInfoImplementation.Name}[{string.Join(", ", fixtureArguments.Select(v => $"{v.EnumValue}"))}]";
3131
}
3232

3333
public IReadOnlyList<SingleFlag> FixtureArguments { get; }

‎test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ private bool IsConnectionFailedException(Exception ex, bool testConnectionInitia
306306
// On Windows we get correct error number.
307307
return true;
308308
}
309-
else if (sqlEx.Number == 0 && sqlEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
309+
else if (sqlEx.Message.Contains("connection", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
310310
{
311311
// On Linux we get 0 error number for various connection problems so we check message string as well.
312312
return true;
@@ -319,7 +319,7 @@ private bool IsConnectionFailedException(Exception ex, bool testConnectionInitia
319319
// On Windows we get correct error number.
320320
return true;
321321
}
322-
else if (sqlEx.Number == 0 && sqlEx.Message.Contains("transport", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
322+
else if (sqlEx.Message.Contains("transport", StringComparison.OrdinalIgnoreCase) && sqlEx.Message.Contains("error", StringComparison.OrdinalIgnoreCase))
323323
{
324324
// On Linux we get 0 error number for various connection problems so we check message string as well.
325325
return true;

‎test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSchemaUpgradeTests.cs

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -331,19 +331,30 @@ private async Task<bool> IsStoredProcedureTextEqual(string connStr1, string conn
331331

332332
private string Normalize(string text)
333333
{
334+
// normalize newlines
335+
text = text.Replace("\r\n", "\n").Replace("\r", "\n");
336+
334337
// remove inline comments
335-
while (text.IndexOf("--") > 0)
338+
while (text.Contains("--"))
336339
{
337340
var indexStart = text.IndexOf("--");
338-
var indexEnd = text.IndexOf(Environment.NewLine, indexStart);
339-
text = text.Substring(0, indexStart) + text.Substring(indexEnd, text.Length - indexEnd);
341+
var indexEnd = text.IndexOf("\n", indexStart);
342+
343+
if (indexEnd == -1)
344+
{
345+
text = text.Substring(0, indexStart);
346+
}
347+
else
348+
{
349+
text = text.Substring(0, indexStart) + text.Substring(indexEnd + 1);
350+
}
340351
}
341352

342353
return text.ToLowerInvariant()
343-
.Replace(Environment.NewLine, " ")
344-
.Replace("\r", " ")
345-
.Replace("\t", " ")
346-
.Replace(";", " ")
354+
.Replace("\n", string.Empty)
355+
.Replace("\r", string.Empty)
356+
.Replace("\t", string.Empty)
357+
.Replace(";", string.Empty)
347358
.Replace(" output,", " out,")
348359
.Replace(" output ", " out ")
349360
.Replace(" inner join ", " join ")

0 commit comments

Comments
 (0)