Re run whole suite / not failed ones

Hi there!

• Operating system and version
  Jasmine version: 2.4.1
  Browser name: chrome
  Browser version: 53.0.2785.143
  Platform: LINUX
  Javascript enabled: true
  Css selectors enabled: true
• Node.js version - v8.1.2
• Protractor version - ~4.0.10
• Protractor flake version : "^3.0.1"
• Protractor configuration file :
  flake :
  #!/usr/bin/env node

/**

• flake is a node script that uses protractor-flake to re-run failed tests. Note
• that it reruns tests at the file level, so if one test fails, it will rerun all
• the tests in that file. Still... awesome.
• usage:
• ./flake conf.js [other protractor args]
  */
  //const rerun = require('./module');
  const protractorFlake = require('protractor-flake');

// skip first two passed args (node and self)
let protractorArgs = process.argv.splice(2);
protractorFlake({
protractorPath: 'node_modules/.bin/protractor',
maxAttempts: 2,
parser: 'standard', //even tried with multi
nodeBin: 'node',
color: 'magenta',
protractorArgs: protractorArgs
}, function(status, output) {
process.exit(status);
});

Package.json :

{
"version": "0.0.0",
"private": true,
"name": "weardex-app-bazooka-test",
"description": "weardex",
"license": "MIT",
"devDependencies": {
"jasmine-reporters": "~1.0.1",
"jasmine-spec-reporter": "^4.2.1",
"moment": "~2.9.0",
"mysql": "~2.8.0",
"protractor": "~4.0.10",
"protractor-console": "^3.0.0",
"protractor-html-screenshot-reporter": "0.0.19",
"protractor-jasmine2-screenshot-reporter": "^0.4.0"
},
"scripts": {
"postinstall": "node_modules/protractor/bin/webdriver-manager update",
"prestart": "npm install",
"start": "node_modules/protractor/bin/webdriver-manager start",
"pretest": "npm install",
"test": "./flake conf.js",
"debug": "npm install;node_modules/protractor/bin/protractor debug config.js",
"update-index-async": "node -e "require('shelljs/global'); sed('-i', /\/\/@@NG_LOADER_START@@[\s\S]*\/\/@@NG_LOADER_END@@/, '//@@NG_LOADER_START@@\n' + cat('bower_components/angular-loader/angular-loader.min.js') + '\n//@@NG_LOADER_END@@', 'app/index-async.html');""
},
"dependencies": {
"node-redshift": "0.0.6",
"protractor-flake": "^3.0.1"
}
}
conf.js :

/**

• Protractor config file for running LIVE test #cases
  */
  var path = require('path');
  var SpecReporter = require('jasmine-spec-reporter').SpecReporter;
  var reporter = new SpecReporter({
  spec: {
  displayStackTrace: true;
  }
  })
  // var HtmlScreenshotReporter = require('protractor-jasmine2-screenshot-reporter');
  // var reporter = new HtmlScreenshotReporter({
  // dest: 'screenshots',
  // filename: 'Dashboard_E2E_Results.html',
  // showConfiguration: true,
  // reportFailedUrl: true,
  // inlineImages: true,
  // captureOnlyFailedSpecs: true,
  // reportOnlyFailedSpecs: false,
  // showSummary: true,
  // showQuickLinks: true,
  // reportTitle: "SUMMARY_OF_FAILED_SPECS_E2E_RESULTS",
  // metadataBuilder: function(currentSpec, suites, browserCapabilities) {
  // return { id: currentSpec.id, os: browserCapabilities.get('browserName') };
  // },
  // pathBuilder: function(currentSpec, suites, browserCapabilities) {
  // // will return chrome/your-spec-name.png
  // return browserCapabilities.get('browserName') + '/' + currentSpec.fullName;
  // }
  // });

exports.config = {
// seleniumAddress: 'http://0.0.0.0:4444/wd/hub',
// seleniumAddress: 'http://localhost:4444/wd/hub',
// CSS Selector for the element housing the angular app - this defaults to
// body, but is necessary if ng-app is on a descendant of .
rootElement: 'body',
allScriptsTimeout: 60000,
framework: 'jasmine',
jasmineNodeOpts: {
// If true, display spec names.
isVerbose: true,
// If true, print colors to the terminal.
showColors: true,
// If true, include stack traces in failures.
includeStackTrace: true,
// Default time to wait in ms before a test fails.
defaultTimeoutInterval: 60000,
print: function();
},
params : {
// lang : 'zh_CN'
lang : 'en'
},

    baseUrl: 'https://dashboard.visenze.com',
    multiCapabilities: [
        //{'browserName': 'firefox'},
         {'specs': ['tests/regression/ftpUpload.js'],
         'browserName': 'chrome',
         },
         // {'specs': ['tests/regression/uploadAndTag.js'],
         // 'browserName': 'chrome',
         // },
         {'specs':['tests/regression/forgetPassword.js','tests/regression/changePassword.js','tests/regression/updateProfile.js','tests/regression/application.js'],
         'browserName': 'chrome',
         },
         {'specs':['tests/regression/teamMember/adminOperations.js','tests/regression/teamMember/powerUserOperations.js','tests/regression/teamMember/readOnlyUserOperations.js'],
         'browserName': 'chrome',
         },
         {'specs':['tests/regression/permission.js','tests/regression/overview.js','tests/regression/apiIntegration.js','tests/regression/clientSideIntegration.js'],
         'browserName': 'chrome',
         },
         {'specs': ['tests/regression/systemManagement2.js','tests/regression/urlRouting/userOperations.js'],
          'browserName': 'chrome'
         },
         {'specs': ['tests/regression/uploadDatafeed.js','tests/regression/schemaEdit.js','tests/regression/search.js'],
           'browserName': 'chrome'
         },
//        {'specs': ['tests/regression/gettingStarted.js'],
//                  'browserName': 'chrome'
//        },
    ],
     plugins: [{
     package: 'protractor-console',
     logLevels: ['severe']
    }],
    // Maximum number of total browser sessions to run. Tests are queued in
    // sequence if number of browser sessions is limited by this parameter.
    // Use a number less than 1 to denote unlimited. Default is unlimited.
//    maxSessions: 1,


    // onPrepare: function(){
    //     // Add a screenshot reporter and set location for storing screenshots
    //     var now = new Date();
    //     var timeStamp = now.getFullYear() + '-' + (now.getMonth() + 1) + '-' + now.getDate() + '-' + now.getHours() + 'h' + now.getMinutes() + 'm' + now.getSeconds()+'s';
    //     var HtmlReporter = require('protractor-html-screenshot-reporter');
//
//        jasmine.getEnv().addReporter(new HtmlReporter({
//            baseDirectory: './test_results/screenshot_reports',
//            pathBuilder: function pathBuilder(spec, descriptions, results, capabilities) {
//                return path.join(capabilities.caps_.browserName, descriptions.join('_'));
//            },
//            docName: 'html_screenshots_test_report_' + timeStamp + '.html',
//            takeScreenShotsOnlyForFailedSpecs: true
//        }));
        // require('jasmine-reporters');
        // jasmine.getEnv().addReporter(new jasmine.JUnitXmlReporter('./test_results/xml_reports', true, true, ''));
 
  //  }
  //  Setup the report before any tests start
// beforeLaunch: function () {
//     return new Promise(function (resolve) {
//         reporter.beforeLaunch(resolve);
//     });
// },


// // Close the report after all tests finish
// afterLaunch: function (exitCode) {
//     return new Promise(function (resolve) {
//         reporter.afterLaunch(resolve.bind(this, exitCode));
//     });
// },
//Onprepare function
onPrepare: function () {
    var width = 1300;
    var height = 1200;
    browser.driver.manage().window().maximize();
    browser.driver.manage().timeouts().implicitlyWait(20000);
    browser.driver.manage().window().setSize(width, height);
    jasmine.getEnv().addReporter(reporter);
    afterAll(function (done) {
        process.nextTick(done);
    });
}


};

You can collect the first few items easily with the trouble cli npx @nicktomlin/trouble protractor protractor-flake (or npm i -g @nicktomlin/trouble && trouble protractor protractor-flake)

1. Even i have comment out jasmine 2 screenshot reporter thinking the screenshot will have impact . How can i use screenshot reporter as well as flake together.

2. Does this lead to browser hang ? Because i saw couple of times browser hanging not sure anyone facing same issue

@NickTomlin : Would you please suggest me work around to resolve this issue

[NickTomlin]

@trisha1212 I'm confused by your output. I see multiple invocations of npm test and there are multiple parsers being used here:

Using standard to parse output
Re-running tests: test attempt 2

 Re-running tests: test attempt 2
 Re-running the following test files:
 /Users/trishachetani/Desktop/automation/weardex-app-bazooka-test/tests/regression/ftpUpload.js

It looks like the multi parser is finding a failing spec file and re-running it. Can we focus on a simplified example using only the multi parser? Otherwise it's difficult to debug because there is a lot of output.

[NickTomlin]

@trisha1212 I'm confused by your output. I see multiple invocations of npm test and there are multiple parsers being used here:

Using standard to parse output
Re-running tests: test attempt 2

 Re-running tests: test attempt 2
 Re-running the following test files:
 /Users/trishachetani/Desktop/automation/weardex-app-bazooka-test/tests/regression/ftpUpload.js

It looks like the multi parser is finding a failing spec file and re-running it. Can we focus on a simplified example using only the multi parser? Otherwise it's difficult to debug because there is a lot of output.

[surendraJ]

@trisha1212 You have lot of logs and really painful to read all those properly. You can test against few files with few spec with minimum logs so that other can read properly and help you. And configuration file also looks unreadable due to your comment and improper format...

Thank you .

I reran the test and attached the gist.

https://gist.github.com/Trisha1212/929f8de55e6fc9ea91eb5db69102d1b1
Flake with multi as parser : This time its try rerunning one specs multiple times not all failed test .

I again rerun the test attached the gist
https://gist.github.com/Trisha1212/d1e588e34907abbc9216096a831aa367
Flake with standard as parser : This time whole test suite re run instead of all failed test.

@NickTomlin , @surendraJ : Please let me know any other details helps you i will try to provide from my side.

Your help is appreciated.

[surendraJ]

@trisha1212 only thing I can say..protractor flake's parser are not sufficient to match all of the log to re run again...we need to have custom parser to match all of them.

You can check with pattern here: https://github.com/NickTomlin/protractor-flake/blob/master/src/parsers/multi.js

This issue might be useful: #15

[NickTomlin]

@trisha1212 As @surendraJ mentions, the multi parser is looking for output to attempt to match specs being run against any failures. First it looks for

1. Look for ------------------------------------ lines (which correspond to a sharded test run)
2. In each of these groups
   • look for Specs:, use these as the specfile being run
   • Look for 0 failures. If this is found mark the group as passed. Otherwise, use the files we gathered from Sepcs: above and re-run

If no files are found and we exit with a non 0 statusCode, we re-run the entire suite.

With that context established, it looks like some multiCapabilities items are logging differently. E.g ftpUpload.js correctly logs the spec name but the preceeding group does not

For an experiment can you change one of the specs: [] items with more than one entry to a single entry?

An additional question, why do you need multiple multiCapabilities entries for the same browser?

[rvowles]

I am seeing this in 2.6 - it appears that protractor-flake is adding items to "capabilities" and not null-ing multiCapabilities, so protractor just ignores the capabilities section.

`[cd11] Using cucumberMulti to parse output

[cd11] Re-running tests: test attempt 2

[cd11] Re-running the following test files:

[cd11] business-process/event-e2e/features/portal-william/williamDownloadsGeraldineFiles.feature

[cd11] Import and download dirs: /var/lib/jenkins_home/workspace/XXX-build_.....

[cd11] 2017-11-17T09:59:34.838Z (node:25291) [DEP0022] DeprecationWarning: os.tmpDir() is deprecated. Use os.tmpdir() instead.

[cd11] 2017-11-17T09:59:34.840Z [09:59:34] W/launcher - You have specified both capabilities and multiCapabilities. This will result in capabilities being ignored

[cd11] 2017-11-17T09:59:34.856Z [09:59:34] I/launcher - Running 2 instances of WebDriver`

[rvowles]

And that is because of this problem: angular/protractor#2231

feel free to ignore my comment :-) i can code around this!

Following on the tickets :
I tried with

• capabilities/multi capabilities
• standard parser /multi parser
• direct connect /selenium grid

issue is still reproducible :
Test Run logs : https://pastebin.com/t1UmpVSu ( logs are for standard and capabilities)
conf.js : https://pastebin.com/StM4uvg4
package.json
{
"name": "dashboard-new-admin-e2e",
"version": "1.0.0",
"main": "index.js",
"license": "MIT",
"scripts": {
"webdriver-update": "webdriver-manager update",
"test": "protractor-flake -- conf.js"
},
"devDependencies": {
"protractor": "^5.2.2",
"protractor-console": "^3.0.0"
},
"dependencies": {
"jasmine-spec-reporter": "^4.2.1",
"moment": "^2.20.0",
"mysql": "~2.8.0",
"protractor-beautiful-reporter": "^1.1.1",
"protractor-flake": "^3.0.1"
}
}

[rvowles]

if it is of any use, to make this work we use the following at the end of our config files. It checks if it is running inside a flake retry and if so, drops the multi capabilities. It also forces stdout/err to blocking because otherwise the results from the longer run often don't make it back to Flake.

console.log('looking for retry args from flake:');
process.argv.forEach(function (val, index, array) {
console.log('protractor param ', index + ': ' + val);

if (val === '--params.flake.retry') {
    console.log('found so removing multi capabilities.');
    delete exports.config.multiCapabilities;
}

});

process.stdout._handle.setBlocking(true);
process.stderr._handle.setBlocking(true);

[anandhi-credible]

@rvowles Your solution worked for me when rerunning flaky tests with multi capabilities. Thank you so much