Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion nightwatch/globals.js
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,12 @@ module.exports = {
// calls `browser.end()` in afterEach, all earlier sessions stay
// unmarked ("done") on the Automate dashboard. We use the same
// browserstack_executor action the user would call manually.
if (typeof browser !== 'undefined' && browser.sessionId) {
// LTS runs on its own grid (browserstackAutomation=false), where the
// executor string is not intercepted: Chrome evaluates it as JS and
// fails with "Unexpected token ':'", which then lands in the run's
// failure evidence as a script error. The LTS runner owns session
// status there, so skip the call.
if (typeof browser !== 'undefined' && browser.sessionId && !helper.isLoadTestingSession()) {
try {
const testName = test?.testcase;
const eventData = (testName && test?.envelope?.[testName]?.testcase) || null;
Expand Down
16 changes: 16 additions & 0 deletions src/testObservability.js
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,22 @@ class TestObservability {
}
}
}
// LTS-only: a step recorded by bstackStep() is the authoritative failure.
// The LCNC helper throws its own Error, which Nightwatch keeps out of the
// command list, so the first failed command is often an internal probe
// (e.g. an executeAsyncScript that hit "script timeout") whose result has
// no message or stack. That shipped `backtrace: ['', '']` and TRA had
// nothing to cluster; with no failed command at all it reported 'passed'.
if (helper.isLoadTestingSession()) {
const failedStep = helper.getLastFailedLtsStep(global.__bstack_steps);
if (failedStep) {
const message = stripAnsi(String(failedStep.failure));
testData.result = 'failed';
testData.failure = [{'backtrace': [message]}];
testData.failure_reason = message;
testData.failure_type = /Assert/.test(message) ? 'AssertionError' : 'UnhandledError';
}
}
await this.processTestRunData (eventData, uuid);
}

Expand Down
16 changes: 16 additions & 0 deletions src/utils/helper.js
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,22 @@ exports.shouldInstallLtsStopPath = (localTunnelRunning) => this.isLoadTestingSes
!!process.env.BROWSERSTACK_TESTHUB_UUID && !!process.env.BROWSERSTACK_TESTHUB_JWT &&
!!process.env.BROWSERSTACK_LTS_SDK_PID_FILE && !localTunnelRunning;

// The last step bstackStep() recorded as failed with a non-empty failure
// message, or null. Steps are pushed in execution order, so the last failed
// one is the step that ended the test (a parent wrapping a failed child is
// pushed first, so the child's more specific message wins).
exports.getLastFailedLtsStep = (steps) => {
if (!Array.isArray(steps)) {return null}
for (let i = steps.length - 1; i >= 0; i -= 1) {
const step = steps[i];
if (step && step.result === 'failed' && typeof step.failure === 'string' && step.failure.trim()) {
return step;
}
}

return null;
};

exports.generateLocalIdentifier = () => {
const formattedDate = new Intl.DateTimeFormat('en-GB', {
month: 'short',
Expand Down
8 changes: 5 additions & 3 deletions test/src/test-observability/ltsStopPath.js
Original file line number Diff line number Diff line change
Expand Up @@ -619,13 +619,15 @@ describe('globals - LT SIGTERM path (LTAR-392)', function () {

it('suppresses a TestRunFinished already past its entry check when the signal lands', async function () {
const uuid = await startTest('finish in flight');
const sessionStatus = deferred();
global.browser.execute = sandbox.stub().returns(sessionStatus.promise);
// Hold the handler at its last await before the stop check. LTS runs skip
// the setSessionStatus executor (LTAR-420), so afterEachExecution is it.
const afterEach = deferred();
AccessibilityAutomation.prototype.afterEachExecution.returns(afterEach.promise);
const finishing = handlers['TestRunFinished'](buildTest('finish in flight'));
await flush();

await sigterm();
sessionStatus.resolve();
afterEach.resolve();
await finishing;

assert.strictEqual(uploads.filter((u) => u.event_type === 'TestRunFinished' && u.test_run.uuid === uuid).length, 0,
Expand Down
66 changes: 66 additions & 0 deletions test/src/test-observability/sendTestRunEvent.js
Original file line number Diff line number Diff line change
Expand Up @@ -235,4 +235,70 @@ describe('TestObservability - sendTestRunEvent (suppressNotFoundErrors)', functi
const meta = this.uploaded.test_run.meta;
assert.ok(!meta || !('steps' in meta), 'expected no meta.steps when nothing was recorded');
});

// LTAR-420: the LCNC helper throws its own Error, which Nightwatch keeps out
// of the command list, so the first failed command was an internal selector
// probe (executeAsyncScript "script timeout") whose result has no message or
// stack. TestRunFinished shipped backtrace ['', ''] and TRA could not cluster
// the failure. The failed bstackStep() is now the authoritative failure.
describe('LTS failed step', () => {
const selectorMessage = '[Step 14: Click on "disc Brake Pad" link [click, stepId=f4a69331]] No selector matched after 30000ms.';
const steps = [
{id: 'a', text: 'Step 13', result: 'passed', failure: null},
{id: 'b', text: 'Step 14', result: 'failed', failure: selectorMessage}
];
const timedOutProbe = {name: 'executeAsyncScript', args: [], status: 'fail', result: {status: -1, value: null}};

afterEach(() => {
delete global.__bstack_steps;
});

it('uses the failed step message when the failed command carries none', async () => {
this.sandbox.stub(helper, 'isLoadTestingSession').returns(true);
global.__bstack_steps = steps;

await this.testObservability.sendTestRunEvent('TestRunFinished',
buildTest([timedOutProbe], {status: 'fail', failed: 0, errors: 1}), 'uuid-lts-fail-1');

const run = this.uploaded.test_run;
assert.strictEqual(run.result, 'failed');
assert.deepStrictEqual(run.failure, [{backtrace: [selectorMessage]}]);
assert.strictEqual(run.failure_reason, selectorMessage);
assert.strictEqual(run.failure_type, 'UnhandledError');
});

it('reports failed (not passed) when no Nightwatch command failed', async () => {
this.sandbox.stub(helper, 'isLoadTestingSession').returns(true);
global.__bstack_steps = steps;

await this.testObservability.sendTestRunEvent('TestRunFinished',
buildTest([{name: 'url', args: ['https://example.com'], status: 'pass'}], {status: 'fail', failed: 0, errors: 1}), 'uuid-lts-fail-2');

assert.strictEqual(this.uploaded.test_run.result, 'failed');
assert.strictEqual(this.uploaded.test_run.failure_reason, selectorMessage);
});

it('keeps the command-derived failure on LTS runs with no failed step', async () => {
this.sandbox.stub(helper, 'isLoadTestingSession').returns(true);
global.__bstack_steps = [steps[0]];
const failedAssert = {name: 'assert.visible', args: [], status: 'fail', result: {message: 'Element was not visible', stack: 'at x', name: 'NightwatchAssertError'}};

await this.testObservability.sendTestRunEvent('TestRunFinished',
buildTest([failedAssert], {status: 'fail', failed: 1, errors: 0}), 'uuid-lts-fail-3');

assert.strictEqual(this.uploaded.test_run.failure_reason, 'Element was not visible');
assert.strictEqual(this.uploaded.test_run.failure_type, 'AssertionError');
});

it('ignores the step buffer on non-LTS runs', async () => {
this.sandbox.stub(helper, 'isLoadTestingSession').returns(false);
global.__bstack_steps = steps;

await this.testObservability.sendTestRunEvent('TestRunFinished',
buildTest([{name: 'url', args: ['https://example.com'], status: 'pass'}]), 'uuid-lts-fail-4');

assert.strictEqual(this.uploaded.test_run.result, 'passed');
assert.ok(!('failure' in this.uploaded.test_run));
});
});
});
28 changes: 28 additions & 0 deletions test/src/utils/helper.js
Original file line number Diff line number Diff line change
Expand Up @@ -419,3 +419,31 @@ describe('isSuppressedFailure', () => {
});

});

describe('getLastFailedLtsStep', () => {
let getLastFailedLtsStep;
before(() => {
getLastFailedLtsStep = require('../../../src/utils/helper').getLastFailedLtsStep;
});

it('returns null for a missing or non-array buffer', () => {
expect(getLastFailedLtsStep(undefined)).to.be.null;
expect(getLastFailedLtsStep(null)).to.be.null;
expect(getLastFailedLtsStep({})).to.be.null;
});

it('returns null when no step failed', () => {
expect(getLastFailedLtsStep([{id: 'a', result: 'passed', failure: null}])).to.be.null;
});

it('returns the last failed step that carries a message', () => {
const parent = {id: 'p', result: 'failed', failure: 'outer'};
const child = {id: 'c', result: 'failed', failure: 'No selector matched after 30000ms.'};
expect(getLastFailedLtsStep([{id: 'a', result: 'passed'}, parent, child])).to.equal(child);
});

it('skips a failed step with an empty or non-string failure', () => {
const real = {id: 'b', result: 'failed', failure: 'real cause'};
expect(getLastFailedLtsStep([real, {id: 'c', result: 'failed', failure: ' '}, {id: 'd', result: 'failed', failure: null}])).to.equal(real);
});
});