diff --git a/.gitignore b/.gitignore index 4afed9191..1439146bc 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ package-lock.json yarn.lock /.vs typings/types.d.ts -typings/promiseBasedTypes.d.ts \ No newline at end of file +typings/promiseBasedTypes.d.ts +reflection/ \ No newline at end of file diff --git a/docs/debugging.md b/docs/debugging.md new file mode 100644 index 000000000..00c8c87fc --- /dev/null +++ b/docs/debugging.md @@ -0,0 +1,288 @@ +--- +permalink: /debugging +title: Debugging Tests +--- + +# Debugging Tests + +CodeceptJS provides powerful tools for debugging your tests at every level: from verbose output to interactive breakpoints with step-by-step execution. + +## Output Verbosity + +When something goes wrong, start by increasing the output level: + +```bash +npx codeceptjs run --steps # print each step +npx codeceptjs run --debug # print steps + debug info +npx codeceptjs run --verbose # print everything +``` + +| Flag | What you see | +|------|-------------| +| `--steps` | Each executed step (`I click`, `I see`, etc.) | +| `--debug` | Steps + helper/plugin info, URLs, loaded config | +| `--verbose` | Everything above + internal promise queue, retries, timeouts | + +We recommend **always using `--debug`** when developing tests. + +For full internal logs, use the `DEBUG` environment variable: + +```bash +DEBUG=codeceptjs:* npx codeceptjs run +``` + +You can narrow it down to specific modules: + +```bash +DEBUG=codeceptjs:recorder npx codeceptjs run # promise chain only +DEBUG=codeceptjs:pause npx codeceptjs run # pause session only +``` + +## Interactive Pause + +The most powerful debugging tool in CodeceptJS is **interactive pause**. It stops test execution and opens a live shell where you can run steps directly in the browser. + +Add `pause()` anywhere in your test: + +```js +Scenario('debug checkout', ({ I }) => { + I.amOnPage('/products') + I.click('Add to Cart') + pause() // <-- test stops here, shell opens + I.click('Checkout') +}) +``` + +When the shell opens, you'll see: + +``` + Interactive shell started + Use JavaScript syntax to try steps in action + - Press ENTER to run the next step + - Press TAB twice to see all available commands + - Type exit + Enter to exit the interactive shell + - Prefix => to run js commands +``` + +### Available Commands + +| Input | What it does | +|-------|-------------| +| `click('Login')` | Runs `I.click('Login')` — the `I.` prefix is added automatically | +| `see('Welcome')` | Runs `I.see('Welcome')` | +| `grabCurrentUrl()` | Runs a grab method and prints the result | +| `=> myVar` | Evaluates JavaScript expression | +| *ENTER* (empty) | Runs the next test step and pauses again | +| `exit` or `resume` | Exits the shell and continues the test | +| *TAB TAB* | Shows all available I.* methods | + +You can also pass variables into the shell: + +```js +const userData = { name: 'John', email: 'john@test.com' } +pause({ userData }) +// in shell: => userData.name +``` + +### Using Pause as a Stop Point + +`pause()` works as a **breakpoint** in your test. Place it before a failing step to inspect the page state: + +```js +Scenario('fix login bug', ({ I }) => { + I.amOnPage('/login') + I.fillField('Email', 'user@test.com') + I.fillField('Password', 'secret') + pause() // stop here, inspect the form before submitting + I.click('Sign In') + I.see('Dashboard') +}) +``` + +You can also add it in hooks: + +```js +After(({ I }) => { + pause() // pause after every test to inspect final state +}) +``` + +## Pause On Plugin + +For automated debugging without modifying test code, use the `pauseOn` plugin. It pauses tests based on different triggers, controlled entirely from the command line. + +### Pause on Failure + +Automatically enters interactive pause when a step fails: + +```bash +npx codeceptjs run -p pauseOn:fail +``` + +This is the most common debug workflow — run your tests, and when one fails, you land in the interactive shell with the browser in the exact state of the failure. You can inspect elements, try different selectors, and figure out what went wrong. + +> The older `pauseOnFail` plugin still works: `npx codeceptjs run -p pauseOnFail` + +### Pause on Every Step + +Enters interactive pause at the start of the test. Use *ENTER* to advance step by step: + +```bash +npx codeceptjs run -p pauseOn:step +``` + +This gives you full step-by-step execution. After each step, you're back in the interactive shell where you can inspect the page before pressing ENTER to continue. + +### Pause on File (Breakpoint) + +Pauses when execution reaches a specific file: + +```bash +npx codeceptjs run -p pauseOn:file:tests/login_test.js +``` + +With a specific line number: + +```bash +npx codeceptjs run -p pauseOn:file:tests/login_test.js:43 +``` + +This works like a breakpoint — the test runs normally until it hits a step defined at that file and line, then opens the interactive shell. + +### Pause on URL + +Pauses when the browser navigates to a matching URL: + +```bash +npx codeceptjs run -p pauseOn:url:/users/1 +``` + +Supports `*` wildcards: + +```bash +npx codeceptjs run -p pauseOn:url:/api/*/edit +npx codeceptjs run -p pauseOn:url:/checkout/* +``` + +This is useful when you want to inspect a specific page regardless of which test step navigates there. + +## IDE Debugging + +### VS Code + +You can use the built-in Node.js debugger in VS Code to set breakpoints in test files. + +Add this configuration to `.vscode/launch.json`: + +```json +{ + "type": "node", + "request": "launch", + "name": "codeceptjs", + "args": ["run", "--grep", "@your_test_tag", "--debug"], + "program": "${workspaceFolder}/node_modules/codeceptjs/bin/codecept.js" +} +``` + +Set breakpoints in your test files, then press F5 to start debugging. You'll be able to step through code, inspect variables, and use the VS Code debug console — all while the browser is open and controlled by the test. + +Combine with `pause()` for the best experience: set a VS Code breakpoint to inspect JavaScript state, then add `pause()` to interact with the browser. + +### WebStorm + +```bash +node $NODE_DEBUG_OPTION ./node_modules/.bin/codeceptjs run +``` + +### Node.js Inspector + +```bash +node --inspect ./node_modules/.bin/codeceptjs run +node --inspect-brk ./node_modules/.bin/codeceptjs run # break on first line +``` + +## Debugging on Failure + +Several built-in plugins capture information when tests fail. These are most useful on CI where you can't use interactive debugging. + +### Screenshots on Failure + +Enabled by default. Saves a screenshot when a test fails: + +```js +plugins: { + screenshotOnFail: { + enabled: true, + uniqueScreenshotNames: true, + fullPageScreenshots: true, + } +} +``` + +Screenshots are saved in the `output` directory. + +### Page Info on Failure + +Captures URL, HTML errors, and browser console logs on failure: + +```js +plugins: { + pageInfo: { + enabled: true, + } +} +``` + +### Step-by-Step Report + +Generates a slideshow of screenshots taken after every step — a visual replay of what the test did: + +```js +plugins: { + stepByStepReport: { + enabled: true, + deleteSuccessful: true, // keep only failed tests + fullPageScreenshots: true, + } +} +``` + +```bash +npx codeceptjs run -p stepByStepReport +``` + +After the run, open `output/records.html` to browse through the slideshows. + +## AI-Powered Debugging + +### AI in Interactive Pause + +When AI is configured, the interactive pause shell accepts natural language commands: + +``` + AI is enabled! (experimental) Write what you want and make AI run it + +I.$ fill the login form with test credentials and submit +``` + +AI reads the current page HTML and generates CodeceptJS steps. See [Testing with AI](/ai) for setup. + +### AI Trace Plugin + +The `aiTrace` plugin captures rich execution traces for AI analysis — screenshots, HTML snapshots, ARIA trees, browser logs, and network requests at every step: + +```js +plugins: { + aiTrace: { + enabled: true, + } +} +``` + +After a test run, trace files are generated in `output/trace_*/trace.md`. Feed these to an AI assistant (like Claude Code) for automated failure analysis. See [AI Trace Plugin](/aitrace) for details. + +### MCP Server + +CodeceptJS includes an [MCP server](/mcp) that allows AI agents to control tests programmatically — list tests, run them step by step, capture artifacts, and analyze results. This enables AI-driven debugging workflows where an agent can investigate failures autonomously. + +> AI agent integration and MCP server will be covered in detail on a dedicated page. diff --git a/docs/heal.md b/docs/heal.md index 1cc1f217f..d03ab0732 100644 --- a/docs/heal.md +++ b/docs/heal.md @@ -8,15 +8,40 @@ Browser and Mobile tests can fail for vareity of reasons. However, on a big proj ![](/img/healing.png) -Let's start with an example the most basic healing recipe. If after a click test has failed, try to reload page, and continue. +Let's start with a common scenario. If a user suddenly becomes unauthorized and is moved to a sign-in page, or receives an unauthorized message on the page, we can heal by navigating to the `/login` page and trying to enter credentials again. + +```js +heal.addRecipe('loginOnUnauthorized', { + priority: 10, + steps: ['click', 'see', 'amOnPage'], + prepare: { + url: ({ I }) => I.grabCurrentUrl(), + html: ({ I }) => I.grabHTMLFrom('body'), + }, + fn: async ({ url, error, step, html }) => { + if (!url.includes('/login') && !error.message.toLowerCase().includes('unauthorized') && !html.toLowerCase().includes('unauthorized')) return; + + return ({ I }) => { + I.amOnPage('/login'); + I.fillField('Email', 'test@example.com'); + I.fillField('Password', '123456'); + I.click('Sign in'); + I[step.name](...step.args); + }; + }, +}); +``` + +Another example is a very basic healing recipe. If after a click test has failed, try to reload page, and continue. ```js heal.addRecipe('reload', { priority: 10, steps: ['click'], - fn: async () => { + fn: async ({ step }) => { return ({ I }) => { I.refreshPage(); + I[step.name](...step.args); }; }, }); @@ -32,6 +57,7 @@ The example above is only one way a test can be healed. But you can define as ma There are some ideas where healing can be useful to you: +* **Authorization**. If a user suddenly becomes unauthorized and is moved to a sign-in page, or receives an unauthorized message on the page, we can heal by navigating to the `/login` page and trying to enter credentials. * **Networking**. If a test depends on a remote resource, and fails because this resource is not available, you may try to send API request to restore that resource before throwing an error. * **Data Consistency**. A test may fail because you noticed the data glitch in a system. Instead of failing a test you may try to clean up the data and try again to proceed. * **UI Change**. If there is a planned UI migration of a component, for instance Button was changed to Dropdown. You can prepare test so if it fails clicking Button it can try to do so with Dropdown. @@ -67,9 +93,9 @@ Require `recipes` file and add `heal` plugin to `codecept.conf` file: ```js -require('./heal') +import './heal'; -exports.config = { +export const config = { // ... plugins: { heal: { @@ -134,7 +160,8 @@ heal.addRecipe('reloadPageOnUserAccount', { // probably you should do something more sophisticated // to heal the test I.reloadPage(); - I.wait(1); + I.wait(1); + I[step.name](...stepArgs); }; }, }); diff --git a/lib/command/dryRun.js b/lib/command/dryRun.js index 1d46caf85..499c39fd5 100644 --- a/lib/command/dryRun.js +++ b/lib/command/dryRun.js @@ -20,8 +20,7 @@ export default async function (test, options) { if (config.plugins) { // disable all plugins by default, they can be enabled with -p option for (const plugin in config.plugins) { - // if `-p all` is passed, then enabling all plugins, otherwise plugins could be enabled by `-p customLocator,commentStep,tryTo` - config.plugins[plugin].enabled = options.plugins === 'all' + config.plugins[plugin].enabled = false } } diff --git a/lib/container.js b/lib/container.js index f18e4ecf8..faa1bde06 100644 --- a/lib/container.js +++ b/lib/container.js @@ -690,14 +690,28 @@ async function loadPluginFallback(modulePath, config) { async function createPlugins(config, options = {}) { const plugins = {} - const enabledPluginsByOptions = (options.plugins || '').split(',') + const pluginOptionMap = new Map() + for (const token of (options.plugins || '').split(',').filter(Boolean)) { + const parts = token.split(':') + pluginOptionMap.set(parts[0], parts.slice(1)) + } + + for (const [name] of pluginOptionMap) { + if (!config[name]) config[name] = {} + } + for (const pluginName in config) { if (!config[pluginName]) config[pluginName] = {} const pluginConfig = config[pluginName] - if (!pluginConfig.enabled && enabledPluginsByOptions.indexOf(pluginName) < 0) { + const enabledByCli = pluginOptionMap.has(pluginName) + if (!pluginConfig.enabled && !enabledByCli) { continue // plugin is disabled } + if (enabledByCli && pluginOptionMap.get(pluginName).length > 0) { + pluginConfig._args = pluginOptionMap.get(pluginName) + } + // Generic workers gate: // - runInWorker / runInWorkers controls plugin execution inside worker threads. // - runInParent / runInMain can disable plugin in workers parent process. diff --git a/lib/plugin/aiTrace.js b/lib/plugin/aiTrace.js index e9ad15ed4..f2668afbc 100644 --- a/lib/plugin/aiTrace.js +++ b/lib/plugin/aiTrace.js @@ -438,7 +438,7 @@ export default function (config) { const traceFile = path.join(dir, 'trace.md') fs.writeFileSync(traceFile, markdown) - output.print(`🤖 AI Trace: ${colors.white.bold(`file://${traceFile}`)}`) + output.print(`Trace Saved: file://${traceFile}`) if (!test.artifacts) test.artifacts = {} test.artifacts.aiTrace = traceFile diff --git a/lib/plugin/pauseOn.js b/lib/plugin/pauseOn.js new file mode 100644 index 000000000..dea25dea5 --- /dev/null +++ b/lib/plugin/pauseOn.js @@ -0,0 +1,167 @@ +import event from '../event.js' +import pause from '../pause.js' +import recorder from '../recorder.js' +import Container from '../container.js' +import output from '../output.js' + +const supportedHelpers = Container.STANDARD_ACTING_HELPERS + +/** + * Pauses test execution in different modes. Unlike `pauseOnFail`, this plugin supports + * multiple triggers for pausing and is controlled via CLI arguments. + * + * Enable it via `-p` option with a mode: + * + * ``` + * npx codeceptjs run -p pauseOn:fail + * npx codeceptjs run -p pauseOn:step + * npx codeceptjs run -p pauseOn:file:tests/login_test.js + * npx codeceptjs run -p pauseOn:file:tests/login_test.js:43 + * npx codeceptjs run -p pauseOn:url:/users/* + * ``` + * + * #### Modes + * + * * **fail** — pause when a step fails (same as `pauseOnFail` plugin) + * * **step** — pause before first step, use `next` to advance step-by-step + * * **file** — pause when execution reaches a specific file (and optionally line) + * * **url** — pause when the browser URL matches a pattern (supports `*` wildcards) + * + */ +export default function (config = {}) { + const args = config._args || [] + const mode = args[0] || 'fail' + + switch (mode) { + case 'fail': + return initFailMode() + case 'step': + return initStepMode() + case 'file': + return initFileMode(args.slice(1)) + case 'url': + return initUrlMode(args.slice(1)) + default: + output.error(`pauseOn: unknown mode "${mode}". Available: fail, step, file, url`) + } +} + +function initFailMode() { + let failed = false + + event.dispatcher.on(event.test.started, () => { + failed = false + }) + + event.dispatcher.on(event.step.failed, () => { + failed = true + }) + + event.dispatcher.on(event.test.after, () => { + if (failed) pause() + }) +} + +function initStepMode() { + let activated = false + + event.dispatcher.on(event.test.before, () => { + if (activated) return + activated = true + recorder.add('pauseOn:step', () => pause()) + }) +} + +function initFileMode(fileArgs) { + if (fileArgs.length === 0) { + output.error('pauseOn:file requires a path. Usage: -p pauseOn:file:[:]') + return + } + + const targetFile = fileArgs[0] + const targetLine = fileArgs[1] ? parseInt(fileArgs[1], 10) : null + let paused = false + + event.dispatcher.on(event.step.before, (step) => { + if (paused) return + + const stepLine = step.line() + if (!stepLine) return + + const match = parseStepLine(stepLine) + if (!match) return + + const fileMatches = match.file.includes(targetFile) || match.file.endsWith(targetFile) + if (!fileMatches) return + + if (targetLine !== null && match.line !== targetLine) return + + paused = true + recorder.add('pauseOn:file', () => pause()) + }) +} + +function initUrlMode(urlArgs) { + if (urlArgs.length === 0) { + output.error('pauseOn:url requires a pattern. Usage: -p pauseOn:url:') + return + } + + const urlPattern = urlArgs.join(':') + + const helpers = Container.helpers() + let helper = null + for (const helperName of supportedHelpers) { + if (Object.keys(helpers).indexOf(helperName) > -1) { + helper = helpers[helperName] + } + } + + if (!helper) { + output.error('pauseOn:url requires a browser helper (Playwright, WebDriver, Puppeteer, Appium)') + return + } + + const regex = patternToRegex(urlPattern) + let paused = false + + event.dispatcher.on(event.step.after, () => { + if (paused) return + + recorder.add('pauseOn:url check', async () => { + if (paused) return + try { + const currentUrl = await helper.grabCurrentUrl() + if (regex.test(currentUrl)) { + paused = true + return pause() + } + } catch (err) { + // page may not be loaded yet + } + }) + }) +} + +function parseStepLine(stepLine) { + let line = stepLine.trim() + if (line.startsWith('at ')) line = line.substring(3).trim() + + const lastColon = line.lastIndexOf(':') + if (lastColon < 0) return null + const secondLastColon = line.lastIndexOf(':', lastColon - 1) + if (secondLastColon < 0) return null + + const file = line.substring(0, secondLastColon) + const lineNum = parseInt(line.substring(secondLastColon + 1, lastColon), 10) + + if (isNaN(lineNum)) return null + + return { file, line: lineNum } +} + +function patternToRegex(pattern) { + const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&') + const regexStr = escaped.replace(/\*/g, '.*') + return new RegExp(regexStr) +} diff --git a/test/runner/dry_run_test.js b/test/runner/dry_run_test.js index 97d59859a..300969cb9 100644 --- a/test/runner/dry_run_test.js +++ b/test/runner/dry_run_test.js @@ -186,17 +186,6 @@ describe('dry-run command', () => { }) }) - it('should enable all plugins in dry-mode when passing -p all', done => { - exec(`${codecept_run_config('codecept.customLocator.js')} --verbose -p all`, (err, stdout) => { - expect(stdout).toContain('Plugins: screenshotOnFail, customLocator') - expect(stdout).toContain("I see element {xpath: .//*[@data-testid='COURSE']//a}") - expect(stdout).toContain('OK | 1 passed') - expect(stdout).toContain('--- DRY MODE: No tests were executed ---') - expect(err).toBeFalsy() - done() - }) - }) - it('should enable a particular plugin in dry-mode when passing it to -p', done => { exec(`${codecept_run_config('codecept.customLocator.js')} --verbose -p customLocator`, (err, stdout) => { expect(stdout).toContain('Plugins: customLocator')