diff --git a/.circleci/config.yml b/.circleci/config.yml index b3d91eca1..43923136d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: - browser-tools: circleci/browser-tools@1.4.5 + browser-tools: circleci/browser-tools@1.4.8 defaults: &defaults machine: @@ -11,8 +11,8 @@ defaults: &defaults - checkout - run: .circleci/build.sh - browser-tools/install-chrome: - chrome-version: 116.0.5845.96 - replace-existing: true + chrome-version: latest # TODO: remove until: https://github.com/CircleCI-Public/browser-tools-orb/issues/75 + replace-existing: true # TODO: remove until: https://github.com/CircleCI-Public/browser-tools-orb/issues/75 - run: command: docker-compose run --rm test-rest working_directory: test diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml index 122b534c3..d08b16c98 100644 --- a/.github/workflows/acceptance-tests.yml +++ b/.github/workflows/acceptance-tests.yml @@ -27,6 +27,12 @@ jobs: - name: Checkout Repository uses: actions/checkout@v3 + # Install Docker Compose + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + # Run rest tests using docker-compose - name: Run REST Tests run: docker-compose run --rm test-rest diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4dfcda903..237f17b0c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,9 +4,8 @@ on: push: branches: - 3.x - # Build and push Docker images *only* for releases. release: - types: [published] # , created, edited + types: [published] jobs: push_to_registry: @@ -14,33 +13,27 @@ jobs: runs-on: ubuntu-22.04 steps: - - name: Check out the repo with latest code + - name: Check out the repo with the latest code uses: actions/checkout@v4 - - name: Push latest to Docker Hub - uses: docker/build-push-action@v6 # Info: https://github.com/docker/build-push-action/tree/releases/v1#tags + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: ${{ secrets.DOCKERHUB_REPOSITORY }} - tag_with_ref: true # Info: https://github.com/docker/build-push-action/tree/releases/v1#tag_with_ref - tag_with_sha: true # Info: https://github.com/docker/build-push-action/tree/releases/v1#tag_with_sha - tags: latest - - name: 'Get the current tag' + - name: Get the current tag id: currentTag - uses: actions/checkout@v4 - - run: git fetch --prune --unshallow && TAG=$(git describe --tags --abbrev=0) && echo $TAG && echo "TAG="$TAG >> "$GITHUB_ENV" - - name: Check out the repo with tag - uses: actions/checkout@v4 - with: - ref: ${{ env.TAG }} + run: git fetch --prune --unshallow && TAG=$(git describe --tags --abbrev=0) && echo $TAG && echo "TAG="$TAG >> "$GITHUB_ENV" - - name: Push current tag to Docker Hub - uses: docker/build-push-action@v6 # Info: https://github.com/docker/build-push-action/tree/releases/v1#tags + - name: Build and push Docker image + uses: docker/build-push-action@v6 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - repository: ${{ secrets.DOCKERHUB_REPOSITORY }} - tag_with_ref: true # Info: https://github.com/docker/build-push-action/tree/releases/v1#tag_with_ref - tag_with_sha: true # Info: https://github.com/docker/build-push-action/tree/releases/v1#tag_with_sha - tags: ${{ env.TAG }} + context: . + push: true + tags: | + ${{ secrets.DOCKERHUB_REPOSITORY }}:latest + ${{ secrets.DOCKERHUB_REPOSITORY }}:${{ env.TAG }} \ No newline at end of file diff --git a/.github/workflows/softExpectHelper.yml b/.github/workflows/softExpectHelper.yml new file mode 100644 index 000000000..141c056b8 --- /dev/null +++ b/.github/workflows/softExpectHelper.yml @@ -0,0 +1,34 @@ +name: Soft Expect Helper Tests + +on: + push: + branches: + - 3.x + pull_request: + branches: + - '**' + +env: + CI: true + # Force terminal colors. @see https://www.npmjs.com/package/colors + FORCE_COLOR: 1 + +jobs: + build: + + runs-on: ubuntu-22.04 + + strategy: + matrix: + node-version: [20.x] + + steps: + - uses: actions/checkout@v4 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - name: npm install + run: npm i --force + - name: run unit tests + run: ./node_modules/.bin/mocha test/helper/SoftExpect_test.js --timeout 5000 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7767540bf..a6def637d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,65 @@ +## 3.6.6 + +❤️ Thanks all to those who contributed to make this release! ❤️ + +🛩️ *Features* +* feat(locator): add withAttrEndsWith, withAttrStartsWith, withAttrContains (#4334) - by @Maksym-Artemenko +* feat: soft assert (#4473) - by @kobenguyent + * Soft assert + +Zero-configuration when paired with other helpers like REST, Playwright: + +```js +// inside codecept.conf.js +{ + helpers: { + Playwright: {...}, + SoftExpectHelper: {}, + } +} +``` + +```js +// in scenario +I.softExpectEqual('a', 'b') +I.flushSoftAssertions() // Throws an error if any soft assertions have failed. The error message contains all the accumulated failures. +``` +* feat(cli): print failed hooks (#4476) - by @kobenguyent + * run command + ![Screenshot 2024-09-02 at 15 25 20](https://github.com/user-attachments/assets/625c6b54-03f6-41c6-9d0c-cd699582404a) + + * run workers command +![Screenshot 2024-09-02 at 15 24 53](https://github.com/user-attachments/assets/efff0312-1229-44b6-a94f-c9b9370b9a64) + +🐛 *Bug Fixes* +* fix(AI): minor AI improvements - by @DavertMik +* fix(AI): add missing await in AI.js (#4486) - by @tomaculum +* fix(playwright): no async save video page (#4472) - by @kobenguyent +* fix(rest): httpAgent condition (#4484) - by @kobenguyent +* fix: DataCloneError error when `I.executeScript` command is used with `run-workers` (#4483) - by @code4muktesh +* fix: no error thrown from rerun script (#4494) - by @lin-brian-l + + +```js +// fix the validation of httpAgent config. we could now pass ca, instead of key/cert. +{ + helpers: { + REST: { + endpoint: 'http://site.com/api', + prettyPrintJson: true, + httpAgent: { + ca: fs.readFileSync(__dirname + '/path/to/ca.pem'), + rejectUnauthorized: false, + keepAlive: true + } + } + } +} +``` + +📖 *Documentation* +* doc(AI): minor AI improvements - by @DavertMik + ## 3.6.5 ❤️ Thanks all to those who contributed to make this release! ❤️ diff --git a/docs/ai.md b/docs/ai.md index 5d99712d1..b3b1ea5ee 100644 --- a/docs/ai.md +++ b/docs/ai.md @@ -1,6 +1,6 @@ --- permalink: /ai -title: Testing with AI 🪄 +title: Testing with AI 🪄 --- # 🪄 Testing with AI @@ -37,7 +37,7 @@ AI providers have limits on input tokens but HTML pages can be huge. However, so Even though, the HTML is still quite big and may exceed the token limit. So we recommend using models with at least 16K input tokens, (approx. 50K of HTML text), which should be enough for most web pages. It is possible to strictly limit the size of HTML to not exceed tokens limit. -> ❗AI features require sending HTML contents to AI provider. Choosing one may depend on the descurity policy of your company. Ask your security department which AI providers you can use. +> ❗AI features require sending HTML contents to AI provider. Choosing one may depend on the descurity policy of your company. Ask your security department which AI providers you can use. @@ -91,7 +91,7 @@ ai: { model: 'gpt-3.5-turbo-0125', messages, }); - + return completion?.choices[0]?.message?.content; } } @@ -146,7 +146,7 @@ ai: { model: 'claude-2.1', max_tokens: 1024, messages - }); + }); return resp.content.map((c) => c.text).join('\n\n'); } } @@ -167,7 +167,7 @@ ai: { const { OpenAIClient, AzureKeyCredential } = require("@azure/openai"); const client = new OpenAIClient( - "https://.openai.azure.com/", + "https://.openai.azure.com/", new AzureKeyCredential("") ); const { choices } = await client.getCompletions("", messages); @@ -260,7 +260,7 @@ async function makeApiRequest(endpoint, deploymentId, messages) { } ``` -### Writing Tests with AI Copilot +## Writing Tests with AI Copilot If AI features are enabled when using [interactive pause](/basics/#debug) with `pause()` command inside tests: @@ -302,11 +302,11 @@ GPT will generate code and data and CodeceptJS will try to execute its code. If This AI copilot works best with long static forms. In the case of complex and dynamic single-page applications, it may not perform as well, as the form may not be present on HTML page yet. For instance, interacting with calendars or inputs with real-time validations (like credit cards) can not yet be performed by AI. -Please keep in mind that GPT can't react to page changes and operates with static text only. This is why it is not ready yet to write the test completely. However, if you are new to CodeceptJS and automated testing AI copilot may help you write tests more efficiently. +Please keep in mind that GPT can't react to page changes and operates with static text only. This is why it is not ready yet to write the test completely. However, if you are new to CodeceptJS and automated testing AI copilot may help you write tests more efficiently. > 👶 Enable AI copilot for junior test automation engineers. It may help them to get started with CodeceptJS and to write good semantic locators. -### Self-Healing Tests +## Self-Healing Tests In large test suites, the cost of maintaining tests goes exponentially. That's why any effort that can improve the stability of tests pays itself. That's why CodeceptJS has concept of [heal recipes](./heal), functions that can be executed on a test failure. Those functions can try to revive the test and continue execution. When combined with AI, heal recipe can ask AI provider how to fix the test. It will provide error message, step being executed and HTML context of a page. Based on this information AI can suggest the code to be executed to fix the failing test. @@ -315,12 +315,28 @@ AI healing can solve exactly one problem: if a locator of an element has changed > You can define your own [heal recipes](./heal) that won't use AI to revive failing tests. -Heal actions **work only on actions like `click`, `fillField`**, etc, and won't work on assertions, waiters, grabbers, etc. Assertions can't be guessed by AI, the same way as grabbers, as this may lead to unpredictable results. +Heal actions **work only on actions like `click`, `fillField`, etc, and won't work on assertions, waiters, grabbers, etc. Assertions can't be guessed by AI, the same way as grabbers, as this may lead to unpredictable results. If Heal plugin successfully fixes the step, it will print a suggested change at the end of execution. Take it as actionable advice and use it to update the codebase. Heal plugin is supposed to be used on CI, and works automatically without human assistance. -To start, make sure [AI provider is connected](#set-up-ai-provider), and [heal recipes were created](./heal#how-to-start-healing) and included into `codecept.conf.js` or `codecept.conf.ts` config file. Then enable `heal` plugin: +To start, make sure [AI provider is connected](#set-up-ai-provider), and [heal recipes were created](/heal#how-to-start-healing) by running this command: + +``` +npx codeceptjs generate:heal +``` + +Heal recipes should be included into `codecept.conf.js` or `codecept.conf.ts` config file: + +```js + +require('./heal') + +exports.config = { + // ... your codeceptjs config +``` + +Then enable `heal` plugin: ```js plugins: { @@ -330,7 +346,7 @@ plugins: { } ``` -If you tests in AI mode and test fails, a request to AI provider will be sent +If you run tests in AI mode and a test fails, a request to AI provider will be sent ``` npx codeceptjs run --ai @@ -342,7 +358,7 @@ When execution finishes, you will receive information on token usage and code su By evaluating this information you will be able to check how effective AI can be for your case. -### Arbitrary GPT Prompts +## Arbitrary Prompts What if you want to take AI on the journey of test automation and ask it questions while browsing pages? @@ -396,7 +412,7 @@ npx codeceptjs shell --ai Also this is availble from `pause()` if AI helper is enabled, -Ensure that browser is started in window mode, then browse the web pages on your site. +Ensure that browser is started in window mode, then browse the web pages on your site. On a page you want to create PageObject execute `askForPageObject()` command. The only required parameter is the name of a page: ```js @@ -422,6 +438,8 @@ If page object has `clickForgotPassword` method you can execute it as: => page.clickForgotPassword() ``` +Here is an example of a session: + ```shell Page object for login is saved to .../output/loginPage-1718579784751.js Page object registered for this session as `page` variable @@ -465,11 +483,11 @@ GPT prompts and HTML compression can also be configured inside `ai` section of ` ```js ai: { - // define how requests to AI are sent + // define how requests to AI are sent request: (messages) => { // ... } - // redefine prompts + // redefine prompts prompts: { // {} }, diff --git a/docs/changelog.md b/docs/changelog.md index 7ef0f19da..d028f8a18 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,35 @@ layout: Section # Releases +## 3.6.5 + +❤️ Thanks all to those who contributed to make this release! ❤️ + +🛩️ *Features* +* feat(helper): playwright > wait for disabled ([#4412](https://github.com/codeceptjs/CodeceptJS/issues/4412)) - by **[kobenguyent](https://github.com/kobenguyent)** +``` +it('should wait for input text field to be disabled', () => + I.amOnPage('/form/wait_disabled').then(() => I.waitForDisabled('#text', 1))) + + it('should wait for input text field to be enabled by xpath', () => + I.amOnPage('/form/wait_disabled').then(() => I.waitForDisabled("//*[@name = 'test']", 1))) + + it('should wait for a button to be disabled', () => + I.amOnPage('/form/wait_disabled').then(() => I.waitForDisabled('#text', 1))) + +Waits for element to become disabled (by default waits for 1sec). +Element can be located by CSS or XPath. + **[param](https://github.com/param)** {CodeceptJS.LocatorOrString} locator element located by CSS|XPath|strict locator. **[param](https://github.com/param)** {number} [sec=1] (optional) time in seconds to wait, 1 by default. **[returns](https://github.com/returns)** {void} automatically synchronized promise through #recorder +``` + +🐛 *Bug Fixes* +* fix(AI): AI is not triggered ([#4422](https://github.com/codeceptjs/CodeceptJS/issues/4422)) - by **[kobenguyent](https://github.com/kobenguyent)** +* fix(plugin): stepByStep > report doesn't sync properly ([#4413](https://github.com/codeceptjs/CodeceptJS/issues/4413)) - by **[kobenguyent](https://github.com/kobenguyent)** +* fix: Locator > Unsupported pseudo selector 'has' ([#4448](https://github.com/codeceptjs/CodeceptJS/issues/4448)) - by **[anils92](https://github.com/anils92)** + +📖 *Documentation* +* docs: setup azure open ai using bearer token ([#4434](https://github.com/codeceptjs/CodeceptJS/issues/4434)) - by **[kobenguyent](https://github.com/kobenguyent)** + ## 3.6.4 ❤️ Thanks all to those who contributed to make this release! ❤️ diff --git a/docs/community-helpers.md b/docs/community-helpers.md index 7394b79eb..2f135e440 100644 --- a/docs/community-helpers.md +++ b/docs/community-helpers.md @@ -43,7 +43,6 @@ Please **add your own** by editing this page. * [codeceptjs-slack-reporter](https://www.npmjs.com/package/codeceptjs-slack-reporter) Get a Slack notification when one or more scenarios fail. * [codeceptjs-browserlogs-plugin](https://github.com/pavkam/codeceptjs-browserlogs-plugin) Record the browser logs for failed tests. * [codeceptjs-testrail](https://github.com/PeterNgTr/codeceptjs-testrail) - a plugin to integrate with [Testrail](https://www.gurock.com/testrail) -* [codeceptjs-monocart-coverage](https://github.com/cenfun/codeceptjs-monocart-coverage) - a plugin to generate coverage reports, it integrate with [monocart coverage reports](https://github.com/cenfun/monocart-coverage-reports) ## Browser request control * [codeceptjs-resources-check](https://github.com/luarmr/codeceptjs-resources-check) Load a URL with Puppeteer and listen to the requests while the page is loading. Enabling count the number or check the sizes of the requests. diff --git a/docs/helpers/Detox.md b/docs/helpers/Detox.md index 759f1468a..45401848e 100644 --- a/docs/helpers/Detox.md +++ b/docs/helpers/Detox.md @@ -83,6 +83,7 @@ Options: - `reloadReactNative` - should be enabled for React Native applications. - `reuse` - reuse application for tests. By default, Detox reinstalls and relaunches app. - `registerGlobals` - (default: true) Register Detox helper functions `by`, `element`, `expect`, `waitFor` globally. +- `url` - URL to open via deep-link each time the app is launched (android) or immediately afterwards (iOS). Useful for opening a bundle URL at the beginning of tests when working with Expo. ### Parameters diff --git a/docs/helpers/REST.md b/docs/helpers/REST.md index 953208444..d35e8ff20 100644 --- a/docs/helpers/REST.md +++ b/docs/helpers/REST.md @@ -69,6 +69,22 @@ Type: [object][4] } ``` +```js +{ + helpers: { + REST: { + endpoint: 'http://site.com/api', + prettyPrintJson: true, + httpAgent: { + ca: fs.readFileSync(__dirname + '/path/to/ca.pem'), + rejectUnauthorized: false, + keepAlive: true + } + } + } +} +``` + ## Access From Helpers Send REST requests by accessing `_executeRequest` method: diff --git a/docs/helpers/SoftExpectHelper.md b/docs/helpers/SoftExpectHelper.md new file mode 100644 index 000000000..ab4e62da0 --- /dev/null +++ b/docs/helpers/SoftExpectHelper.md @@ -0,0 +1,357 @@ +--- +permalink: /helpers/SoftExpectHelper +editLink: false +sidebar: auto +title: SoftExpectHelper +--- + + + +## SoftAssertHelper + +**Extends ExpectHelper** + +SoftAssertHelper is a utility class for performing soft assertions. +Unlike traditional assertions that stop the execution on failure, +soft assertions allow the execution to continue and report all failures at the end. + +### Examples + +Zero-configuration when paired with other helpers like REST, Playwright: + +```js +// inside codecept.conf.js +{ + helpers: { + Playwright: {...}, + SoftExpectHelper: {}, + } +} +``` + +```js +// in scenario +I.softExpectEqual('a', 'b') +I.flushSoftAssertions() // Throws an error if any soft assertions have failed. The error message contains all the accumulated failures. +``` + +## Methods + +### flushSoftAssertions + +Throws an error if any soft assertions have failed. +The error message contains all the accumulated failures. + +- Throws **[Error][1]** If there are any soft assertion failures. + +### softAssert + +Performs a soft assertion by executing the provided assertion function. +If the assertion fails, the error is caught and stored without halting the execution. + +#### Parameters + +- `assertionFn` **[Function][2]** The assertion function to execute. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectAbove + +Softly asserts that the target data is above a specified value. + +#### Parameters + +- `targetData` **any** The data to check. +- `aboveThan` **any** The value that the target data should be above. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectBelow + +Softly asserts that the target data is below a specified value. + +#### Parameters + +- `targetData` **any** The data to check. +- `belowThan` **any** The value that the target data should be below. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectContain + +Softly asserts that a value contains the expected value. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValueToContain` **any** The value that should be contained within the actual value. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectDeepEqual + +Softly asserts that two values are deeply equal. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValue` **any** The expected value. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectDeepEqualExcluding + +Softly asserts that two objects are deeply equal, excluding specified fields. + +#### Parameters + +- `actualValue` **[Object][4]** The actual object. +- `expectedValue` **[Object][4]** The expected object. +- `fieldsToExclude` **[Array][5]<[string][3]>** The fields to exclude from the comparison. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectDeepIncludeMembers + +Softly asserts that an array (superset) deeply includes all members of another array (set). + +#### Parameters + +- `superset` **[Array][5]** The array that should contain the expected members. +- `set` **[Array][5]** The array with members that should be included. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectDeepMembers + +Softly asserts that two arrays have deep equality, considering members in any order. + +#### Parameters + +- `actualValue` **[Array][5]** The actual array. +- `expectedValue` **[Array][5]** The expected array. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectEmpty + +Softly asserts that the target data is empty. + +#### Parameters + +- `targetData` **any** The data to check. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectEndsWith + +Softly asserts that a value ends with the expected value. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValueToEndWith` **any** The value that the actual value should end with. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectEqual + +Softly asserts that two values are equal. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValue` **any** The expected value. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectEqualIgnoreCase + +Softly asserts that two values are equal, ignoring case. + +#### Parameters + +- `actualValue` **[string][3]** The actual string value. +- `expectedValue` **[string][3]** The expected string value. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectFalse + +Softly asserts that the target data is false. + +#### Parameters + +- `targetData` **any** The data to check. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectHasAProperty + +Softly asserts that the target data has a property with the specified name. + +#### Parameters + +- `targetData` **any** The data to check. +- `propertyName` **[string][3]** The property name to check for. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectHasProperty + +Softly asserts that the target data has the specified property. + +#### Parameters + +- `targetData` **any** The data to check. +- `propertyName` **[string][3]** The property name to check for. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion + fails. + +### softExpectJsonSchema + +Softly asserts that the target data matches the given JSON schema. + +#### Parameters + +- `targetData` **any** The data to validate. +- `jsonSchema` **[Object][4]** The JSON schema to validate against. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectJsonSchemaUsingAJV + +Softly asserts that the target data matches the given JSON schema using AJV. + +#### Parameters + +- `targetData` **any** The data to validate. +- `jsonSchema` **[Object][4]** The JSON schema to validate against. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. +- `ajvOptions` **[Object][4]** Options to pass to AJV. + +### softExpectLengthAboveThan + +Softly asserts that the length of the target data is above a specified value. + +#### Parameters + +- `targetData` **any** The data to check. +- `lengthAboveThan` **[number][6]** The length that the target data should be above. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectLengthBelowThan + +Softly asserts that the length of the target data is below a specified value. + +#### Parameters + +- `targetData` **any** The data to check. +- `lengthBelowThan` **[number][6]** The length that the target data should be below. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectLengthOf + +Softly asserts that the target data has a specified length. + +#### Parameters + +- `targetData` **any** The data to check. +- `length` **[number][6]** The expected length. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectMatchesPattern + +Softly asserts that a value matches the expected pattern. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedPattern` **any** The pattern the value should match. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectNotContain + +Softly asserts that a value does not contain the expected value. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValueToNotContain` **any** The value that should not be contained within the actual value. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectNotDeepEqual + +Softly asserts that two values are not deeply equal. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValue` **any** The expected value. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectNotEndsWith + +Softly asserts that a value does not end with the expected value. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValueToNotEndWith` **any** The value that the actual value should not end with. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectNotEqual + +Softly asserts that two values are not equal. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValue` **any** The expected value. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectNotStartsWith + +Softly asserts that a value does not start with the expected value. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValueToNotStartWith` **any** The value that the actual value should not start with. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectStartsWith + +Softly asserts that a value starts with the expected value. + +#### Parameters + +- `actualValue` **any** The actual value. +- `expectedValueToStartWith` **any** The value that the actual value should start with. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectToBeA + +Softly asserts that the target data is of a specific type. + +#### Parameters + +- `targetData` **any** The data to check. +- `type` **[string][3]** The expected type (e.g., 'string', 'number'). +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectToBeAn + +Softly asserts that the target data is of a specific type (alternative for articles). + +#### Parameters + +- `targetData` **any** The data to check. +- `type` **[string][3]** The expected type (e.g., 'string', 'number'). +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +### softExpectTrue + +Softly asserts that the target data is true. + +#### Parameters + +- `targetData` **any** The data to check. +- `customErrorMsg` **[string][3]** A custom error message to display if the assertion fails. + +[1]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Error + +[2]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function + +[3]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String + +[4]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object + +[5]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array + +[6]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number diff --git a/lib/cli.js b/lib/cli.js index 269430287..0d02c7765 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -145,6 +145,7 @@ class Cli extends Base { result() { const stats = this.stats; + stats.failedHooks = 0; console.log(); // passes @@ -216,8 +217,14 @@ class Cli extends Base { console.log(); } + this.failures.forEach((failure) => { + if (failure.constructor.name === 'Hook') { + stats.failures -= stats.failures + stats.failedHooks += 1 + } + }) event.emit(event.all.failures, { failuresLog, stats }); - output.result(stats.passes, stats.failures, stats.pending, ms(stats.duration)); + output.result(stats.passes, stats.failures, stats.pending, ms(stats.duration), stats.failedHooks); if (stats.failures && output.level() < 3) { output.print(output.styles.debug('Run with --verbose flag to see complete NodeJS stacktrace')); diff --git a/lib/command/run-rerun.js b/lib/command/run-rerun.js index 62f17bdae..ccb18bcfd 100644 --- a/lib/command/run-rerun.js +++ b/lib/command/run-rerun.js @@ -17,10 +17,6 @@ module.exports = async function (test, options) { const testRoot = getTestRoot(configFile) createOutputDir(config, testRoot) - function processError(err) { - printError(err) - process.exit(1) - } const codecept = new Codecept(config, options) try { diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 13efa1b41..48ce85127 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -161,7 +161,7 @@ function initializeListeners() { actor: step.actor, name: step.name, status: step.status, - args: _args, + args: JSON.stringify(_args), startedAt: step.startedAt, startTime: step.startTime, endTime: step.endTime, @@ -264,7 +264,10 @@ function collectStats() { event.dispatcher.on(event.test.passed, () => { stats.passes++; }); - event.dispatcher.on(event.test.failed, () => { + event.dispatcher.on(event.test.failed, (test) => { + if (test.ctx._runnable.title.includes('hook: AfterSuite')) { + stats.failedHooks += 1; + } stats.failures++; }); event.dispatcher.on(event.test.skipped, () => { diff --git a/lib/helper/AI.js b/lib/helper/AI.js index d59b15531..4b2d75978 100644 --- a/lib/helper/AI.js +++ b/lib/helper/AI.js @@ -74,7 +74,7 @@ class AI extends Helper { for (const chunk of htmlChunks) { const messages = [ { role: gtpRole.user, content: prompt }, - { role: gtpRole.user, content: `Within this HTML: ${minifyHtml(chunk)}` }, + { role: gtpRole.user, content: `Within this HTML: ${await minifyHtml(chunk)}` }, ] if (htmlChunks.length > 1) @@ -110,7 +110,7 @@ class AI extends Helper { const messages = [ { role: gtpRole.user, content: prompt }, - { role: gtpRole.user, content: `Within this HTML: ${minifyHtml(html)}` }, + { role: gtpRole.user, content: `Within this HTML: ${await minifyHtml(html)}` }, ] const response = await this._processAIRequest(messages) diff --git a/lib/helper/Playwright.js b/lib/helper/Playwright.js index 0921fb1ad..8e68fc74d 100644 --- a/lib/helper/Playwright.js +++ b/lib/helper/Playwright.js @@ -2396,9 +2396,9 @@ class Playwright extends Helper { } if (this.options.recordVideo && this.page && this.page.video()) { - test.artifacts.video = await saveVideoForPage(this.page, `${test.title}.failed`) + test.artifacts.video = saveVideoForPage(this.page, `${test.title}.failed`) for (const sessionName in this.sessionPages) { - test.artifacts[`video_${sessionName}`] = await saveVideoForPage( + test.artifacts[`video_${sessionName}`] = saveVideoForPage( this.sessionPages[sessionName], `${test.title}_${sessionName}.failed`, ) @@ -2424,9 +2424,9 @@ class Playwright extends Helper { async _passed(test) { if (this.options.recordVideo && this.page && this.page.video()) { if (this.options.keepVideoForPassedTests) { - test.artifacts.video = await saveVideoForPage(this.page, `${test.title}.passed`) + test.artifacts.video = saveVideoForPage(this.page, `${test.title}.passed`) for (const sessionName of Object.keys(this.sessionPages)) { - test.artifacts[`video_${sessionName}`] = await saveVideoForPage( + test.artifacts[`video_${sessionName}`] = saveVideoForPage( this.sessionPages[sessionName], `${test.title}_${sessionName}.passed`, ) @@ -3917,7 +3917,7 @@ async function refreshContextSession() { } } -async function saveVideoForPage(page, name) { +function saveVideoForPage(page, name) { if (!page.video()) return null const fileName = `${`${global.output_dir}${pathSeparator}videos${pathSeparator}${uuidv4()}_${clearString(name)}`.slice(0, 245)}.webm` page @@ -3928,11 +3928,10 @@ async function saveVideoForPage(page, name) { page .video() .delete() - .catch((e) => {}) + .catch(() => {}) }) return fileName } - async function saveTraceForContext(context, name) { if (!context) return if (!context.tracing) return diff --git a/lib/helper/REST.js b/lib/helper/REST.js index bd79b2ce6..4d36be1d9 100644 --- a/lib/helper/REST.js +++ b/lib/helper/REST.js @@ -63,6 +63,22 @@ const config = {} * } * ``` * + * ```js + * { + * helpers: { + * REST: { + * endpoint: 'http://site.com/api', + * prettyPrintJson: true, + * httpAgent: { + * ca: fs.readFileSync(__dirname + '/path/to/ca.pem'), + * rejectUnauthorized: false, + * keepAlive: true + * } + * } + * } + * } + * ``` + * * ## Access From Helpers * * Send REST requests by accessing `_executeRequest` method: @@ -101,9 +117,13 @@ class REST extends Helper { // Create an agent with SSL certificate if (this.options.httpAgent) { - if (!this.options.httpAgent.key || !this.options.httpAgent.cert) + // if one of those keys is there, all good to go + if (this.options.httpAgent.ca || this.options.httpAgent.key || this.options.httpAgent.cert) { + this.httpsAgent = new Agent(this.options.httpAgent) + } else { + // otherwise, throws an error of httpAgent config throw Error('Please recheck your httpAgent config!') - this.httpsAgent = new Agent(this.options.httpAgent) + } } this.axios = this.httpsAgent ? axios.create({ httpsAgent: this.httpsAgent }) : axios.create() diff --git a/lib/helper/SoftExpectHelper.js b/lib/helper/SoftExpectHelper.js new file mode 100644 index 000000000..326a8698c --- /dev/null +++ b/lib/helper/SoftExpectHelper.js @@ -0,0 +1,381 @@ +const ExpectHelper = require('./ExpectHelper') + +/** + * SoftAssertHelper is a utility class for performing soft assertions. + * Unlike traditional assertions that stop the execution on failure, + * soft assertions allow the execution to continue and report all failures at the end. + * + * ### Examples + * + * Zero-configuration when paired with other helpers like REST, Playwright: + * + * ```js + * // inside codecept.conf.js + * { + * helpers: { + * Playwright: {...}, + * SoftExpectHelper: {}, + * } + * } + * ``` + * + * ```js + * // in scenario + * I.softExpectEqual('a', 'b') + * I.flushSoftAssertions() // Throws an error if any soft assertions have failed. The error message contains all the accumulated failures. + * ``` + * + * ## Methods + */ +class SoftAssertHelper extends ExpectHelper { + constructor() { + super() + this.errors = [] + } + + /** + * Performs a soft assertion by executing the provided assertion function. + * If the assertion fails, the error is caught and stored without halting the execution. + * + * @param {Function} assertionFn - The assertion function to execute. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softAssert(assertionFn, customErrorMsg = '') { + try { + assertionFn() + } catch (error) { + this.errors.push({ customErrorMsg, error }) + } + } + + /** + * Throws an error if any soft assertions have failed. + * The error message contains all the accumulated failures. + * + * @throws {Error} If there are any soft assertion failures. + */ + flushSoftAssertions() { + if (this.errors.length > 0) { + let errorMessage = 'Soft assertions failed:\n' + this.errors.forEach((err, index) => { + errorMessage += `\n[${index + 1}] ${err.customErrorMsg}\n${err.error.message}\n` + }) + this.errors = [] + throw new Error(errorMessage) + } + } + + /** + * Softly asserts that two values are equal. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValue - The expected value. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectEqual(actualValue, expectedValue, customErrorMsg = '') { + this.softAssert(() => this.expectEqual(actualValue, expectedValue, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that two values are not equal. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValue - The expected value. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectNotEqual(actualValue, expectedValue, customErrorMsg = '') { + this.softAssert(() => this.expectNotEqual(actualValue, expectedValue, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that two values are deeply equal. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValue - The expected value. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectDeepEqual(actualValue, expectedValue, customErrorMsg = '') { + this.softAssert(() => this.expectDeepEqual(actualValue, expectedValue, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that two values are not deeply equal. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValue - The expected value. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectNotDeepEqual(actualValue, expectedValue, customErrorMsg = '') { + this.softAssert(() => this.expectNotDeepEqual(actualValue, expectedValue, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that a value contains the expected value. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValueToContain - The value that should be contained within the actual value. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectContain(actualValue, expectedValueToContain, customErrorMsg = '') { + this.softAssert(() => this.expectContain(actualValue, expectedValueToContain, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that a value does not contain the expected value. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValueToNotContain - The value that should not be contained within the actual value. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectNotContain(actualValue, expectedValueToNotContain, customErrorMsg = '') { + this.softAssert(() => this.expectNotContain(actualValue, expectedValueToNotContain, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that a value starts with the expected value. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValueToStartWith - The value that the actual value should start with. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectStartsWith(actualValue, expectedValueToStartWith, customErrorMsg = '') { + this.softAssert(() => this.expectStartsWith(actualValue, expectedValueToStartWith, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that a value does not start with the expected value. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValueToNotStartWith - The value that the actual value should not start with. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectNotStartsWith(actualValue, expectedValueToNotStartWith, customErrorMsg = '') { + this.softAssert( + () => this.expectNotStartsWith(actualValue, expectedValueToNotStartWith, customErrorMsg), + customErrorMsg, + ) + } + + /** + * Softly asserts that a value ends with the expected value. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValueToEndWith - The value that the actual value should end with. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectEndsWith(actualValue, expectedValueToEndWith, customErrorMsg = '') { + this.softAssert(() => this.expectEndsWith(actualValue, expectedValueToEndWith, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that a value does not end with the expected value. + * + * @param {*} actualValue - The actual value. + * @param {*} expectedValueToNotEndWith - The value that the actual value should not end with. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectNotEndsWith(actualValue, expectedValueToNotEndWith, customErrorMsg = '') { + this.softAssert( + () => this.expectNotEndsWith(actualValue, expectedValueToNotEndWith, customErrorMsg), + customErrorMsg, + ) + } + + /** + * Softly asserts that the target data matches the given JSON schema. + * + * @param {*} targetData - The data to validate. + * @param {Object} jsonSchema - The JSON schema to validate against. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectJsonSchema(targetData, jsonSchema, customErrorMsg = '') { + this.softAssert(() => this.expectJsonSchema(targetData, jsonSchema, customErrorMsg), customErrorMsg) + } + + /** + * Softly asserts that the target data matches the given JSON schema using AJV. + * + * @param {*} targetData - The data to validate. + * @param {Object} jsonSchema - The JSON schema to validate against. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + * @param {Object} [ajvOptions={ allErrors: true }] - Options to pass to AJV. + */ + softExpectJsonSchemaUsingAJV(targetData, jsonSchema, customErrorMsg = '', ajvOptions = { allErrors: true }) { + this.softAssert( + () => this.expectJsonSchemaUsingAJV(targetData, jsonSchema, customErrorMsg, ajvOptions), + customErrorMsg, + ) + } + + /** + * Softly asserts that the target data has the specified property. + * + * @param {*} targetData - The data to check. + * @param {string} propertyName - The property name to check for. + * @param {string} [customErrorMsg=''] - A custom error message to display if the assertion + fails. */ softExpectHasProperty(targetData, propertyName, customErrorMsg = '') { + this.softAssert(() => this.expectHasProperty(targetData, propertyName, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that the target data has a property with the specified name. + @param {*} targetData - The data to check. + @param {string} propertyName - The property name to check for. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. + */ + softExpectHasAProperty(targetData, propertyName, customErrorMsg = '') { + this.softAssert(() => this.expectHasAProperty(targetData, propertyName, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that the target data is of a specific type. + @param {*} targetData - The data to check. + @param {string} type - The expected type (e.g., 'string', 'number'). + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectToBeA(targetData, type, customErrorMsg = '') { + this.softAssert(() => this.expectToBeA(targetData, type, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that the target data is of a specific type (alternative for articles). + @param {*} targetData - The data to check. + @param {string} type - The expected type (e.g., 'string', 'number'). + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectToBeAn(targetData, type, customErrorMsg = '') { + this.softAssert(() => this.expectToBeAn(targetData, type, customErrorMsg), customErrorMsg) + } + + /* + Softly asserts that the target data matches the specified regular expression. + @param {*} targetData - The data to check. + @param {RegExp} regex - The regular expression to match. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectMatchRegex(targetData, regex, customErrorMsg = '') { + this.softAssert(() => this.expectMatchRegex(targetData, regex, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that the target data has a specified length. + @param {*} targetData - The data to check. + @param {number} length - The expected length. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectLengthOf(targetData, length, customErrorMsg = '') { + this.softAssert(() => this.expectLengthOf(targetData, length, customErrorMsg), customErrorMsg) + } + + /** + + Softly asserts that the target data is empty. + @param {*} targetData - The data to check. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectEmpty(targetData, customErrorMsg = '') { + this.softAssert(() => this.expectEmpty(targetData, customErrorMsg), customErrorMsg) + } + + /** + + Softly asserts that the target data is true. + @param {*} targetData - The data to check. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectTrue(targetData, customErrorMsg = '') { + this.softAssert(() => this.expectTrue(targetData, customErrorMsg), customErrorMsg) + } + + /** + + Softly asserts that the target data is false. + @param {*} targetData - The data to check. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectFalse(targetData, customErrorMsg = '') { + this.softAssert(() => this.expectFalse(targetData, customErrorMsg), customErrorMsg) + } + + /** + + Softly asserts that the target data is above a specified value. + @param {*} targetData - The data to check. + @param {*} aboveThan - The value that the target data should be above. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectAbove(targetData, aboveThan, customErrorMsg = '') { + this.softAssert(() => this.expectAbove(targetData, aboveThan, customErrorMsg), customErrorMsg) + } + + /** + + Softly asserts that the target data is below a specified value. + @param {*} targetData - The data to check. + @param {*} belowThan - The value that the target data should be below. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectBelow(targetData, belowThan, customErrorMsg = '') { + this.softAssert(() => this.expectBelow(targetData, belowThan, customErrorMsg), customErrorMsg) + } + + /** + + Softly asserts that the length of the target data is above a specified value. + @param {*} targetData - The data to check. + @param {number} lengthAboveThan - The length that the target data should be above. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectLengthAboveThan(targetData, lengthAboveThan, customErrorMsg = '') { + this.softAssert(() => this.expectLengthAboveThan(targetData, lengthAboveThan, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that the length of the target data is below a specified value. + @param {*} targetData - The data to check. + @param {number} lengthBelowThan - The length that the target data should be below. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectLengthBelowThan(targetData, lengthBelowThan, customErrorMsg = '') { + this.softAssert(() => this.expectLengthBelowThan(targetData, lengthBelowThan, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that two values are equal, ignoring case. + @param {string} actualValue - The actual string value. + @param {string} expectedValue - The expected string value. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectEqualIgnoreCase(actualValue, expectedValue, customErrorMsg = '') { + this.softAssert(() => this.expectEqualIgnoreCase(actualValue, expectedValue, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that two arrays have deep equality, considering members in any order. + @param {Array} actualValue - The actual array. + @param {Array} expectedValue - The expected array. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectDeepMembers(actualValue, expectedValue, customErrorMsg = '') { + this.softAssert(() => this.expectDeepMembers(actualValue, expectedValue, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that an array (superset) deeply includes all members of another array (set). + @param {Array} superset - The array that should contain the expected members. + @param {Array} set - The array with members that should be included. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectDeepIncludeMembers(superset, set, customErrorMsg = '') { + this.softAssert(() => this.expectDeepIncludeMembers(superset, set, customErrorMsg), customErrorMsg) + } + + /** + Softly asserts that two objects are deeply equal, excluding specified fields. + @param {Object} actualValue - The actual object. + @param {Object} expectedValue - The expected object. + @param {Array} fieldsToExclude - The fields to exclude from the comparison. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectDeepEqualExcluding(actualValue, expectedValue, fieldsToExclude, customErrorMsg = '') { + this.softAssert( + () => this.expectDeepEqualExcluding(actualValue, expectedValue, fieldsToExclude, customErrorMsg), + customErrorMsg, + ) + } + + /** + Softly asserts that a value matches the expected pattern. + @param {*} actualValue - The actual value. + @param {*} expectedPattern - The pattern the value should match. + @param {string} [customErrorMsg=''] - A custom error message to display if the assertion fails. */ + softExpectMatchesPattern(actualValue, expectedPattern, customErrorMsg = '') { + this.softAssert(() => this.expectMatchesPattern(actualValue, expectedPattern, customErrorMsg), customErrorMsg) + } +} +module.exports = SoftAssertHelper diff --git a/lib/locator.js b/lib/locator.js index f10a94399..eef8277f7 100644 --- a/lib/locator.js +++ b/lib/locator.js @@ -299,6 +299,49 @@ class Locator { return new Locator({ xpath }); } + /** + * Adds condition: attribute value starts with text + * (analog of XPATH: [starts-with(@attr,'startValue')] or CSS [attr^='startValue'] + * Example: I.click(locate('a').withAttrStartsWith('href', 'https://'))); + * Works with any attribute: class, href etc. + * @param {string} attrName + * @param {string} startsWith + * @returns {Locator} + */ + withAttrStartsWith(attrName, startsWith) { + const xpath = sprintf('%s[%s]', this.toXPath(), `starts-with(@${attrName}, "${startsWith}")`); + return new Locator({ xpath }); + } + + /** + * Adds condition: attribute value ends with text + * (analog of XPATH: [ends-with(@attr,'endValue')] or CSS [attr$='endValue'] + * Example: I.click(locate('a').withAttrEndsWith('href', '.com'))); + * Works with any attribute: class, href etc. + * @param {string} attrName + * @param {string} endsWith + * @returns {Locator} + */ + withAttrEndsWith(attrName, endsWith) { + const xpath = sprintf('%s[%s]', this.toXPath(), `substring(@${attrName}, string-length(@${attrName}) - string-length("${endsWith}") + 1) = "${endsWith}"`, + ); + return new Locator({ xpath }); + } + + /** + * Adds condition: attribute value contains text + * (analog of XPATH: [contains(@attr,'partOfAttribute')] or CSS [attr*='partOfAttribute'] + * Example: I.click(locate('a').withAttrContains('href', 'google'))); + * Works with any attribute: class, href etc. + * @param {string} attrName + * @param {string} partOfAttrValue + * @returns {Locator} + */ + withAttrContains(attrName, partOfAttrValue) { + const xpath = sprintf('%s[%s]', this.toXPath(), `contains(@${attrName}, "${partOfAttrValue}")`); + return new Locator({ xpath }); + } + /** * @param {String} text * @returns {Locator} diff --git a/lib/output.js b/lib/output.js index 72aa3a053..cb15f7e1d 100644 --- a/lib/output.js +++ b/lib/output.js @@ -206,7 +206,7 @@ module.exports = { * @param {number} skipped * @param {number|string} duration */ - result(passed, failed, skipped, duration) { + result(passed, failed, skipped, duration, failedHooks = 0) { let style = colors.bgGreen; let msg = ` ${passed || 0} passed`; let status = style.bold(' OK '); @@ -215,6 +215,12 @@ module.exports = { status = style.bold(' FAIL '); msg += `, ${failed} failed`; } + + if (failedHooks > 0) { + style = style.bgRed; + status = style.bold(' FAIL '); + msg += `, ${failedHooks} failedHooks`; + } status += style.grey(' |'); if (skipped) { diff --git a/lib/rerun.js b/lib/rerun.js index b39b0b62e..df4549209 100644 --- a/lib/rerun.js +++ b/lib/rerun.js @@ -70,6 +70,7 @@ class CodeceptRerunner extends BaseCodecept { await this.runTests(test); } catch (e) { output.error(e.stack); + throw e; } finally { event.emit(event.all.result, this); event.emit(event.all.after, this); diff --git a/lib/template/heal.js b/lib/template/heal.js index a893de9ea..782627aa0 100644 --- a/lib/template/heal.js +++ b/lib/template/heal.js @@ -5,6 +5,7 @@ heal.addRecipe('ai', { prepare: { html: ({ I }) => I.grabHTMLFrom('body'), }, + suggest: true, steps: [ 'click', 'fillField', diff --git a/lib/workers.js b/lib/workers.js index 78c38a3bf..429091b36 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -357,6 +357,7 @@ class Workers extends EventEmitter { run() { this.stats.start = new Date(); + this.stats.failedHooks = 0 recorder.startUnlessRunning(); event.dispatcher.emit(event.workers.before); process.env.RUNS_WITH_WORKERS = 'true'; @@ -471,6 +472,7 @@ class Workers extends EventEmitter { this.stats.failures += newStats.failures; this.stats.tests += newStats.tests; this.stats.pending += newStats.pending; + this.stats.failedHooks += newStats.failedHooks; } printResults() { @@ -492,7 +494,7 @@ class Workers extends EventEmitter { this.failuresLog.forEach(log => output.print(...log)); } - output.result(this.stats.passes, this.stats.failures, this.stats.pending, ms(this.stats.duration)); + output.result(this.stats.passes, this.stats.failures, this.stats.pending, ms(this.stats.duration), this.stats.failedHooks); process.env.RUNS_WITH_WORKERS = 'false'; } } diff --git a/package.json b/package.json index f032bd87a..f1fb287d6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeceptjs", - "version": "3.6.5", + "version": "3.6.6", "description": "Supercharged End 2 End Testing Framework for NodeJS", "keywords": [ "acceptance", @@ -66,7 +66,8 @@ "types-fix": "node typings/fixDefFiles.js", "dtslint": "npm run types-fix && tsd", "prepare": "husky install", - "prepare-release": "./runok.js versioning && ./runok.js get:commit-log" + "prepare-release": "./runok.js versioning && ./runok.js get:commit-log", + "publish-beta": "./runok.js publish:next-beta-version" }, "dependencies": { "@codeceptjs/configure": "1.0.1", @@ -77,7 +78,7 @@ "@xmldom/xmldom": "0.8.10", "acorn": "8.12.1", "arrify": "2.0.1", - "axios": "1.7.2", + "axios": "1.7.7", "chai": "5.1.1", "chai-deep-match": "1.2.1", "chai-exclude": "2.1.1", @@ -90,7 +91,7 @@ "cross-spawn": "7.0.3", "css-to-xpath": "0.1.0", "csstoxpath": "1.6.0", - "devtools": "8.39.1", + "devtools": "8.40.2", "envinfo": "7.11.1", "escape-string-regexp": "4.0.0", "figures": "3.2.0", @@ -104,57 +105,58 @@ "lodash.clonedeep": "4.5.0", "lodash.merge": "4.6.2", "mkdirp": "1.0.4", - "mocha": "10.6.0", - "monocart-coverage-reports": "2.10.0", + "mocha": "10.7.3", + "monocart-coverage-reports": "2.10.3", "ms": "2.1.3", "ora-classic": "5.4.2", - "pactum": "3.6.9", + "pactum": "3.7.1", "parse-function": "5.6.10", "parse5": "7.1.2", "promise-retry": "1.1.1", "resq": "1.11.0", "sprintf-js": "1.1.1", - "uuid": "9.0" + "uuid": "10.0" }, "optionalDependencies": { - "@codeceptjs/detox-helper": "1.0.8" + "@codeceptjs/detox-helper": "1.1.2" }, "devDependencies": { "@codeceptjs/mock-request": "0.3.1", "@faker-js/faker": "7.6.0", "@pollyjs/adapter-puppeteer": "6.0.6", "@pollyjs/core": "5.1.0", - "@types/chai": "4.3.16", + "@types/chai": "4.3.19", "@types/inquirer": "9.0.3", - "@types/node": "20.11.30", - "@wdio/sauce-service": "8.39.1", + "@types/node": "22.5.5", + "@wdio/sauce-service": "9.0.4", "@wdio/selenium-standalone-service": "8.3.2", - "@wdio/utils": "8.38.2", + "@wdio/utils": "9.0.6", "@xmldom/xmldom": "0.8.10", "apollo-server-express": "2.25.3", "chai-as-promised": "7.1.2", "chai-subset": "1.6.0", "contributor-faces": "1.1.0", "documentation": "12.3.0", - "electron": "31.3.0", + "electron": "31.3.1", "eslint": "8.57.0", "eslint-config-airbnb-base": "15.0.0", - "eslint-plugin-import": "2.29.1", - "eslint-plugin-mocha": "10.4.3", + "eslint-plugin-import": "2.30.0", + "eslint-plugin-mocha": "10.5.0", "expect": "29.7.0", "express": "4.19.2", "graphql": "16.9.0", - "husky": "9.1.1", + "husky": "9.1.5", "inquirer-test": "2.0.1", "jsdoc": "4.0.3", "jsdoc-typeof-plugin": "1.0.0", "json-server": "0.10.1", "playwright": "1.45.3", "prettier": "^3.3.2", - "puppeteer": "22.12.1", + "puppeteer": "23.3.0", "qrcode-terminal": "0.12.0", "rosie": "2.1.1", "runok": "0.9.3", + "semver": "7.6.3", "sinon": "18.0.0", "sinon-chai": "3.7.0", "testcafe": "3.5.0", @@ -162,9 +164,9 @@ "ts-node": "10.9.2", "tsd": "^0.31.0", "tsd-jsdoc": "2.5.0", - "typedoc": "0.26.5", - "typedoc-plugin-markdown": "4.2.1", - "typescript": "5.5.3", + "typedoc": "0.26.7", + "typedoc-plugin-markdown": "4.2.6", + "typescript": "5.6.2", "wdio-docker-service": "1.5.0", "webdriverio": "8.39.1", "xml2js": "0.6.2", @@ -181,4 +183,4 @@ "strict": false } } -} +} \ No newline at end of file diff --git a/runok.js b/runok.js index 2e09ef3e7..6e9f09989 100755 --- a/runok.js +++ b/runok.js @@ -10,6 +10,8 @@ const { runok, } = require('runok') const contributors = require('contributor-faces') +const { execSync } = require('node:child_process') +const semver = require('semver') const helperMarkDownFile = function (name) { return `docs/helpers/${name}.md` @@ -478,6 +480,47 @@ ${changelog}` ) fs.writeFileSync('./README.md', readmeContent) }, + + getCurrentBetaVersion() { + try { + const output = execSync('npm view codeceptjs versions --json').toString() + const versions = JSON.parse(output) + const betaVersions = versions.filter((version) => version.includes('beta')) + const latestBeta = betaVersions.length ? betaVersions[betaVersions.length - 1] : null + console.log(`Current beta version: ${latestBeta}`) + return latestBeta + } catch (error) { + console.error('Error fetching package versions:', error) + process.exit(1) + } + }, + + publishNextBetaVersion() { + const currentBetaVersion = this.getCurrentBetaVersion() + if (!currentBetaVersion) { + console.error('No beta version found.') + process.exit(1) + } + + const nextBetaVersion = semver.inc(currentBetaVersion, 'prerelease', 'beta') + console.log(`Publishing version: ${nextBetaVersion}`) + + try { + // Save original version + const packageJson = JSON.parse(fs.readFileSync('package.json', 'utf8')) + const originalVersion = packageJson.version + execSync(`npm version ${nextBetaVersion} --no-git-tag-version`) + execSync('npm publish --tag beta') + console.log(`Successfully published ${nextBetaVersion}`) + + // Revert to original version + execSync(`npm version ${originalVersion} --no-git-tag-version`) + console.log(`Reverted back to original version: ${originalVersion}`) + } catch (error) { + console.error('Error publishing package:', error) + process.exit(1) + } + }, } async function processChangelog() { diff --git a/test/data/sandbox/configs/run-rerun/codecept.conf.pass_all_test.js b/test/data/sandbox/configs/run-rerun/codecept.conf.pass_all_test.js new file mode 100644 index 000000000..3c2ef30d8 --- /dev/null +++ b/test/data/sandbox/configs/run-rerun/codecept.conf.pass_all_test.js @@ -0,0 +1,16 @@ +exports.config = { + tests: './*_ftest.js', + output: './output', + helpers: { + CustomHelper: { + require: './customHelper.js', + }, + }, + rerun: { + minSuccess: 3, + maxReruns: 3, + }, + bootstrap: null, + mocha: {}, + name: 'run-rerun', +}; diff --git a/test/helper/SoftExpect_test.js b/test/helper/SoftExpect_test.js new file mode 100644 index 000000000..d6f4af136 --- /dev/null +++ b/test/helper/SoftExpect_test.js @@ -0,0 +1,479 @@ +const path = require('path') + +let expect +import('chai').then((chai) => { + expect = chai.expect +}) + +const SoftAssertHelper = require('../../lib/helper/SoftExpectHelper') + +global.codeceptjs = require('../../lib') + +let I + +const goodApple = { + skin: 'thin', + colors: ['red', 'green', 'yellow'], + taste: 10, +} +const badApple = { + colors: ['brown'], + taste: 0, + worms: 2, +} +const fruitSchema = { + title: 'fresh fruit schema v1', + type: 'object', + required: ['skin', 'colors', 'taste'], + properties: { + colors: { + type: 'array', + minItems: 1, + uniqueItems: true, + items: { + type: 'string', + }, + }, + skin: { + type: 'string', + }, + taste: { + type: 'number', + minimum: 5, + }, + }, +} + +describe('Soft Expect Helper', function () { + this.timeout(3000) + this.retries(1) + + before(() => { + global.codecept_dir = path.join(__dirname, '/../data') + + I = new SoftAssertHelper() + }) + + describe('#softExpectEqual', () => { + it('should not show error', () => { + I.softExpectEqual('a', 'a') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectEqual('a', 'b') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain("expected 'a' to equal 'b'") + } + }) + }) + + describe('#softExpectNotEqual', () => { + it('should not show error', () => { + I.softExpectNotEqual('a', 'b') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectNotEqual('a', 'a') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain("expected 'a' to not equal 'a'") + } + }) + }) + + describe('#softExpectContain', () => { + it('should not show error', () => { + I.softExpectContain('abc', 'a') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectContain('abc', 'd') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain("expected 'abc' to include 'd'") + } + }) + }) + + describe('#softExpectNotContain', () => { + it('should not show error', () => { + I.softExpectNotContain('abc', 'd') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectNotContain('abc', 'a') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain("expected 'abc' to not include 'a'") + } + }) + }) + + describe('#softExpectStartsWith', () => { + it('should not show error', () => { + I.softExpectStartsWith('abc', 'a') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectStartsWith('abc', 'b') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected abc to start with b') + } + }) + }) + + describe('#softExpectNotStartsWith', () => { + it('should not show error', () => { + I.softExpectNotStartsWith('abc', 'b') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectNotStartsWith('abc', 'a') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected abc not to start with a') + } + }) + }) + + describe('#softExpectEndsWith', () => { + it('should not show error', () => { + I.softExpectEndsWith('abc', 'c') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectEndsWith('abc', 'd') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected abc to end with d') + } + }) + }) + + describe('#softExpectNotEndsWith', () => { + it('should not show error', () => { + I.softExpectNotEndsWith('abc', 'd') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectNotEndsWith('abc', 'd') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected abc not to end with c') + } + }) + }) + + describe('#softExpectJsonSchema', () => { + it('should not show error', () => { + I.softExpectJsonSchema(goodApple, fruitSchema) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectJsonSchema(badApple, fruitSchema) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected value to match json-schema') + } + }) + }) + + describe('#softExpectHasProperty', () => { + it('should not show error', () => { + I.softExpectHasProperty(goodApple, 'skin') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectHasProperty(badApple, 'skin') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected { Object (colors, taste') + } + }) + }) + + describe('#softExpectHasAProperty', () => { + it('should not show error', () => { + I.softExpectHasAProperty(goodApple, 'skin') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectHasAProperty(badApple, 'skin') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected { Object (colors, taste') + } + }) + }) + + describe('#softExpectToBeA', () => { + it('should not show error', () => { + I.softExpectToBeA(goodApple, 'object') + I.flushSoftAssertions() + }) + }) + + describe('#softExpectToBeAn', () => { + it('should not show error', () => { + I.softExpectToBeAn(goodApple, 'object') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectToBeAn(badApple, 'skin') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected { Object (colors, taste') + } + }) + }) + + describe('#softExpectMatchRegex', () => { + it('should not show error', () => { + I.softExpectMatchRegex('goodApple', /good/) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectMatchRegex('Apple', /good/) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('to match /good/') + } + }) + }) + + describe('#softExpectLengthOf', () => { + it('should not show error', () => { + I.softExpectLengthOf('good', 4) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectLengthOf('Apple', 4) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('to have a length') + } + }) + }) + + describe('#softExpectTrue', () => { + it('should not show error', () => { + I.softExpectTrue(true) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectTrue(false) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected false to be true') + } + }) + }) + + describe('#softExpectEmpty', () => { + it('should not show error', () => { + I.softExpectEmpty('') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectEmpty('false') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain("expected 'false' to be empty") + } + }) + }) + + describe('#softExpectFalse', () => { + it('should not show error', () => { + I.softExpectFalse(false) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectFalse(true) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected true to be false') + } + }) + }) + + describe('#softExpectAbove', () => { + it('should not show error', () => { + I.softExpectAbove(2, 1) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectAbove(1, 2) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected 1 to be above 2') + } + }) + }) + + describe('#softExpectBelow', () => { + it('should not show error', () => { + I.softExpectBelow(1, 2) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectBelow(2, 1) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected 2 to be below 1') + } + }) + }) + + describe('#softExpectLengthAboveThan', () => { + it('should not show error', () => { + I.softExpectLengthAboveThan('hello', 4) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectLengthAboveThan('hello', 5) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('to have a length above 5') + } + }) + }) + + describe('#softExpectLengthBelowThan', () => { + it('should not show error', () => { + I.softExpectLengthBelowThan('hello', 6) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectLengthBelowThan('hello', 4) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('to have a length below 4') + } + }) + }) + + describe('#softExpectLengthBelowThan', () => { + it('should not show error', () => { + I.softExpectEqualIgnoreCase('hEllo', 'hello') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectEqualIgnoreCase('hEllo', 'hell0') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected hEllo to equal hell0') + } + }) + }) + + describe('#softExpectDeepMembers', () => { + it('should not show error', () => { + I.softExpectDeepMembers([1, 2, 3], [1, 2, 3]) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectDeepMembers([1, 2, 3], [3]) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected [ 1, 2, 3 ] to have the same members') + } + }) + }) + + describe('#softExpectDeepIncludeMembers', () => { + it('should not show error', () => { + I.softExpectDeepIncludeMembers([3, 4, 5, 6], [3, 4, 5]) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectDeepIncludeMembers([3, 4, 5], [3, 4, 5, 6]) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected [ 3, 4, 5 ] to be a superset of [ 3, 4, 5, 6 ]') + } + }) + }) + + describe('#softExpectDeepEqualExcluding', () => { + it('should not show error', () => { + I.softExpectDeepEqualExcluding({ a: 1, b: 2 }, { b: 2, a: 1, c: 3 }, 'c') + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectDeepEqualExcluding({ a: 1, b: 2 }, { b: 2, a: 1, c: 3 }, 'a') + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain('expected { b: 2 } to deeply equal') + } + }) + }) + + describe('#softExpectLengthBelowThan', () => { + it('should not show error', () => { + I.softExpectMatchesPattern('123', /123/) + I.flushSoftAssertions() + }) + + it('should show error', () => { + try { + I.softExpectMatchesPattern('123', /1235/) + I.flushSoftAssertions() + } catch (e) { + expect(e.message).to.contain("didn't match target /1235/") + } + }) + }) +}) diff --git a/test/runner/before_failure_test.js b/test/runner/before_failure_test.js index c0b243bb1..6c54ecb33 100644 --- a/test/runner/before_failure_test.js +++ b/test/runner/before_failure_test.js @@ -9,10 +9,10 @@ describe('Failure in before', function () { this.timeout(40000) it('should skip tests that are skipped because of failure in before hook', (done) => { exec(`${codecept_run}`, (err, stdout) => { - stdout.should.include('✔ First test will be passed') - stdout.should.include('S Third test will be skipped @grep') - stdout.should.include('S Fourth test will be skipped') - stdout.should.include('1 passed, 1 failed, 2 skipped') + stdout.should.include('First test will be passed @grep') + stdout.should.include('Third test will be skipped @grep') + stdout.should.include('Fourth test will be skipped') + stdout.should.include('1 passed, 1 failedHooks, 2 skipped') err.code.should.eql(1) done() }) @@ -20,9 +20,9 @@ describe('Failure in before', function () { it('should skip tests correctly with grep options', (done) => { exec(`${codecept_run} --grep @grep`, (err, stdout) => { - stdout.should.include('✔ First test will be passed') - stdout.should.include('S Third test will be skipped @grep') - stdout.should.include('1 passed, 1 failed, 1 skipped') + stdout.should.include('First test will be passed @grep') + stdout.should.include('Third test will be skipped @grep') + stdout.should.include('1 passed, 1 failedHooks, 1 skipped') err.code.should.eql(1) done() }) diff --git a/test/runner/run_rerun_test.js b/test/runner/run_rerun_test.js index 75511583a..a38ba54af 100644 --- a/test/runner/run_rerun_test.js +++ b/test/runner/run_rerun_test.js @@ -83,7 +83,7 @@ describe('run-rerun command', () => { ) }) - it('should display success run if test was fail one time of two attepmts and 3 reruns', (done) => { + it('should display success run if test was fail one time of two attempts and 3 reruns', (done) => { exec( `FAIL_ATTEMPT=0 ${codecept_run_config('codecept.conf.fail_test.js', '@RunRerun - fail second test')} --debug`, (err, stdout) => { @@ -96,4 +96,18 @@ describe('run-rerun command', () => { }, ) }) + + it('should throw exit code 1 if all tests were supposed to pass', (done) => { + exec( + `FAIL_ATTEMPT=0 ${codecept_run_config('codecept.conf.pass_all_test.js', '@RunRerun - fail second test')} --debug`, + (err, stdout) => { + expect(stdout).toContain('Process run 1 of max 3, success runs 1/3') + expect(stdout).toContain('Fail run 2 of max 3, success runs 1/3') + expect(stdout).toContain('Process run 3 of max 3, success runs 2/3') + expect(stdout).toContain('Flaky tests detected!') + expect(err.code).toBe(1) + done() + }, + ) + }) }) diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 063b3d889..7bbfa6b4c 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -40,12 +40,11 @@ describe('CodeceptJS Workers Runner', function () { expect(stdout).toContain('glob current dir') expect(stdout).toContain('From worker @1_grep print message 1') expect(stdout).toContain('From worker @2_grep print message 2') - expect(stdout).toContain('Running tests in 3 workers') + expect(stdout).toContain('Running tests in') expect(stdout).not.toContain('this is running inside worker') expect(stdout).toContain('failed') expect(stdout).toContain('File notafile not found') - expect(stdout).toContain('Scenario Steps:') - expect(stdout).toContain('FAIL | 5 passed, 2 failed') + expect(stdout).toContain('5 passed, 1 failed, 1 failedHooks') // We are not testing order in logs, because it depends on race condition between workers expect(stdout).toContain(') Workers Failing\n') // first fail log expect(stdout).toContain(') Workers\n') // second fail log diff --git a/test/unit/locator_test.js b/test/unit/locator_test.js index 41680548f..0993c416e 100644 --- a/test/unit/locator_test.js +++ b/test/unit/locator_test.js @@ -455,4 +455,22 @@ describe('Locator', () => { const nodes = xpath.select(l.toXPath(), doc) expect(nodes).to.have.length(0, l.toXPath()) }) + + it('should find element with attribute value starts with text', () => { + const l = Locator.build('a').withAttrStartsWith('class', 'ps-menu-button') + const nodes = xpath.select(l.toXPath(), doc) + expect(nodes).to.have.length(10, l.toXPath()) + }) + + it('should find element with attribute value ends with text', () => { + const l = Locator.build('a').withAttrEndsWith('class', 'ps-menu-button') + const nodes = xpath.select(l.toXPath(), doc) + expect(nodes).to.have.length(9, l.toXPath()) + }) + + it('should find element with attribute value contains text', () => { + const l = Locator.build('a').withAttrEndsWith('class', 'active') + const nodes = xpath.select(l.toXPath(), doc) + expect(nodes).to.have.length(1, l.toXPath()) + }) })