diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..43e8b9d3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +ignore = E203, W503 +max-line-length = 119 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..6e94eb9a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps to Reproduce** +Steps to reproduce the behavior: +1. ... +2. ... + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Actual behavior** +What actually happened. + +**Package versions** +Include version info of the following packages: reportportal-client, pytest-reportportal + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..bbcbbe7d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..79231ca4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,129 @@ +# Copyright 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Release Pytest agent + +on: + push: + branches: [ 'master' ] + paths-ignore: + - '.github/**' + - CHANGELOG.md + - README.md + - CONTRIBUTING.rst + +env: + VERSION_FILE: setup.py + VERSION_EXTRACT_PATTERN: >- + __version__\s*=\s*"([^"]+) + VERSION_REPLACE_PATTERN: >- + __version__ = "\1" + TMP_SUFFIX: _updated + CHANGE_LOG_FILE: CHANGELOG.md + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Generate versions + uses: HardNorth/github-version-generate@v1 + with: + version-source: file + version-file: ${{ env.VERSION_FILE }} + version-file-extraction-pattern: ${{ env.VERSION_EXTRACT_PATTERN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: python -m pip install --upgrade pip setuptools wheel + + - name: Build package + run: python setup.py sdist bdist_wheel + + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: ${{ secrets.PYPI_USERNAME }} + password: ${{ secrets.PYPI_PASSWORD }} + + - name: Setup git credentials + uses: oleksiyrudenko/gha-git-credentials@v2-latest + with: + name: 'reportportal.io' + email: 'support@reportportal.io' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Tagging new version + id: newVersionTag + run: | + git tag -a ${{ env.RELEASE_VERSION }} -m "Release ${{ env.RELEASE_VERSION }}" + git push --tags + + - name: Checkout develop branch + uses: actions/checkout@v4 + with: + ref: 'develop' + fetch-depth: 0 + + - name: Merge release branch into develop + id: mergeIntoDevelop + run: | + git merge -m 'Merge master branch into develop after a release' origin/master + git status | (! grep -Fq 'both modified:') || git status | grep -F 'both modified:' \ + | { echo -e 'Unable to merge master into develop, merge conflicts:'; (! grep -Eo '[^ ]+$') } + + - name: Update CHANGELOG.md + id: changelogUpdate + run: | + sed '/\[Unreleased\]/q' ${{ env.CHANGE_LOG_FILE }} >> ${{ env.CHANGE_LOG_FILE }}${{ env.TMP_SUFFIX }} + sed -E '1,/#?#\s*\[Unreleased\]/d' ${{ env.CHANGE_LOG_FILE }} | sed -E '/#?#\s*\[/q' | \ + { echo -e '\n## [${{ env.RELEASE_VERSION }}]'; sed '$d'; } >> ${{ env.CHANGE_LOG_FILE }}${{ env.TMP_SUFFIX }} + grep -E '#?#\s*\[[0-9]' ${{ env.CHANGE_LOG_FILE }} | head -n1 >> ${{ env.CHANGE_LOG_FILE }}${{ env.TMP_SUFFIX }} + sed -E '1,/#?#\s*\[[0-9]/d' ${{ env.CHANGE_LOG_FILE }} >> ${{ env.CHANGE_LOG_FILE }}${{ env.TMP_SUFFIX }} + rm ${{ env.CHANGE_LOG_FILE }} + mv ${{ env.CHANGE_LOG_FILE }}${{ env.TMP_SUFFIX }} ${{ env.CHANGE_LOG_FILE }} + git add ${{ env.CHANGE_LOG_FILE }} + git commit -m "Changelog update" + + - name: Read changelog Entry + id: readChangelogEntry + uses: mindsers/changelog-reader-action@v2 + with: + version: ${{ env.RELEASE_VERSION }} + path: ./${{ env.CHANGE_LOG_FILE }} + + - name: Create Release + id: createRelease + uses: ncipollo/release-action@v1 + with: + tag: ${{ env.RELEASE_VERSION }} + name: Release ${{ env.RELEASE_VERSION }} + body: ${{ steps.readChangelogEntry.outputs.changes }} + + - name: Update version file + id: versionFileUpdate + run: | + export CURRENT_VERSION_VALUE=`echo '${{ env.CURRENT_VERSION }}' | sed -E 's/(.*)/${{ env.VERSION_REPLACE_PATTERN }}/'` + export NEXT_VERSION_VALUE=`echo '${{ env.NEXT_VERSION }}' | sed -E 's/(.*)/${{ env.VERSION_REPLACE_PATTERN }}/'` + sed "s/${CURRENT_VERSION_VALUE}/${NEXT_VERSION_VALUE}/g" ${{ env.VERSION_FILE }} > ${{ env.VERSION_FILE }}${{ env.TMP_SUFFIX }} + rm ${{ env.VERSION_FILE }} + mv ${{ env.VERSION_FILE }}${{ env.TMP_SUFFIX }} ${{ env.VERSION_FILE }} + git add ${{ env.VERSION_FILE }} + git commit -m 'Version update' + git push diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..be901e66 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,48 @@ +# Copyright 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Tests + +on: [ push, pull_request ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13', '3.14' ] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + + - name: Test with tox + run: tox + + - name: Upload coverage to Codecov + if: matrix.python-version == 3.10 && success() + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + flags: unittests + name: codecov-client-reportportal diff --git a/.gitignore b/.gitignore index 62e78209..e142e847 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,137 @@ -# Created by .ignore support plugin (hsz.mobi) -.idea \ No newline at end of file +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +.idea/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +.venv38 +.venv39 +.venv310 +.venv311 +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +AGENTS.md +PROMPTS.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..8b2adf59 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - repo: https://github.com/PyCQA/pydocstyle + rev: 6.3.0 + hooks: + - id: pydocstyle + exclude: | + (?x)^( + tests/.* | + examples/.* + ) + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + args: [ '--check', 'pytest_reportportal', 'tests' ] + - repo: https://github.com/pycqa/isort + rev: 6.0.0 + hooks: + - id: isort + - repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9ed79f1e..00000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: python -python: - - "2.6" - - "2.7" - - "3.3" - - "3.4" - - "3.5" - - "3.6" -script: - - python setup.py -q install diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..d4d7ddf3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,234 @@ +# Changelog + +## [Unreleased] + +## [5.6.11] +### Changed +- Client version updated to [5.7.10](https://github.com/reportportal/client-Python/releases/tag/5.7.10), by @ableeck + +## [5.6.10] +### Changed +- Client version updated to [5.7.9](https://github.com/reportportal/client-Python/releases/tag/5.7.9), by @ableeck + +## [5.6.9] +### Changed +- Client version updated to [5.7.7](https://github.com/reportportal/client-Python/releases/tag/5.7.7), by @ableeck + +## [5.6.8] +### Changed +- Client version updated to [5.7.6](https://github.com/reportportal/client-Python/releases/tag/5.7.6), by @ableeck + +## [5.6.7] +### Added +- Attribute splitting if they are passed as `str` in configs, by @HardNorth + +## [5.6.6] +### Added +- Microseconds precision for timestamps, by @HardNorth +### Changed +- Client version updated to [5.7.4](https://github.com/reportportal/client-Python/releases/tag/5.7.4), by @HardNorth +### Removed +- Redundant item name truncation (now handled on the Client level), by @HardNorth + +## [5.6.5] +### Fixed +- Issue [#418](https://github.com/reportportal/agent-python-pytest/issues/418) parametrize marker IDs, by @drcrazy + +## [5.6.4] +### Fixed +- Agent crash in certain `pytest-bdd` cases, by @HardNorth + +## [5.6.3] +### Changed +- Client version updated to [5.7.2](https://github.com/reportportal/client-Python/releases/tag/5.7.2), by @HardNorth +### Removed +- `aenum` dependency, by @HardNorth +- `dill` dependency, by @HardNorth + +## [5.6.2] +### Added +- `rp_launch_uuid` configuration parameter, by @HardNorth +- `rp_launch_attributes` cmd argument, by @fahadnaeemkhan +### Changed +- Client version updated on [5.7.1](https://github.com/reportportal/client-Python/releases/tag/5.7.1), by @HardNorth + +## [5.6.1] +### Added +- Config variables override by environment variables logic, by @HardNorth +### Fixed +- Code reference generation in `xdist`, by @Sgitario + +## [5.6.0] +### Added +- Official `Python 3.14` support, by @HardNorth +- Issue [#396](https://github.com/reportportal/agent-python-pytest/issues/396) parametrize marker IDs, by @HardNorth +- Custom log level handling with `rp_log_custom_levels` configuration parameter, by @HardNorth +### Changed +- Client version updated on [5.7.0](https://github.com/reportportal/client-Python/releases/tag/5.7.0), by @HardNorth +### Removed +- `Python 3.8` support, by @HardNorth +- Deprecated `retries` parameter, by @HardNorth + +## [5.5.4] +### Added +- Return back deprecated `rp_log_batch_payload_size` parameter for sake of backward compatibility, by @HardNorth + +## [5.5.3] +### Added +- OAuth 2.0 Password Grant authentication, by @HardNorth +### Changed +- Client version updated to [5.6.7](https://github.com/reportportal/client-Python/releases/tag/5.6.7), by @HardNorth +### Fixed +- Some configuration parameter names, which are different in the client, by @HardNorth +### Removed +- `rp_uuid` param support, as it was deprecated pretty while ago, by @HardNorth + +## [5.5.2] +### Fixed +- Issue [#397](https://github.com/reportportal/agent-python-pytest/issues/397) pytest launch stuck with xdist, by @HardNorth + +## [5.5.1] +### Added +- Too long BDD Step name truncation on reporting, by @HardNorth +- `use_index` parameter for `@pytest.mark.parametrize` decorator, which replaces parameter values with their indexes in the list, by @ramir-dn + +## [5.5.0] +### Added +- Issue [#357](https://github.com/reportportal/agent-python-pytest/issues/357) `pytest-bdd` support, by @HardNorth +### Fixed +- Issue [#389](https://github.com/reportportal/agent-python-pytest/issues/389) `rp_tests_attributes` configuration parameter handling, by @HardNorth +- Issue [#390](https://github.com/reportportal/agent-python-pytest/issues/390) INTERNALERROR due to pytest.exit within fixture, by @HardNorth +### Removed +- `Python 3.7` support, by @HardNorth + +## [5.4.7] +### Added +- Issue [#382](https://github.com/reportportal/agent-python-pytest/issues/382): Escaping of binary symbol '\0' in parameters, by @HardNorth +### Changed +- Client version updated on [5.6.0](https://github.com/reportportal/client-Python/releases/tag/5.6.0), by @HardNorth + +## [5.4.6] +### Added +- Support for `Python 3.13`, by @HardNorth +- Support for `name` Pytest marker, by @HardNorth +- `rp_hierarchy_test_file` configuration parameter, which controls display of test file name in the hierarchy, by @ramir-dn, @HardNorth +### Fixed +- Agent crash if Client could not be initialized, by @HardNorth +### Changed +- Client version updated on [5.5.10](https://github.com/reportportal/client-Python/releases/tag/5.5.10), by @HardNorth + +## [5.4.5] +### Fixed +- Issue [#379](https://github.com/reportportal/agent-python-pytest/issues/379): Fix TypeError when using pytest.skip() in fixtures, by @HardNorth + +## [5.4.4] +### Fixed +- Issue [#375](https://github.com/reportportal/agent-python-pytest/issues/375): Fix max Item name length, by @HardNorth + +## [5.4.3] +### Added +- Issue [#332](https://github.com/reportportal/agent-python-pytest/issues/332): Support for fixture reporting, by @HardNorth + +## [5.4.2] +### Fixed +- Issue [#368](https://github.com/reportportal/agent-python-pytest/issues/368): Distutils in the agent, by @HardNorth +- Pytest Tavern plugin support, by @virdok +### Added +- Python 12 support, by @HardNorth + +## [5.4.1] +### Fixed +- Issue [#362](https://github.com/reportportal/agent-python-pytest/issues/362): Debug mode passing, by @HardNorth + +## [5.4.0] +### Added +- Pytest version >= 8 support, by @HardNorth +### Removed +- `Package` and `Dir` item types processing on test collection. This is done to preserve backward compatibility and improve name consistency, by @HardNorth +### Changed +- `rp_issue_system_url` renamed to `rp_bts_issue_url` to be consistent with other agents, by @HardNorth + +## [5.3.2] +### Changed +- Set upper pytest version limit to not include `8.0.0`, by @HardNorth + +## [5.3.1] +### Changed +- Client version updated on [5.5.4](https://github.com/reportportal/client-Python/releases/tag/5.5.4), by @HardNorth +- `rp_launch_id` property handling moved completely on Client side, by @HardNorth + +## [5.3.0] +### Added +- `rp_client_type` configuration variable, by @HardNorth +- `rp_connect_timeout` and `rp_read_timeout` configuration variables, by @HardNorth +### Changed +- Client version updated on [5.5.2](https://github.com/reportportal/client-Python/releases/tag/5.5.2), by @HardNorth + +## [5.2.2] +### Changed +- Client version updated on [5.4.1](https://github.com/reportportal/client-Python/releases/tag/5.4.1), by @HardNorth + +## [5.2.1] +### Fixed + - Log line reference for Python 3.11, by @HardNorth +### Changed +- Client version updated on [5.4.0](https://github.com/reportportal/client-Python/releases/tag/5.4.0), by @HardNorth + +## [5.2.0] +### Added +- `rp_launch_uuid_print` and `rp_launch_uuid_print_output` configuration parameters, by @HardNorth +### Removed +- Python 2.7, 3.6 support, by @HardNorth + +## [5.1.9] +### Added +- `rp_api_retries` configuration parameter, by @HardNorth +### Changed +- Client version updated on [5.3.5](https://github.com/reportportal/client-Python/releases/tag/5.3.5), by @HardNorth +- `rp_uuid` configuration parameter was renamed to `rp_api_key` to maintain common convention, by @HardNorth + +## [5.1.8] +### Fixed +- `rp_thread_logging = False` config parameter handling, by @HardNorth +- Recursive thread init issue in case of `rp_thread_logging = True`, by @HardNorth +### Changed +- Client version updated on [5.3.2](https://github.com/reportportal/client-Python/releases/tag/5.3.2), by @HardNorth + +## [5.1.7] +### Fixed +- Plugin Exception in case of Launch creation timed out, by @HardNorth +### Changed +- Client version updated on [5.3.1](https://github.com/reportportal/client-Python/releases/tag/5.3.1), by @HardNorth + +## [5.1.6] +### Changed +- Client version updated on [5.3.0](https://github.com/reportportal/client-Python/releases/tag/5.3.0), by @HardNorth + +## [5.1.5] +### Added +- Support of runtime issue adding, by @HardNorth + +## [5.1.4] +### Added +- Feature [#325](https://github.com/reportportal/agent-python-pytest/issues/325) Support of runtime attribute adding, by @yakovbabich, @HardNorth +- `rp_launch_timeout` parameter to limit test execution in case of process hanging, by @HardNorth + +## [5.1.3] +### Added +- Support for thread logs and `rp_thread_logging` flag, by @dagansandler + +## [5.1.2] +### Added +- `rp_log_batch_payload_size` parameter, by @HardNorth +### Changed +- Feature [#311](https://github.com/reportportal/agent-python-pytest/issues/311): +Adding log format config option, by @dagansandler + +## [5.1.1] +### Fixed +- Issue [#304](https://github.com/reportportal/agent-python-pytest/issues/304): +SSL certificate flag handling issue, by @HardNorth + +## [5.1.0] +### Changed +- Agent complete rewrite, by @HardNorth diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index a606a7c5..decd681f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,91 +1,146 @@ ============ -Contributing +Contribution ============ -Contributions are welcome, and they are greatly appreciated! Every -little bit helps, and credit will always be given. +Contributions are highly welcomed and appreciated. -You can contribute in many ways: +.. contents:: + :depth: 2 + :backlinks: none -Types of Contributions ----------------------- +Feature requests +---------------- -Report Bugs -~~~~~~~~~~~ +We'd also like to hear about your thoughts and suggestions. Feel free to +`submit them as issues `_ and: -Report bugs at https://github.com/reportportal/agent-python-pytest/issues. +* Explain in detail how they should work. +* Keep the scope as narrow as possible. It will make it easier to implement. -If you are reporting a bug, please include: +Bug reports +----------- + +Report bugs for the agent in the `issue tracker `_. + +If you are reporting a new bug, please include: * Your operating system name and version. -* Any details about your local setup that might be helpful in troubleshooting. +* Python interpreter version, installed libraries, reportportal-client, and agent-python-pytest + version. * Detailed steps to reproduce the bug. -Fix Bugs -~~~~~~~~ +Bug fixes +--------- + +Look through the `GitHub issues for bugs `_. + +If you are gonna fix any of existing bugs, assign that bug to yourself and specify preliminary milestones. +Talk to `contributors `_ in case you need a +consultancy regarding implementation. + +Implement features +------------------ + +Look through the `GitHub issues for enhancements `_. + +Talk to `contributors `_ in case you need a +consultancy regarding implementation. + +Preparing Pull Requests +----------------------- + +What is a "pull request"? It informs the project's core developers about the +changes you want to review and merge. Pull requests are stored on +`GitHub servers `_. +Once you send a pull request, we can discuss its potential modifications and +even add more commits to it later on. There's an excellent tutorial on how Pull +Requests work in the +`GitHub Help Center `_. + +Here is a simple overview below: + +#. Fork the + `agent-python-pytest GitHub repository `_. + +#. Clone your fork locally using `git `_ and create a branch:: + + $ git clone git@github.com:YOUR_GITHUB_USERNAME/agent-python-pytest.git + $ cd agent-python-pytest + # now, create your own branch off the "develop": + + $ git checkout -b your-bugfix-branch-name + + If you need some help with Git, follow this quick start + guide: https://git.wiki.kernel.org/index.php/QuickStart + +#. Install `pre-commit `_ and its hook on the agent-python-pytest repo: + + **Note: pre-commit must be installed as admin, as it will not function otherwise**:: + + + $ pip install --user pre-commit + $ pre-commit install + + Afterwards ``pre-commit`` will run whenever you commit. + + https://pre-commit.com/ is a framework for managing and maintaining multi-language pre-commit hooks + to ensure code-style and code formatting is consistent. -Look through the GitHub issues for bugs. Anything tagged with "bug" -and "help wanted" is open to whoever wants to implement it. +#. Install tox -Implement Features -~~~~~~~~~~~~~~~~~~ + Tox is used to run all the tests and will automatically setup virtualenvs + to run the tests in. + (will implicitly use http://www.virtualenv.org/en/latest/):: -Look through the GitHub issues for features. Anything tagged with "enhancement" -and "help wanted" is open to whoever wants to implement it. + $ pip install tox -Write Documentation -~~~~~~~~~~~~~~~~~~~ +#. Run all the tests -agent-python-pytest could always use more documentation, whether as part of the -official agent-python-pytest docs, in docstrings, or even on the web in blog posts, -articles, and such. + You need to have Python 3.9 available in your system. Now + running tests is as simple as issuing this command:: -Submit Feedback -~~~~~~~~~~~~~~~ + $ tox -e pep,py39 -The best way to send feedback is to file an issue at https://github.com/reportportal/agent-python-pytest/issues. + This command will run tests via the "tox" tool against Python 3.9 + and also perform code style checks. -If you are proposing a feature: +#. You can now edit your local working copy and run the tests again as necessary. Please follow PEP-8 recommendations. -* Explain in detail how it would work. -* Keep the scope as narrow as possible, to make it easier to implement. -* Remember that this is a volunteer-driven project, and that contributions - are welcome :) + You can pass different options to ``tox``. For example, to run tests on Python 3.9 and pass options to pytest + (e.g. enter pdb on failure) to pytest you can do:: -Get Started! ------------- + $ tox -e py39 -- --pdb -Ready to contribute? Here's how to set up `agent-python-pytest` for local development. + Or to only run tests in a particular test module on Python 3.9:: -1. Fork the `agent-python-pytest` repo on GitHub. -2. Clone your fork locally:: + $ tox -e py39 -- tests/test_service.py - $ git clone git@github.com:your_name_here/agent-python-pytest -3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + When committing, ``pre-commit`` will re-format the files if necessary. - $ mkvirtualenv agent_python_pytest - $ cd agent_python_pytest/ - $ python setup.py develop +#. If instead of using ``tox`` you prefer to run the tests directly, then we suggest to create a virtual environment and use + an editable install with the ``testing`` extra:: -4. Create a branch for local development:: + $ python3 -m venv .venv + $ source .venv/bin/activate # Linux + $ .venv/Scripts/activate.bat # Windows + $ pip install -e ".[testing]" - $ git checkout -b name-of-your-bugfix-or-feature + Afterwards, you can edit the files and run pytest normally:: - Now you can make your changes locally. + $ pytest tests/test_service.py -5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: - $ flake8 agent_python_pytest tests - $ tox +#. Commit and push once your tests pass and you are happy with your change(s):: - To get flake8 and tox, just pip install them into your virtualenv. + $ git commit -m "" + $ git push -u -6. Commit your changes and push your branch to GitHub:: - $ git add . - $ git commit -m "Your detailed description of your changes." - $ git push origin name-of-your-bugfix-or-feature +#. Finally, submit a pull request through the GitHub website using this data:: -7. Submit a pull request through the GitHub website. + head-fork: YOUR_GITHUB_USERNAME/agent-python-pytest + compare: your-branch-name + base-fork: reportportal/agent-python-pytest + base: develop diff --git a/LICENSE b/LICENSE index 9cecc1d4..261eeb9e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,674 +1,201 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - {one line to give the program's name and a brief idea of what it does.} - Copyright (C) {year} {name of author} - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - {project} Copyright (C) {year} {fullname} - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in index 026ce585..f7f98afd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ include MANIFEST.in -include README.rst CONTRIBUTING.rst +include README.rst CONTRIBUTING.rst requirements.txt diff --git a/README.md b/README.md new file mode 100644 index 00000000..274488fd --- /dev/null +++ b/README.md @@ -0,0 +1,130 @@ +# ReportPortal integration for pytest framework + +Pytest plugin for reporting test results of the Pytest to the ReportPortal. + +> **DISCLAIMER**: We use Google Analytics for sending anonymous usage information such as agent's and client's names, +> and their versions after a successful launch start. This information might help us to improve both ReportPortal +> backend and client sides. It is used by the ReportPortal team only and is not supposed for sharing with 3rd parties. + +[![PyPI](https://img.shields.io/pypi/v/pytest-reportportal.svg?maxAge=259200)](https://pypi.python.org/pypi/pytest-reportportal) +[![Python versions](https://img.shields.io/pypi/pyversions/pytest-reportportal.svg)](https://pypi.org/project/pytest-reportportal) +[![Tests](https://github.com/reportportal/agent-python-pytest/actions/workflows/tests.yml/badge.svg)](https://github.com/reportportal/agent-python-pytest/actions/workflows/tests.yml) +[![codecov](https://codecov.io/gh/reportportal/agent-python-pytest/graph/badge.svg?token=x5ZHqZKJFV)](https://codecov.io/gh/reportportal/agent-python-pytest) +[![Join Slack chat!](https://img.shields.io/badge/slack-join-brightgreen.svg)](https://slack.epmrpp.reportportal.io/) +[![stackoverflow](https://img.shields.io/badge/reportportal-stackoverflow-orange.svg?style=flat)](http://stackoverflow.com/questions/tagged/reportportal) +[![Build with Love](https://img.shields.io/badge/build%20with-❤%EF%B8%8F%E2%80%8D-lightgrey.svg)](http://reportportal.io?style=flat) + +## Installation + +To install pytest plugin execute next command in a terminal: + +```bash +pip install pytest-reportportal +``` + +Look through the `CONTRIBUTING.rst` for contribution guidelines. + +## Configuration + +Prepare the config file `pytest.ini` in root directory of tests or specify any one using pytest command line option: + +```bash +py.test -c config.cfg +``` + +The `pytest.ini` file should have next mandatory fields: + +- `rp_project` - name of project in ReportPortal +- `rp_endpoint` - address of ReportPortal Server + +And one type of authorization: API Key or OAuth 2.0 Password grant. You can do this by setting: +- `rp_api_key` or `RP_API_KEY` environment variable. You can get it in the User Profile section on the UI. + +Or: +- `rp_oauth_uri` - OAuth 2.0 token endpoint URL for password grant authentication. **Required** if API key is not used. +- `rp_oauth_username` - OAuth 2.0 username for password grant authentication. **Required** if OAuth 2.0 is used. +- `rp_oauth_password` - OAuth 2.0 password for password grant authentication. **Required** if OAuth 2.0 is used. +- `rp_oauth_client_id` - OAuth 2.0 client identifier. **Required** if OAuth 2.0 is used. +- `rp_oauth_client_secret` - OAuth 2.0 client secret. **Optional** for OAuth 2.0 authentication. +- `rp_oauth_scope` - OAuth 2.0 access token scope. **Optional** for OAuth 2.0 authentication. + +Example of `pytest.ini`: + +```text +[pytest] +rp_api_key = fb586627-32be-47dd-93c1-678873458a5f +rp_endpoint = http://192.168.1.10:8080 +rp_project = user_personal +rp_launch = AnyLaunchName +rp_launch_attributes = 'PyTest' 'Smoke' +rp_launch_description = 'Smoke test' +rp_ignore_attributes = 'xfail' 'usefixture' +``` + +There are also optional parameters: +https://reportportal.io/docs/log-data-in-reportportal/test-framework-integration/Python/pytest/ + +## Examples + +For logging of the test item flow to ReportPortal, please, use the python logging handler provided by plugin like +below: + +in `conftest.py`: + +```python +import logging + +import pytest + +from reportportal_client import RPLogger + + +@pytest.fixture(scope="session") +def rp_logger(): + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + logging.setLoggerClass(RPLogger) + return logger +``` + +in tests: + +```python +# In this case only INFO messages will be sent to the ReportPortal. +def test_one(rp_logger): + rp_logger.info("Case1. Step1") + x = "this" + rp_logger.info("x is: %s", x) + assert 'h' in x + + # Message with an attachment. + import subprocess + free_memory = subprocess.check_output("free -h".split()) + rp_logger.info( + "Case1. Memory consumption", + attachment={ + "name": "free_memory.txt", + "data": free_memory, + "mime": "application/octet-stream", + }, + ) + + # This debug message will not be sent to the ReportPortal. + rp_logger.debug("Case1. Debug message") +``` + +## Launching + +To run test with ReportPortal you must provide `--reportportal` flag: + +```bash +py.test ./tests --reportportal +``` + +Check the documentation to find more detailed information about how to integrate pytest with ReportPortal using the +agent: +https://reportportal.io/docs/log-data-in-reportportal/test-framework-integration/Python/pytest/ + +## Copyright Notice + +Licensed under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license (see the LICENSE file). diff --git a/README.rst b/README.rst deleted file mode 100644 index 45637beb..00000000 --- a/README.rst +++ /dev/null @@ -1,174 +0,0 @@ -=================== -agent-python-pytest -=================== - - -**Important:** this is BETA2 version. Please post issue in case if any found - -Pytest plugin for reporting test results of Pytest to the 'Reportal Portal'. - -Usage ------ - -Installation -~~~~~~~~~~~~ - -To install pytest plugin execute next command in a terminal: - -.. code-block:: bash - - pip install pytest-reportportal - - -Configuration -~~~~~~~~~~~~~ - -Prepare the config file :code:`pytest.ini` in root directory of tests or specify -any one using pytest command line option: - -.. code-block:: bash - - py.test -c config.cfg - - -The :code:`pytest.ini` file should have next mandatory fields: - -- :code:`rp_uuid` - value could be found in the User Profile section -- :code:`rp_project` - name of project in Report Potal -- :code:`rp_endpoint` - address of Report Portal Server - -Example of :code:`pytest.ini`: - -.. code-block:: text - - [pytest] - rp_uuid = fb586627-32be-47dd-93c1-678873458a5f - rp_endpoint = http://192.168.1.10:8080 - rp_project = user_personal - rp_launch = AnyLaunchName - rp_launch_tags = 'PyTest' 'Smoke' - rp_launch_description = 'Smoke test' - rp_ignore_errors = True - rp_ignore_tags = 'xfail' 'usefixture' - -The following parameters are optional: - -- :code:`rp_launch = AnyLaunchName` - launch name (could be overridden - by pytest --rp-launch option, default value is 'Pytest Launch') -- :code:`rp_launch_tags = 'PyTest' 'Smoke'` - list of tags -- :code:`rp_launch_description = 'Smoke test'` - launch description -- :code:`rp_log_batch_size = 20` - size of batch log request -- :code:`rp_ignore_errors = True` - Ignore Report Portal errors (exit otherwise) -- :code:`rp_ignore_tags = 'xfail' 'usefixture'` - Ignore specified pytest markers - - -Examples -~~~~~~~~ - -For logging of the test item flow to Report Portal, please, use the python -logging handler provided by plugin like bellow: - -.. code-block:: python - - import logging - # Import Report Portal logger and handler to the test module. - from pytest_reportportal import RPLogger, RPLogHandler - # Setting up a logging. - logging.setLoggerClass(RPLogger) - logger = logging.getLogger(__name__) - logger.setLevel(logging.DEBUG) - # Create handler for Report Portal. - rp_handler = RPLogHandler() - # Set INFO level for Report Portal handler. - rp_handler.setLevel(logging.INFO) - # Add handler to the logger. - logger.addHandler(rp_handler) - - - # In this case only INFO messages will be sent to the Report Portal. - def test_one(): - logger.info("Case1. Step1") - x = "this" - logger.info("x is: %s", x) - assert 'h' in x - - # Message with an attachment. - import subprocess - free_memory = subprocess.check_output("free -h".split()) - logger.info( - "Case1. Memory consumption", - attachment={ - "name": "free_memory.txt", - "data": free_memory, - "mime": "application/octet-stream", - }, - ) - - # This debug message will not be sent to the Report Portal. - logger.debug("Case1. Debug message") - -Plugin can report doc-strings of tests as :code:`descriptions`: - -.. code-block:: python - - def test_one(): - """ - Description of the test case which will be sent to Report Portal - """ - pass - -Pytest markers will be attached as :code:`tags` to Report Portal items. -In the following example tags 'linux' and 'win32' will be used: - -.. code-block:: python - - import pytest - - @pytest.mark.win32 - @pytest.mark.linux - def test_one(): - pass - -If you don't want to attach specific markers, list them in :code:`rp_ignore_tags` parameter - - -Launching -~~~~~~~~~ - -To run test with Report Portal you can specify name of :code:`launch`: - -.. code-block:: bash - - py.test ./tests --rp-launch AnyLaunchName - - -Troubleshooting -~~~~~~~~~ - -In case you have connectivity issues (or similar problems) with Report Portal, -it's possible to ignore exceptions raised by :code:`pytest_reportportal` plugin. -For this, please, add following option to :code:`pytest.ini` configuration file. - -.. code-block:: text - - [pytest] - ... - rp_ignore_errors = True - -With option above all exceptions raised by Report Portal will be printed out to -`stderr` without causing test failures. - -If you would like to temporary disable integrations with Report Portal just -deactivate :code:`pytest_reportportal` plugin with command like: - -.. code-block:: bash - - py.test -p no:pytest_reportportal ./tests - - -Copyright Notice ----------------- - -Licensed under the GPLv3_ license (see the LICENSE file). - -.. _GPLv3: https://www.gnu.org/licenses/quick-guide-gplv3.html diff --git a/examples/attributes/test_one_attribute.py b/examples/attributes/test_one_attribute.py new file mode 100644 index 00000000..78a41e37 --- /dev/null +++ b/examples/attributes/test_one_attribute.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.scope("smoke") +def test_custom_attributes_report(): + """ + This is a test with one custom marker which shall appear on + ReportPortal on test's item + """ + assert True diff --git a/examples/attributes/test_runtime_attribute.py b/examples/attributes/test_runtime_attribute.py new file mode 100644 index 00000000..55fad91a --- /dev/null +++ b/examples/attributes/test_runtime_attribute.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.scope("smoke") +def test_custom_attributes_report(request): + """ + This is a test with one custom marker as a decorator and one custom marker + added at runtime which shall both appear on ReportPortal on test's item + """ + request.node.add_marker(pytest.mark.runtime()) + assert True diff --git a/examples/attributes/test_two_attributes_with_same_key.py b/examples/attributes/test_two_attributes_with_same_key.py new file mode 100644 index 00000000..571afa6f --- /dev/null +++ b/examples/attributes/test_two_attributes_with_same_key.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.scope("smoke") +@pytest.mark.scope("regression") +def test_custom_attributes_report(): + """ + This is a test with multiple custom markers which shall appear on + ReportPortal on test's item as different attributes + """ + assert True diff --git a/examples/bdd/features/arguments_four_steps.feature b/examples/bdd/features/arguments_four_steps.feature new file mode 100644 index 00000000..a71ab7c5 --- /dev/null +++ b/examples/bdd/features/arguments_four_steps.feature @@ -0,0 +1,8 @@ +Feature: Four step arguments + Scenario: Arguments for given, when, and, then + Given there are 5 cucumbers + + When I eat 3 cucumbers + And I eat 2 cucumbers + + Then I should have 0 cucumbers diff --git a/examples/bdd/features/arguments_four_steps_description.feature b/examples/bdd/features/arguments_four_steps_description.feature new file mode 100644 index 00000000..9d4285c5 --- /dev/null +++ b/examples/bdd/features/arguments_four_steps_description.feature @@ -0,0 +1,11 @@ +Feature: Four step arguments + Description for the feature + + Scenario: Arguments for given, when, and, then + Description for the scenario + + Given there are 5 cucumbers + When I eat 3 cucumbers + And I eat 2 cucumbers + + Then I should have 0 cucumbers diff --git a/examples/bdd/features/arguments_four_steps_docstring.feature b/examples/bdd/features/arguments_four_steps_docstring.feature new file mode 100644 index 00000000..de6956d3 --- /dev/null +++ b/examples/bdd/features/arguments_four_steps_docstring.feature @@ -0,0 +1,15 @@ +Feature: Four step arguments + Description for the feature + + Scenario: Arguments for given, when, and, then + Description for the scenario + + Given there are 5 cucumbers + """ + Docstring for the step + """ + + When I eat 3 cucumbers + And I eat 2 cucumbers + + Then I should have 0 cucumbers diff --git a/examples/bdd/features/arguments_three_steps.feature b/examples/bdd/features/arguments_three_steps.feature new file mode 100644 index 00000000..da740d74 --- /dev/null +++ b/examples/bdd/features/arguments_three_steps.feature @@ -0,0 +1,7 @@ +Feature: Three step arguments + Scenario: Arguments for given, when, then, less steps + Given there are 5 cucumbers + + When I eat 5 cucumbers + + Then I should have 0 cucumbers diff --git a/examples/bdd/features/arguments_two_scenarios.feature b/examples/bdd/features/arguments_two_scenarios.feature new file mode 100644 index 00000000..63d02215 --- /dev/null +++ b/examples/bdd/features/arguments_two_scenarios.feature @@ -0,0 +1,15 @@ +Feature: Two scenarios step arguments + Scenario: Arguments for given, when, then + Given there are 5 cucumbers + + When I eat 5 cucumbers + + Then I should have 0 cucumbers + + Scenario: Arguments for given, when, and, then + Given there are 5 cucumbers + + When I eat 3 cucumbers + And I eat 2 cucumbers + + Then I should have 0 cucumbers diff --git a/examples/bdd/features/background_scenario.feature b/examples/bdd/features/background_scenario.feature new file mode 100644 index 00000000..c67e4887 --- /dev/null +++ b/examples/bdd/features/background_scenario.feature @@ -0,0 +1,10 @@ +Feature: Test scenario with a background + + Background: Init our scenario + Given I have empty step + + Scenario: The first scenario + Then I have another empty step + + Scenario: The second scenario + Then I have one more empty step diff --git a/examples/bdd/features/background_two_steps.feature b/examples/bdd/features/background_two_steps.feature new file mode 100644 index 00000000..87075ea8 --- /dev/null +++ b/examples/bdd/features/background_two_steps.feature @@ -0,0 +1,7 @@ +Feature: Test scenario with a background with two steps + Background: + Given I have first empty step + And I have second empty step + + Scenario: The scenario + Then I have main step diff --git a/examples/bdd/features/belly.feature b/examples/bdd/features/belly.feature new file mode 100644 index 00000000..66861838 --- /dev/null +++ b/examples/bdd/features/belly.feature @@ -0,0 +1,8 @@ +@smoke @test @feature:belly +Feature: Belly + + @ok @key:value + Scenario: a few cukes + Given I have 42 cukes in my belly + When I wait 1 hour + Then my belly should growl diff --git a/examples/bdd/features/custom_test_case_id.feature b/examples/bdd/features/custom_test_case_id.feature new file mode 100644 index 00000000..9d6618de --- /dev/null +++ b/examples/bdd/features/custom_test_case_id.feature @@ -0,0 +1,5 @@ +Feature: Test dummy scenario + + @tc_id:my_tc_id + Scenario: The scenario + Given I have empty step diff --git a/examples/bdd/features/data_table_parameter.feature b/examples/bdd/features/data_table_parameter.feature new file mode 100644 index 00000000..3753571b --- /dev/null +++ b/examples/bdd/features/data_table_parameter.feature @@ -0,0 +1,6 @@ +Feature: A basic test with a Data Table parameter + + Scenario: Test with Data Table + Given a step with a data table: + | key | value | + | myKey | myValue | diff --git a/examples/bdd/features/doc_string_parameters.feature b/examples/bdd/features/doc_string_parameters.feature new file mode 100644 index 00000000..32162593 --- /dev/null +++ b/examples/bdd/features/doc_string_parameters.feature @@ -0,0 +1,8 @@ +Feature: Basic test with a docstring parameter + + Scenario: Test with a docstring parameter + Given I have a docstring parameter: + """ + My very long parameter + With some new lines + """ diff --git a/examples/bdd/features/dynamic_scenario_outline_names.feature b/examples/bdd/features/dynamic_scenario_outline_names.feature new file mode 100644 index 00000000..0f3ed07b --- /dev/null +++ b/examples/bdd/features/dynamic_scenario_outline_names.feature @@ -0,0 +1,11 @@ +Feature: Dynamic scenario outline names + + Scenario Outline: Test with the parameter + Given It is test with parameters + When I have parameter + Then I emit number on level info + + Examples: + | str | parameters | + | "first" | 123 | + | "second" | 12345 | diff --git a/examples/bdd/features/examples_tags.feature b/examples/bdd/features/examples_tags.feature new file mode 100644 index 00000000..77b37070 --- /dev/null +++ b/examples/bdd/features/examples_tags.feature @@ -0,0 +1,13 @@ +Feature: Basic test with parameters + + Scenario Outline: Test with different parameters + Given It is test with parameters + When I have parameter + Then I emit number on level info + + @test + Examples: + | str | parameters | + | "first" | 123 | + | "second" | 12345 | + | "third" | 12345678 | diff --git a/examples/bdd/features/failed_scenario.feature b/examples/bdd/features/failed_scenario.feature new file mode 100644 index 00000000..861fb44c --- /dev/null +++ b/examples/bdd/features/failed_scenario.feature @@ -0,0 +1,4 @@ +Feature: Test failed scenario + + Scenario: The scenario + Given I have a failed step diff --git a/examples/bdd/features/long_step_scenario.feature b/examples/bdd/features/long_step_scenario.feature new file mode 100644 index 00000000..8c727854 --- /dev/null +++ b/examples/bdd/features/long_step_scenario.feature @@ -0,0 +1,4 @@ +Feature: Test long step scenario + + Scenario: The scenario + Given A very long step. Seriously, this step is ridiculously long. Our users require such a long step and here it is. This is not even a half of the step, it will last really long. I believe you will get tired before you reach the end of the step. Why? Because our users send here a megabyte-length JSONs or things like that, despite our words that this is not good for user experience, since it's hard to read such a long lines on UI and for backend performance, since we use this field in full-text search. OK I'm out of thought what else I can write here, but the step must go on. Probably you will see some random typing soon to get the step longer than it's right now. Because only now we crossed a half of the length we need. Alright, here we go: 39248cht42 3x,r093mhxr0,3hr c089r3423 xk309,,r3 k302uk032 3249xul398u3 at 34r9k39489 aumvi xkr293jed0 i 93u9f32u smhf 09ktc903 a tu09r328 ef5u0fu3v 0k8utf2 u59du9v u423kuc 9f5kv 39kd3uf39 -3940u5kfu5 b3-90485l-k3f. Are you tired? I'm too, but we still need a few words at the end. diff --git a/examples/bdd/features/rule_description.feature b/examples/bdd/features/rule_description.feature new file mode 100644 index 00000000..5d5ecdb4 --- /dev/null +++ b/examples/bdd/features/rule_description.feature @@ -0,0 +1,7 @@ +Feature: Test rule keyword + + Rule: The first rule + Description for the Rule + + Scenario: The first scenario + Given I have empty step diff --git a/examples/bdd/features/rule_keyword.feature b/examples/bdd/features/rule_keyword.feature new file mode 100644 index 00000000..0b373150 --- /dev/null +++ b/examples/bdd/features/rule_keyword.feature @@ -0,0 +1,15 @@ +Feature: Test rule keyword + + Rule: The first rule + Scenario: The first scenario + Given I have empty step + Then I have another empty step + + Scenario: The second scenario + Given I have empty step + Then I have one more empty step + + Rule: The second rule + Scenario: The third scenario + Given I have empty step + Then I have one more else empty step diff --git a/examples/bdd/features/scenario_outline_background.feature b/examples/bdd/features/scenario_outline_background.feature new file mode 100644 index 00000000..a1dddd3c --- /dev/null +++ b/examples/bdd/features/scenario_outline_background.feature @@ -0,0 +1,14 @@ +Feature: Basic test with parameters and background + + Background: + Given I have empty step in background + + Scenario Outline: Test with different parameters + Given It is test with parameters + When I have parameter + Then I emit number on level info + + Examples: + | str | parameters | + | "first" | 123 | + | "second" | 12345 | diff --git a/examples/bdd/features/scenario_outline_description.feature b/examples/bdd/features/scenario_outline_description.feature new file mode 100644 index 00000000..371ef7cf --- /dev/null +++ b/examples/bdd/features/scenario_outline_description.feature @@ -0,0 +1,13 @@ +Feature: Basic test with parameters and description + + Scenario Outline: Test with different parameters + The description for the scenario outline + + Given It is test with parameters + When I have parameter + Then I emit number on level info + + Examples: + | str | parameters | + | "first" | 123 | + | "second" | 12345 | diff --git a/examples/bdd/features/scenario_outline_fail.feature b/examples/bdd/features/scenario_outline_fail.feature new file mode 100644 index 00000000..049a70c4 --- /dev/null +++ b/examples/bdd/features/scenario_outline_fail.feature @@ -0,0 +1,12 @@ +Feature: Basic test with parameters which fails + + Scenario Outline: Test with different parameters failing + Given It is test with parameters + When I have parameter + Then I emit number on level info + Then I fail + + Examples: + | str | parameters | + | "first" | 123 | + | "second" | 12345 | diff --git a/examples/bdd/features/scenario_outline_parameters.feature b/examples/bdd/features/scenario_outline_parameters.feature new file mode 100644 index 00000000..6913cbe3 --- /dev/null +++ b/examples/bdd/features/scenario_outline_parameters.feature @@ -0,0 +1,12 @@ +Feature: Basic test with parameters + + Scenario Outline: Test with different parameters + Given It is test with parameters + When I have parameter + Then I emit number on level info + + Examples: + | str | parameters | + | "first" | 123 | + | "second" | 12345 | + | "third" | 12345678 | diff --git a/examples/bdd/features/scenario_outline_test_case_id.feature b/examples/bdd/features/scenario_outline_test_case_id.feature new file mode 100644 index 00000000..b14a57e7 --- /dev/null +++ b/examples/bdd/features/scenario_outline_test_case_id.feature @@ -0,0 +1,11 @@ +Feature: Basic test with parameters + + @tc_id:outline_tc_id + Scenario Outline: Test with different parameters + Given It is test with parameters + When I have parameter + Then I emit number on level info + + Examples: + | str | parameters | + | "first" | 123 | diff --git a/examples/bdd/step_defs/__init__.py b/examples/bdd/step_defs/__init__.py new file mode 100644 index 00000000..81753872 --- /dev/null +++ b/examples/bdd/step_defs/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/examples/bdd/step_defs/custom_test_case_id_steps.py b/examples/bdd/step_defs/custom_test_case_id_steps.py new file mode 100644 index 00000000..6528831e --- /dev/null +++ b/examples/bdd/step_defs/custom_test_case_id_steps.py @@ -0,0 +1,27 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, scenarios + +# Import the scenario from the feature file +scenarios("../features/custom_test_case_id.feature") + +LOGGER = logging.getLogger(__name__) + + +@given("I have empty step") +def step_with_custom_test_case_id(): + LOGGER.info("I have empty step") diff --git a/examples/bdd/step_defs/data_table_parameter_steps.py b/examples/bdd/step_defs/data_table_parameter_steps.py new file mode 100644 index 00000000..be4aceb5 --- /dev/null +++ b/examples/bdd/step_defs/data_table_parameter_steps.py @@ -0,0 +1,33 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import Dict + +from pytest_bdd import given, scenarios + +# Import the scenario from the feature file +scenarios("../features/data_table_parameter.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("a step with a data table:") +def step_with_data_table(datatable: Dict[str, str]) -> None: + """Step that receives a data table and logs its content. + + :param datatable: Data table from the feature file + """ + LOGGER.info("Data table content: %s", datatable) diff --git a/examples/bdd/step_defs/doc_string_parameters_steps.py b/examples/bdd/step_defs/doc_string_parameters_steps.py new file mode 100644 index 00000000..a319ab9b --- /dev/null +++ b/examples/bdd/step_defs/doc_string_parameters_steps.py @@ -0,0 +1,32 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, scenarios + +# Import the scenario from the feature file +scenarios("../features/doc_string_parameters.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("I have a docstring parameter:") +def step_with_docstring(docstring: str) -> None: + """Step that receives a docstring and logs its content. + + :param docstring: Multi-line docstring from the feature file + """ + LOGGER.info("Docstring content: %s", docstring) diff --git a/examples/bdd/step_defs/example_tags_steps.py b/examples/bdd/step_defs/example_tags_steps.py new file mode 100644 index 00000000..9eef907e --- /dev/null +++ b/examples/bdd/step_defs/example_tags_steps.py @@ -0,0 +1,38 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, parsers, scenarios, then, when + +# Import the scenario from the feature file +scenarios("../features/examples_tags.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("It is test with parameters") +def step_with_parameters(): + LOGGER.info("It is test with parameters") + + +@when(parsers.parse('I have parameter "{parameter}"')) +def have_parameter_str(parameter: str): + LOGGER.info("String parameter %s", parameter) + + +@then(parsers.parse("I emit number {parameters:d} on level info")) +def emit_number_info(parameters): + LOGGER.info("Test with parameters: %d", parameters) diff --git a/examples/bdd/step_defs/long_step_scenario_steps.py b/examples/bdd/step_defs/long_step_scenario_steps.py new file mode 100644 index 00000000..32846a76 --- /dev/null +++ b/examples/bdd/step_defs/long_step_scenario_steps.py @@ -0,0 +1,30 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, scenarios + +# Import the scenario from the feature file +scenarios("../features/long_step_scenario.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given( + "A very long step. Seriously, this step is ridiculously long. Our users require such a long step and here it is. This is not even a half of the step, it will last really long. I believe you will get tired before you reach the end of the step. Why? Because our users send here a megabyte-length JSONs or things like that, despite our words that this is not good for user experience, since it's hard to read such a long lines on UI and for backend performance, since we use this field in full-text search. OK I'm out of thought what else I can write here, but the step must go on. Probably you will see some random typing soon to get the step longer than it's right now. Because only now we crossed a half of the length we need. Alright, here we go: 39248cht42 3x,r093mhxr0,3hr c089r3423 xk309,,r3 k302uk032 3249xul398u3 at 34r9k39489 aumvi xkr293jed0 i 93u9f32u smhf 09ktc903 a tu09r328 ef5u0fu3v 0k8utf2 u59du9v u423kuc 9f5kv 39kd3uf39 -3940u5kfu5 b3-90485l-k3f. Are you tired? I'm too, but we still need a few words at the end." # noqa: E501 +) +def step_with_long_text(): + LOGGER.info("Step with extremely long text executed successfully") diff --git a/examples/bdd/step_defs/scenario_outline_background_steps.py b/examples/bdd/step_defs/scenario_outline_background_steps.py new file mode 100644 index 00000000..ee75256c --- /dev/null +++ b/examples/bdd/step_defs/scenario_outline_background_steps.py @@ -0,0 +1,44 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, parsers, scenarios, then, when + +# Import the scenario from the feature file +scenarios("../features/scenario_outline_background.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("I have empty step in background") +def empty_step(): + """Empty step implementation.""" + pass + + +@given("It is test with parameters") +def step_with_parameters(): + LOGGER.info("It is test with parameters") + + +@when(parsers.parse('I have parameter "{parameter}"')) +def have_parameter_str(parameter: str): + LOGGER.info("String parameter %s", parameter) + + +@then(parsers.parse("I emit number {parameters:d} on level info")) +def emit_number_info(parameters): + LOGGER.info("Test with parameters: %d", parameters) diff --git a/examples/bdd/step_defs/scenario_outline_description_steps.py b/examples/bdd/step_defs/scenario_outline_description_steps.py new file mode 100644 index 00000000..4ff17655 --- /dev/null +++ b/examples/bdd/step_defs/scenario_outline_description_steps.py @@ -0,0 +1,38 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, parsers, scenarios, then, when + +# Import the scenario from the feature file +scenarios("../features/scenario_outline_description.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("It is test with parameters") +def step_with_parameters(): + LOGGER.info("It is test with parameters") + + +@when(parsers.parse('I have parameter "{parameter}"')) +def have_parameter_str(parameter: str): + LOGGER.info("String parameter %s", parameter) + + +@then(parsers.parse("I emit number {parameters:d} on level info")) +def emit_number_info(parameters): + LOGGER.info("Test with parameters: %d", parameters) diff --git a/examples/bdd/step_defs/scenario_outline_fail_steps.py b/examples/bdd/step_defs/scenario_outline_fail_steps.py new file mode 100644 index 00000000..ebca0a39 --- /dev/null +++ b/examples/bdd/step_defs/scenario_outline_fail_steps.py @@ -0,0 +1,43 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, parsers, scenarios, then, when + +# Import the scenario from the feature file +scenarios("../features/scenario_outline_fail.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("It is test with parameters") +def step_with_parameters(): + LOGGER.info("It is test with parameters") + + +@when(parsers.parse('I have parameter "{parameter}"')) +def have_parameter_str(parameter: str): + LOGGER.info("String parameter %s", parameter) + + +@then(parsers.parse("I emit number {parameters:d} on level info")) +def emit_number_info(parameters): + LOGGER.info("Test with parameters: %d", parameters) + + +@then("I fail") +def fail_step(): + raise AssertionError("This step always fails") diff --git a/examples/bdd/step_defs/scenario_outline_name_steps.py b/examples/bdd/step_defs/scenario_outline_name_steps.py new file mode 100644 index 00000000..75b6d690 --- /dev/null +++ b/examples/bdd/step_defs/scenario_outline_name_steps.py @@ -0,0 +1,38 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, parsers, scenarios, then, when + +# Import the scenario from the feature file +scenarios("../features/dynamic_scenario_outline_names.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("It is test with parameters") +def step_with_parameters(): + LOGGER.info("It is test with parameters") + + +@when(parsers.parse('I have parameter "{parameter}"')) +def have_parameter_str(parameter: str): + LOGGER.info("String parameter %s", parameter) + + +@then(parsers.parse("I emit number {parameters:d} on level info")) +def emit_number_info(parameters): + LOGGER.info("Test with parameters: %d", parameters) diff --git a/examples/bdd/step_defs/scenario_outline_parameters_steps.py b/examples/bdd/step_defs/scenario_outline_parameters_steps.py new file mode 100644 index 00000000..ece05e13 --- /dev/null +++ b/examples/bdd/step_defs/scenario_outline_parameters_steps.py @@ -0,0 +1,38 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, parsers, scenarios, then, when + +# Import the scenario from the feature file +scenarios("../features/scenario_outline_parameters.feature") + + +LOGGER = logging.getLogger(__name__) + + +@given("It is test with parameters") +def step_with_parameters(): + LOGGER.info("It is test with parameters") + + +@when(parsers.parse('I have parameter "{parameter}"')) +def have_parameter_str(parameter: str): + LOGGER.info("String parameter %s", parameter) + + +@then(parsers.parse("I emit number {parameters:d} on level info")) +def emit_number_info(parameters): + LOGGER.info("Test with parameters: %d", parameters) diff --git a/examples/bdd/step_defs/scenario_outline_test_case_id_steps.py b/examples/bdd/step_defs/scenario_outline_test_case_id_steps.py new file mode 100644 index 00000000..d5174aa7 --- /dev/null +++ b/examples/bdd/step_defs/scenario_outline_test_case_id_steps.py @@ -0,0 +1,37 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from pytest_bdd import given, parsers, scenarios, then, when + +# Import the scenario from the feature file +scenarios("../features/scenario_outline_test_case_id.feature") + +LOGGER = logging.getLogger(__name__) + + +@given("It is test with parameters") +def step_with_parameters(): + LOGGER.info("It is test with parameters") + + +@when(parsers.parse('I have parameter "{parameter}"')) +def have_parameter_str(parameter: str): + LOGGER.info("String parameter %s", parameter) + + +@then(parsers.parse("I emit number {parameters:d} on level info")) +def emit_number_info(parameters): + LOGGER.info("Test with parameters: %d", parameters) diff --git a/examples/bdd/step_defs/test_arguments.py b/examples/bdd/step_defs/test_arguments.py new file mode 100644 index 00000000..97e27fbb --- /dev/null +++ b/examples/bdd/step_defs/test_arguments.py @@ -0,0 +1,32 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, parsers, scenarios, then, when + +scenarios("../features/arguments_four_steps.feature") + + +@given(parsers.parse("there are {start:d} cucumbers"), target_fixture="cucumbers") +def given_cucumbers(start): + return {"start": start, "eat": 0} + + +@when(parsers.parse("I eat {eat:d} cucumbers")) +def eat_cucumbers(cucumbers, eat): + cucumbers["eat"] += eat + + +@then(parsers.parse("I should have {left:d} cucumbers")) +def should_have_left_cucumbers(cucumbers, left): + assert cucumbers["start"] - cucumbers["eat"] == left diff --git a/examples/bdd/step_defs/test_arguments_description.py b/examples/bdd/step_defs/test_arguments_description.py new file mode 100644 index 00000000..d1301b7f --- /dev/null +++ b/examples/bdd/step_defs/test_arguments_description.py @@ -0,0 +1,32 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, parsers, scenarios, then, when + +scenarios("../features/arguments_four_steps_description.feature") + + +@given(parsers.parse("there are {start:d} cucumbers"), target_fixture="cucumbers") +def given_cucumbers(start): + return {"start": start, "eat": 0} + + +@when(parsers.parse("I eat {eat:d} cucumbers")) +def eat_cucumbers(cucumbers, eat): + cucumbers["eat"] += eat + + +@then(parsers.parse("I should have {left:d} cucumbers")) +def should_have_left_cucumbers(cucumbers, left): + assert cucumbers["start"] - cucumbers["eat"] == left diff --git a/examples/bdd/step_defs/test_arguments_two_features.py b/examples/bdd/step_defs/test_arguments_two_features.py new file mode 100644 index 00000000..0f268c55 --- /dev/null +++ b/examples/bdd/step_defs/test_arguments_two_features.py @@ -0,0 +1,32 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, parsers, scenarios, then, when + +scenarios("../features/arguments_four_steps.feature", "../features/arguments_three_steps.feature") + + +@given(parsers.parse("there are {start:d} cucumbers"), target_fixture="cucumbers") +def given_cucumbers(start): + return {"start": start, "eat": 0} + + +@when(parsers.parse("I eat {eat:d} cucumbers")) +def eat_cucumbers(cucumbers, eat): + cucumbers["eat"] += eat + + +@then(parsers.parse("I should have {left:d} cucumbers")) +def should_have_left_cucumbers(cucumbers, left): + assert cucumbers["start"] - cucumbers["eat"] == left diff --git a/examples/bdd/step_defs/test_arguments_two_scenarios.py b/examples/bdd/step_defs/test_arguments_two_scenarios.py new file mode 100644 index 00000000..7e7db797 --- /dev/null +++ b/examples/bdd/step_defs/test_arguments_two_scenarios.py @@ -0,0 +1,32 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, parsers, scenarios, then, when + +scenarios("../features/arguments_two_scenarios.feature") + + +@given(parsers.parse("there are {start:d} cucumbers"), target_fixture="cucumbers") +def given_cucumbers(start): + return {"start": start, "eat": 0} + + +@when(parsers.parse("I eat {eat:d} cucumbers")) +def eat_cucumbers(cucumbers, eat): + cucumbers["eat"] += eat + + +@then(parsers.parse("I should have {left:d} cucumbers")) +def should_have_left_cucumbers(cucumbers, left): + assert cucumbers["start"] - cucumbers["eat"] == left diff --git a/examples/bdd/step_defs/test_background.py b/examples/bdd/step_defs/test_background.py new file mode 100644 index 00000000..fff783df --- /dev/null +++ b/examples/bdd/step_defs/test_background.py @@ -0,0 +1,32 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, scenarios, then + +scenarios("../features/background_scenario.feature") + + +@given("I have empty step") +def empty_step(): + pass + + +@then("I have another empty step") +def another_empty_step(): + pass + + +@then("I have one more empty step") +def one_more_empty_step(): + pass diff --git a/examples/bdd/step_defs/test_background_two_steps.py b/examples/bdd/step_defs/test_background_two_steps.py new file mode 100644 index 00000000..ad81b760 --- /dev/null +++ b/examples/bdd/step_defs/test_background_two_steps.py @@ -0,0 +1,35 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, scenarios, then + +scenarios("../features/background_two_steps.feature") + + +@given("I have first empty step") +def first_empty_step(): + """First empty step implementation.""" + pass + + +@given("I have second empty step") +def second_empty_step(): + """Second empty step implementation.""" + pass + + +@then("I have main step") +def main_step(): + """Main step implementation.""" + pass diff --git a/examples/bdd/step_defs/test_belly.py b/examples/bdd/step_defs/test_belly.py new file mode 100644 index 00000000..5afcd972 --- /dev/null +++ b/examples/bdd/step_defs/test_belly.py @@ -0,0 +1,32 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, parsers, scenarios, then, when + +scenarios("../features/belly.feature") + + +@given(parsers.parse("I have {start:d} cukes in my belly"), target_fixture="cucumbers") +def given_cucumbers(start): + return {"start": start, "wait": 0} + + +@when(parsers.parse("I wait {hours:d} hour")) +def then_wait(cucumbers, hours): + cucumbers["wait"] += hours + + +@then("my belly should growl") +def assert_growl(cucumbers): + assert cucumbers["start"] == cucumbers["wait"] * 42 diff --git a/examples/bdd/step_defs/test_failed_step.py b/examples/bdd/step_defs/test_failed_step.py new file mode 100644 index 00000000..e49b854b --- /dev/null +++ b/examples/bdd/step_defs/test_failed_step.py @@ -0,0 +1,22 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytest_bdd import given, scenarios + +scenarios("../features/failed_scenario.feature") + + +@given("I have a failed step") +def failed_step(): + assert False diff --git a/examples/bdd/step_defs/test_rule_description_steps.py b/examples/bdd/step_defs/test_rule_description_steps.py new file mode 100644 index 00000000..60beafa8 --- /dev/null +++ b/examples/bdd/step_defs/test_rule_description_steps.py @@ -0,0 +1,25 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rule keyword test module.""" + +from pytest_bdd import given, scenarios + +scenarios("../features/rule_description.feature") + + +@given("I have empty step") +def empty_step(): + """Empty step implementation.""" + pass diff --git a/examples/bdd/step_defs/test_rule_steps.py b/examples/bdd/step_defs/test_rule_steps.py new file mode 100644 index 00000000..e981126f --- /dev/null +++ b/examples/bdd/step_defs/test_rule_steps.py @@ -0,0 +1,42 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rule keyword test module.""" +from pytest_bdd import given, scenarios, then + +scenarios("../features/rule_keyword.feature") + + +@given("I have empty step") +def empty_step(): + """Empty step implementation.""" + pass + + +@then("I have another empty step") +def another_empty_step(): + """Another empty step implementation.""" + pass + + +@then("I have one more empty step") +def one_more_empty_step(): + """One more empty step implementation.""" + pass + + +@then("I have one more else empty step") +def one_more_else_empty_step(): + """One more else empty step implementation.""" + pass diff --git a/examples/custom_name/test_custom_name_args.py b/examples/custom_name/test_custom_name_args.py new file mode 100644 index 00000000..a20c3c6d --- /dev/null +++ b/examples/custom_name/test_custom_name_args.py @@ -0,0 +1,22 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +TEST_NAME_ARGS = "Test name by mark" + + +@pytest.mark.name(TEST_NAME_ARGS) +def test_name_by_mark_args(): + """Simple example test with the name comes from Pytest mark.""" + assert True diff --git a/examples/custom_name/test_custom_name_empty.py b/examples/custom_name/test_custom_name_empty.py new file mode 100644 index 00000000..57ac5263 --- /dev/null +++ b/examples/custom_name/test_custom_name_empty.py @@ -0,0 +1,22 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +TEST_NAME_EMPTY = "examples/custom_name/test_custom_name_empty.py::test_name_by_mark_empty" + + +@pytest.mark.name() +def test_name_by_mark_empty(): + """Simple example test with the name comes from Pytest mark.""" + assert True diff --git a/examples/custom_name/test_custom_name_kwargs.py b/examples/custom_name/test_custom_name_kwargs.py new file mode 100644 index 00000000..997fdb9d --- /dev/null +++ b/examples/custom_name/test_custom_name_kwargs.py @@ -0,0 +1,22 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +TEST_NAME_KWARGS = "Test name by mark, kwargs" + + +@pytest.mark.name(name=TEST_NAME_KWARGS) +def test_name_by_mark_kwargs(): + """Simple example test with the name comes from Pytest mark.""" + assert True diff --git a/examples/empty/README.txt b/examples/empty/README.txt new file mode 100644 index 00000000..802a5b81 --- /dev/null +++ b/examples/empty/README.txt @@ -0,0 +1 @@ +A placeholder file for git diff --git a/examples/fixtures/class_fixture_return/conftest.py b/examples/fixtures/class_fixture_return/conftest.py new file mode 100644 index 00000000..33e6d2bb --- /dev/null +++ b/examples/fixtures/class_fixture_return/conftest.py @@ -0,0 +1,23 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + + +@pytest.fixture(scope="class") +def class_fixture_return_config(): + print("setup") + return mock.Mock() diff --git a/examples/fixtures/class_fixture_return/test_fixture_class_setup.py b/examples/fixtures/class_fixture_return/test_fixture_class_setup.py new file mode 100644 index 00000000..59009574 --- /dev/null +++ b/examples/fixtures/class_fixture_return/test_fixture_class_setup.py @@ -0,0 +1,41 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class TestClassOne: + def test_fixture_class_setup_first(self, class_fixture_return_config): + assert class_fixture_return_config is not None + + def test_fixture_class_setup_second(self, class_fixture_return_config): + assert class_fixture_return_config is not None + + +class TestClassTwo: + def test_fixture_class_setup_forth(self, class_fixture_return_config): + assert class_fixture_return_config is not None + + def test_fixture_class_setup_fifth(self, class_fixture_return_config): + assert class_fixture_return_config is not None diff --git a/examples/fixtures/module_fixture_return/conftest.py b/examples/fixtures/module_fixture_return/conftest.py new file mode 100644 index 00000000..8d72be37 --- /dev/null +++ b/examples/fixtures/module_fixture_return/conftest.py @@ -0,0 +1,23 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + + +@pytest.fixture(scope="module") +def module_fixture_return_config(): + print("setup") + return mock.Mock() diff --git a/examples/fixtures/module_fixture_return/test_fixture_module_setup.py b/examples/fixtures/module_fixture_return/test_fixture_module_setup.py new file mode 100644 index 00000000..cfac0db7 --- /dev/null +++ b/examples/fixtures/module_fixture_return/test_fixture_module_setup.py @@ -0,0 +1,33 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_module_setup_first(module_fixture_return_config): + assert module_fixture_return_config is not None + + +def test_fixture_module_setup_second(module_fixture_return_config): + assert module_fixture_return_config is not None diff --git a/examples/fixtures/package_fixture_return/__init__.py b/examples/fixtures/package_fixture_return/__init__.py new file mode 100644 index 00000000..2c4530c5 --- /dev/null +++ b/examples/fixtures/package_fixture_return/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/examples/fixtures/package_fixture_return/conftest.py b/examples/fixtures/package_fixture_return/conftest.py new file mode 100644 index 00000000..a9bb4367 --- /dev/null +++ b/examples/fixtures/package_fixture_return/conftest.py @@ -0,0 +1,23 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + + +@pytest.fixture(scope="package") +def package_fixture_return_config(): + print("setup") + return mock.Mock() diff --git a/examples/fixtures/package_fixture_return/test_fixture_package_setup_first.py b/examples/fixtures/package_fixture_return/test_fixture_package_setup_first.py new file mode 100644 index 00000000..10373540 --- /dev/null +++ b/examples/fixtures/package_fixture_return/test_fixture_package_setup_first.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_package_setup_first(package_fixture_return_config): + assert package_fixture_return_config is not None diff --git a/examples/fixtures/package_fixture_return/test_fixture_package_setup_second.py b/examples/fixtures/package_fixture_return/test_fixture_package_setup_second.py new file mode 100644 index 00000000..f66e7388 --- /dev/null +++ b/examples/fixtures/package_fixture_return/test_fixture_package_setup_second.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_package_setup_second(package_fixture_return_config): + assert package_fixture_return_config is not None diff --git a/examples/fixtures/session_fixture_return/conftest.py b/examples/fixtures/session_fixture_return/conftest.py new file mode 100644 index 00000000..fd08838f --- /dev/null +++ b/examples/fixtures/session_fixture_return/conftest.py @@ -0,0 +1,23 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + + +@pytest.fixture(scope="session") +def session_fixture_return_config(): + print("setup") + return mock.Mock() diff --git a/examples/fixtures/session_fixture_return/test_fixture_session_setup_first.py b/examples/fixtures/session_fixture_return/test_fixture_session_setup_first.py new file mode 100644 index 00000000..3480e949 --- /dev/null +++ b/examples/fixtures/session_fixture_return/test_fixture_session_setup_first.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_session_setup_first(session_fixture_return_config): + assert session_fixture_return_config is not None diff --git a/examples/fixtures/session_fixture_return/test_fixture_session_setup_second.py b/examples/fixtures/session_fixture_return/test_fixture_session_setup_second.py new file mode 100644 index 00000000..9640d580 --- /dev/null +++ b/examples/fixtures/session_fixture_return/test_fixture_session_setup_second.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_session_setup_second(session_fixture_return_config): + assert session_fixture_return_config is not None diff --git a/examples/fixtures/test_failure_fixture_teardown/conftest.py b/examples/fixtures/test_failure_fixture_teardown/conftest.py new file mode 100644 index 00000000..e3df9875 --- /dev/null +++ b/examples/fixtures/test_failure_fixture_teardown/conftest.py @@ -0,0 +1,33 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from unittest import mock + +import pytest +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_BEFORE_YIELD = "Log message before yield and test failure" +LOG_MESSAGE_TEARDOWN = "Log message for teardown after test failure" + + +@pytest.fixture +def test_failure_fixture_teardown_config(): + logging.error(LOG_MESSAGE_BEFORE_YIELD) + yield mock.Mock() + logging.error(LOG_MESSAGE_TEARDOWN) diff --git a/examples/fixtures/test_failure_fixture_teardown/test_failure_fixture_teardown.py b/examples/fixtures/test_failure_fixture_teardown/test_failure_fixture_teardown.py new file mode 100644 index 00000000..390feaae --- /dev/null +++ b/examples/fixtures/test_failure_fixture_teardown/test_failure_fixture_teardown.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_failure_fixture_teardown(test_failure_fixture_teardown_config): + assert test_failure_fixture_teardown_config is None diff --git a/examples/fixtures/test_fixture_exit/test_fixture_exit.py b/examples/fixtures/test_fixture_exit/test_fixture_exit.py new file mode 100644 index 00000000..a7a02083 --- /dev/null +++ b/examples/fixtures/test_fixture_exit/test_fixture_exit.py @@ -0,0 +1,24 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture +def fixture_demo(): + pytest.exit("Some Message") + + +def test_exit(fixture_demo): + assert True diff --git a/examples/fixtures/test_fixture_return_none/conftest.py b/examples/fixtures/test_fixture_return_none/conftest.py new file mode 100644 index 00000000..a3209120 --- /dev/null +++ b/examples/fixtures/test_fixture_return_none/conftest.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import pytest +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_SETUP = "Log message for setup and return None" + + +@pytest.fixture +def test_fixture_return_none_config(): + LOGGER.warning(LOG_MESSAGE_SETUP) diff --git a/examples/fixtures/test_fixture_return_none/test_fixture_return_none.py b/examples/fixtures/test_fixture_return_none/test_fixture_return_none.py new file mode 100644 index 00000000..e0003762 --- /dev/null +++ b/examples/fixtures/test_fixture_return_none/test_fixture_return_none.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_setup_none(test_fixture_return_none_config): + assert test_fixture_return_none_config is None diff --git a/examples/fixtures/test_fixture_setup/conftest.py b/examples/fixtures/test_fixture_setup/conftest.py new file mode 100644 index 00000000..7d74f8e9 --- /dev/null +++ b/examples/fixtures/test_fixture_setup/conftest.py @@ -0,0 +1,31 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from unittest import mock + +import pytest +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_SETUP = "Log message for setup" + + +@pytest.fixture +def test_fixture_setup_config(): + logging.error(LOG_MESSAGE_SETUP) + return mock.Mock() diff --git a/examples/fixtures/test_fixture_setup/test_fixture_setup.py b/examples/fixtures/test_fixture_setup/test_fixture_setup.py new file mode 100644 index 00000000..cc2f8566 --- /dev/null +++ b/examples/fixtures/test_fixture_setup/test_fixture_setup.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_setup(test_fixture_setup_config): + assert test_fixture_setup_config is not None diff --git a/examples/fixtures/test_fixture_setup_failure/conftest.py b/examples/fixtures/test_fixture_setup_failure/conftest.py new file mode 100644 index 00000000..5995d547 --- /dev/null +++ b/examples/fixtures/test_fixture_setup_failure/conftest.py @@ -0,0 +1,30 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import pytest +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_SETUP = "Log message for setup failure" + + +@pytest.fixture +def test_fixture_setup_failure_config(): + logging.error(LOG_MESSAGE_SETUP) + raise Exception("Fixture setup failed") diff --git a/examples/fixtures/test_fixture_setup_failure/test_fixture_setup_failure.py b/examples/fixtures/test_fixture_setup_failure/test_fixture_setup_failure.py new file mode 100644 index 00000000..db4fe95b --- /dev/null +++ b/examples/fixtures/test_fixture_setup_failure/test_fixture_setup_failure.py @@ -0,0 +1,40 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_TEST = "Log message for test of setup failure" + + +def test_fixture_setup_failure(test_fixture_setup_failure_config): + logging.error(LOG_MESSAGE_TEST) + assert test_fixture_setup_failure_config is not None diff --git a/examples/fixtures/test_fixture_skipped/test_fixture_skipped.py b/examples/fixtures/test_fixture_skipped/test_fixture_skipped.py new file mode 100644 index 00000000..cabbd15c --- /dev/null +++ b/examples/fixtures/test_fixture_skipped/test_fixture_skipped.py @@ -0,0 +1,30 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + + +@pytest.fixture(scope="session") +def base_fixture(): + return False + + +@pytest.fixture() +def skip_fixture(base_fixture): + if not base_fixture: + pytest.skip("Skip if base condition is false") + + +def test_will_skip(skip_fixture): + pass diff --git a/examples/fixtures/test_fixture_teardown/conftest.py b/examples/fixtures/test_fixture_teardown/conftest.py new file mode 100644 index 00000000..10d3d6a9 --- /dev/null +++ b/examples/fixtures/test_fixture_teardown/conftest.py @@ -0,0 +1,33 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from unittest import mock + +import pytest +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_BEFORE_YIELD = "Log message before yield" +LOG_MESSAGE_TEARDOWN = "Log message for teardown" + + +@pytest.fixture +def test_fixture_teardown_config(): + logging.error(LOG_MESSAGE_BEFORE_YIELD) + yield mock.Mock() + logging.error(LOG_MESSAGE_TEARDOWN) diff --git a/examples/fixtures/test_fixture_teardown/test_fixture_teardown.py b/examples/fixtures/test_fixture_teardown/test_fixture_teardown.py new file mode 100644 index 00000000..aef60447 --- /dev/null +++ b/examples/fixtures/test_fixture_teardown/test_fixture_teardown.py @@ -0,0 +1,32 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + + +def test_fixture_teardown(test_fixture_teardown_config): + sleep(0.001) + assert test_fixture_teardown_config is not None diff --git a/examples/fixtures/test_fixture_teardown_failure/conftest.py b/examples/fixtures/test_fixture_teardown_failure/conftest.py new file mode 100644 index 00000000..17c5e21b --- /dev/null +++ b/examples/fixtures/test_fixture_teardown_failure/conftest.py @@ -0,0 +1,34 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from unittest import mock + +import pytest +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_BEFORE_YIELD = "Log message before yield and failure" +LOG_MESSAGE_TEARDOWN = "Log message for failure teardown" + + +@pytest.fixture +def test_fixture_teardown_failure_config(): + logging.error(LOG_MESSAGE_BEFORE_YIELD) + yield mock.Mock() + logging.error(LOG_MESSAGE_TEARDOWN) + raise Exception("Fixture teardown failed") diff --git a/examples/fixtures/test_fixture_teardown_failure/test_fixture_teardown_failure.py b/examples/fixtures/test_fixture_teardown_failure/test_fixture_teardown_failure.py new file mode 100644 index 00000000..1a7e257e --- /dev/null +++ b/examples/fixtures/test_fixture_teardown_failure/test_fixture_teardown_failure.py @@ -0,0 +1,32 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + + +def test_fixture_teardown_failure(test_fixture_teardown_failure_config): + sleep(0.001) + assert test_fixture_teardown_failure_config is not None diff --git a/examples/fixtures/test_fixture_yield_none/conftest.py b/examples/fixtures/test_fixture_yield_none/conftest.py new file mode 100644 index 00000000..e35b52a1 --- /dev/null +++ b/examples/fixtures/test_fixture_yield_none/conftest.py @@ -0,0 +1,30 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import pytest +from reportportal_client import RPLogger + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) +logging.setLoggerClass(RPLogger) + +LOG_MESSAGE_SETUP = "Log message for setup and yield None" + + +@pytest.fixture +def test_fixture_yield_none_config(): + LOGGER.warning(LOG_MESSAGE_SETUP) + yield None diff --git a/examples/fixtures/test_fixture_yield_none/test_fixture_yield_none.py b/examples/fixtures/test_fixture_yield_none/test_fixture_yield_none.py new file mode 100644 index 00000000..cf73de1c --- /dev/null +++ b/examples/fixtures/test_fixture_yield_none/test_fixture_yield_none.py @@ -0,0 +1,29 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_fixture_yield_none(test_fixture_yield_none_config): + assert test_fixture_yield_none_config is None diff --git a/examples/hierarchy/another_inner/test_another_inner_simple.py b/examples/hierarchy/another_inner/test_another_inner_simple.py new file mode 100644 index 00000000..eab9eacc --- /dev/null +++ b/examples/hierarchy/another_inner/test_another_inner_simple.py @@ -0,0 +1,6 @@ +"""Simple example test.""" + + +def test_simple(): + """Simple example test.""" + assert True diff --git a/examples/hierarchy/inner/test_inner_simple.py b/examples/hierarchy/inner/test_inner_simple.py new file mode 100644 index 00000000..eab9eacc --- /dev/null +++ b/examples/hierarchy/inner/test_inner_simple.py @@ -0,0 +1,6 @@ +"""Simple example test.""" + + +def test_simple(): + """Simple example test.""" + assert True diff --git a/examples/hierarchy/test_in_class.py b/examples/hierarchy/test_in_class.py new file mode 100644 index 00000000..2d6cb1c0 --- /dev/null +++ b/examples/hierarchy/test_in_class.py @@ -0,0 +1,9 @@ +"""A simple example test in a class.""" + + +class Tests: + def test_in_class(self): + """ + This is my test inside `Tests` class + """ + assert True diff --git a/examples/hierarchy/test_in_class_in_class.py b/examples/hierarchy/test_in_class_in_class.py new file mode 100644 index 00000000..36a50619 --- /dev/null +++ b/examples/hierarchy/test_in_class_in_class.py @@ -0,0 +1,10 @@ +"""A simple example test in a class in a class.""" + + +class Tests: + class Test: + def test_in_class_in_class(self): + """ + This is my test inside `Test` class + """ + assert True diff --git a/examples/params/test_binary_symbol_in_parameters.py b/examples/params/test_binary_symbol_in_parameters.py new file mode 100644 index 00000000..241a1f69 --- /dev/null +++ b/examples/params/test_binary_symbol_in_parameters.py @@ -0,0 +1,14 @@ +"""A simple example test with different parameter types.""" + +import pytest + +BINARY_TEXT = "Some text with binary symbol \0" + + +@pytest.mark.parametrize(["text"], [[BINARY_TEXT]]) +def test_in_class_parameterized(text): + """ + This is my test with different parameter types. + """ + assert text == BINARY_TEXT + assert text != BINARY_TEXT.replace("\0", "\\0") diff --git a/examples/params/test_different_parameter_types.py b/examples/params/test_different_parameter_types.py new file mode 100644 index 00000000..d219884a --- /dev/null +++ b/examples/params/test_different_parameter_types.py @@ -0,0 +1,11 @@ +"""A simple example test with different parameter types.""" + +import pytest + + +@pytest.mark.parametrize(["integer", "floating_point", "boolean", "none"], [(1, 1.5, True, None)]) +def test_in_class_parameterized(integer, floating_point, boolean, none): + """ + This is my test with different parameter types. + """ + assert True diff --git a/examples/params/test_in_class_parameterized.py b/examples/params/test_in_class_parameterized.py new file mode 100644 index 00000000..8c3403c9 --- /dev/null +++ b/examples/params/test_in_class_parameterized.py @@ -0,0 +1,13 @@ +"""A simple example test in a class with a parameter.""" + +import pytest + + +class Tests: + + @pytest.mark.parametrize("param", ["param"]) + def test_in_class_parameterized(self, param): + """ + This is my test inside `Tests` class with a parameter + """ + assert True diff --git a/examples/skip/test_simple_skip.py b/examples/skip/test_simple_skip.py new file mode 100644 index 00000000..69ce838e --- /dev/null +++ b/examples/skip/test_simple_skip.py @@ -0,0 +1,20 @@ +"""Simple example skipped test.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +import pytest + + +@pytest.mark.skip(reason="no way of currently testing this") +def test_simple_skip(): + assert False diff --git a/examples/skip/test_skip_issue.py b/examples/skip/test_skip_issue.py new file mode 100644 index 00000000..3674616f --- /dev/null +++ b/examples/skip/test_skip_issue.py @@ -0,0 +1,25 @@ +"""Simple example skipped test.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +import pytest + +ID = "ABC-1234" +REASON = "some_bug" +TYPE = "PB" + + +@pytest.mark.issue(issue_id=ID, reason=REASON, issue_type=TYPE) +@pytest.mark.skip(reason="no way of currently testing this") +def test_simple_skip(): + assert False diff --git a/examples/test_case_id/test_case_id_decorator.py b/examples/test_case_id/test_case_id_decorator.py new file mode 100644 index 00000000..9bbda942 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator.py @@ -0,0 +1,23 @@ +"""A simple example test with Test Case ID decorator.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-123" + + +@pytest.mark.tc_id(TEST_CASE_ID) +def test_case_id_decorator(): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_no_id.py b/examples/test_case_id/test_case_id_decorator_no_id.py new file mode 100644 index 00000000..5d59a680 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_no_id.py @@ -0,0 +1,22 @@ +"""A simple example test with Test Case ID decorator no arguments.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_no_id_params_false.py b/examples/test_case_id/test_case_id_decorator_no_id_params_false.py new file mode 100644 index 00000000..005046a4 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_no_id_params_false.py @@ -0,0 +1,22 @@ +"""A simple example test with Test Case ID decorator no arguments.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(parameterized=False) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_no_id_params_true.py b/examples/test_case_id/test_case_id_decorator_no_id_params_true.py new file mode 100644 index 00000000..5ff41b2a --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_no_id_params_true.py @@ -0,0 +1,22 @@ +"""A simple example test with Test Case ID decorator no arguments.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(parameterized=True) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_no_id_partial_params.py b/examples/test_case_id/test_case_id_decorator_no_id_partial_params.py new file mode 100644 index 00000000..2a980bde --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_no_id_partial_params.py @@ -0,0 +1,22 @@ +"""A simple example test with Test Case ID decorator no arguments.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(params=["param2"]) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_no_id_partial_params_true.py b/examples/test_case_id/test_case_id_decorator_no_id_partial_params_true.py new file mode 100644 index 00000000..2222af64 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_no_id_partial_params_true.py @@ -0,0 +1,22 @@ +"""A simple example test with Test Case ID decorator no arguments.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(parameterized=True, params=["param2"]) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_params_false.py b/examples/test_case_id/test_case_id_decorator_params_false.py new file mode 100644 index 00000000..0830783d --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_params_false.py @@ -0,0 +1,24 @@ +"""A simple example test with Test Case ID decorator and parameters.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-321" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(TEST_CASE_ID, parameterized=False) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_params_mark.py b/examples/test_case_id/test_case_id_decorator_params_mark.py new file mode 100644 index 00000000..275528e8 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_params_mark.py @@ -0,0 +1,23 @@ +"""A simple example test with Test Case ID decorator and parameters.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-231" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")], ids=[TEST_CASE_ID]) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_params_no.py b/examples/test_case_id/test_case_id_decorator_params_no.py new file mode 100644 index 00000000..00cf143d --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_params_no.py @@ -0,0 +1,24 @@ +"""A simple example test with Test Case ID decorator and parameters.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-132" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(TEST_CASE_ID) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_params_partially.py b/examples/test_case_id/test_case_id_decorator_params_partially.py new file mode 100644 index 00000000..88995faf --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_params_partially.py @@ -0,0 +1,24 @@ +"""A simple example test with Test Case ID decorator and parameters.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-213" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(TEST_CASE_ID, parameterized=True, params=["param1"]) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_params_true.py b/examples/test_case_id/test_case_id_decorator_params_true.py new file mode 100644 index 00000000..bb3db5a0 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_params_true.py @@ -0,0 +1,24 @@ +"""A simple example test with Test Case ID decorator and parameters.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-231" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(TEST_CASE_ID, parameterized=True) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_use_index_false.py b/examples/test_case_id/test_case_id_decorator_use_index_false.py new file mode 100644 index 00000000..e685e4a9 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_use_index_false.py @@ -0,0 +1,24 @@ +"""A simple example test with Test Case ID decorator using parameter values.""" + +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-USE-INDEX-FALSE" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(TEST_CASE_ID, parameterized=True, use_index=False) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_use_index_partial_params.py b/examples/test_case_id/test_case_id_decorator_use_index_partial_params.py new file mode 100644 index 00000000..c130f2e3 --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_use_index_partial_params.py @@ -0,0 +1,24 @@ +"""A simple example test with Test Case ID decorator using parameter indices with selected params.""" + +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-USE-INDEX-PARTIAL" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(TEST_CASE_ID, parameterized=True, params=["param1"], use_index=True) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_case_id/test_case_id_decorator_use_index_true.py b/examples/test_case_id/test_case_id_decorator_use_index_true.py new file mode 100644 index 00000000..7b8bf1bc --- /dev/null +++ b/examples/test_case_id/test_case_id_decorator_use_index_true.py @@ -0,0 +1,24 @@ +"""A simple example test with Test Case ID decorator using parameter indices.""" + +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +TEST_CASE_ID = "ISSUE-USE-INDEX-TRUE" + + +@pytest.mark.parametrize(("param1", "param2"), [("value1", "value2")]) +@pytest.mark.tc_id(TEST_CASE_ID, parameterized=True, use_index=True) +def test_case_id_decorator(param1, param2): + assert True diff --git a/examples/test_issue_id.py b/examples/test_issue_id.py new file mode 100644 index 00000000..38abcf18 --- /dev/null +++ b/examples/test_issue_id.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +ID = "ABC-1234" +REASON = "some_bug" +TYPE = "PB" + + +@pytest.mark.issue(issue_id=ID, reason=REASON, issue_type=TYPE) +def test_issue_id(): + assert False diff --git a/examples/test_issue_id_pass.py b/examples/test_issue_id_pass.py new file mode 100644 index 00000000..d4347d88 --- /dev/null +++ b/examples/test_issue_id_pass.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +ID = "ABC-1234" +REASON = "some_bug" +TYPE = "PB" + + +@pytest.mark.issue(issue_id=ID, reason=REASON, issue_type=TYPE) +def test_issue_id(): + assert True diff --git a/examples/test_max_item_name.py b/examples/test_max_item_name.py new file mode 100644 index 00000000..4e929a2b --- /dev/null +++ b/examples/test_max_item_name.py @@ -0,0 +1,18 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_thi_is_simple_example_test_with_the_name_longer_than_maximum_allowed_lorem_ipsum_dolor_sit_amet_consectetur_adipiscing_elit_sed_do_eiusmod_tempor_incididunt_ut_labore_et_dolore_magna_aliqua_ut_enim_ad_minim_veniam_quis_nostrud_exercitation_ullamco_laboris_nisi_ut_aliquip_ex_ea_commodo_consequat_duis_aute_irure_dolor_in_reprehenderit_in_voluptate_velit_esse_cillum_dolore_eu_fugiat_nulla_pariatur_excepteur_sint_occaecat_cupidatat_non_proident_sunt_in_culpa_qui_officia_deserunt_mollit_anim_id_est_laborum_sed_ut_perspiciatis_unde_omnis_iste_natus_error_sit_voluptatem_accusantium_doloremque_laudantium_totam_rem_aperiam_eaque_ipsa_quae_ab_illo_inventore_veritatis_et_quasi_architecto_beatae_vitae_dicta_sunt_explicabo_nemo_enim_ipsam_voluptatem_quia_voluptas_sit_aspernatur_aut_odit_aut_fugit_sed_quia_consequuntur_magni_dolores_eos_qui_ratione_voluptatem_sequi_nesciunt_neque_porro_quisquam_est_qui_dolorem_ipsum_quia_dolor_sit_amet_consectetur_adipisci_velit_sed_quia_non_numquam_eius_modi_tempora_incidunt_ut_labore_et_dolore_magnam_aliquam_quaerat_voluptatem(): # noqa: E501 + """Simple example test with the name longer than maximum allowed.""" + assert True diff --git a/examples/test_rp_custom_logging.py b/examples/test_rp_custom_logging.py new file mode 100644 index 00000000..cfc018c3 --- /dev/null +++ b/examples/test_rp_custom_logging.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import logging + +logging.basicConfig(level=logging.INFO) + +logger = logging.getLogger(__name__) + +LOG_LEVEL: int = 35 +LOG_MESSAGE: str = "Assertion error" + + +def test_report_portal_logging(): + logger.log(LOG_LEVEL, LOG_MESSAGE) diff --git a/examples/test_rp_logging.py b/examples/test_rp_logging.py new file mode 100644 index 00000000..a7edf794 --- /dev/null +++ b/examples/test_rp_logging.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import logging + +logging.basicConfig(level=logging.INFO) + +logger = logging.getLogger(__name__) + +LOG_MESSAGE = "Standard logger logs to Report Portal" + + +def test_report_portal_logging(): + logger.info(LOG_MESSAGE) diff --git a/examples/test_simple.py b/examples/test_simple.py new file mode 100644 index 00000000..34f2441b --- /dev/null +++ b/examples/test_simple.py @@ -0,0 +1,19 @@ +"""Simple example test.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + + +def test_simple(): + """Simple example test.""" + assert True diff --git a/examples/test_simple_fail.py b/examples/test_simple_fail.py new file mode 100644 index 00000000..91094d5f --- /dev/null +++ b/examples/test_simple_fail.py @@ -0,0 +1,18 @@ +"""Simple example test which fails.""" + +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + + +def test_simple_fail(): + assert False diff --git a/examples/threads/test_thread_logging.py b/examples/threads/test_thread_logging.py new file mode 100644 index 00000000..3e1b4a2c --- /dev/null +++ b/examples/threads/test_thread_logging.py @@ -0,0 +1,19 @@ +import logging +import threading + +from reportportal_client.steps import step + +log = logging.getLogger(__name__) + + +def worker(): + log.info("TEST_INFO") + log.debug("TEST_DEBUG") + + +def test_log(): + t = threading.Thread(target=worker) + log.info("TEST_BEFORE_THREADING") + with step("Some nesting where the thread logs should go"): + t.start() + t.join() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..4cadcfa7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = [ + "setuptools>=68.0.0", + "setuptools-scm", + "wheel==0.46.2", +] +build-backend = "setuptools.build_meta" + +[tool.isort] +py_version=310 +line_length = 119 +profile = "black" +skip_gitignore = true + +[tool.black] +line-length = 119 +target-version = ["py310"] + +[tool.pytest.ini_options] +minversion = "6.0" +required_plugins = "pytest-cov" +testpaths = ["tests"] diff --git a/pytest_reportportal/__init__.py b/pytest_reportportal/__init__.py index de33f24b..83db8bab 100644 --- a/pytest_reportportal/__init__.py +++ b/pytest_reportportal/__init__.py @@ -1,3 +1,18 @@ -from .rp_logging import RPLogger, RPLogHandler +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License -__all__ = ['RPLogger', 'RPLogHandler'] +"""This package contains Pytest agent's code for the Report Portal.""" + +__all__ = ["LAUNCH_WAIT_TIMEOUT"] + +LAUNCH_WAIT_TIMEOUT = 10 diff --git a/pytest_reportportal/config.py b/pytest_reportportal/config.py new file mode 100644 index 00000000..ba0b70bf --- /dev/null +++ b/pytest_reportportal/config.py @@ -0,0 +1,247 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module contains class that stores RP agent configuration data.""" + +import logging +import warnings +from os import getenv +from typing import Any, Optional, Union + +from _pytest.config import Config +from reportportal_client import ClientType, OutputType +from reportportal_client.helpers import to_bool +from reportportal_client.logs import MAX_LOG_BATCH_PAYLOAD_SIZE + +ATTRIBUTES_SEPARATOR = ";" + + +def normalize_attributes(attributes: Optional[Any]) -> Optional[Any]: + """Split a string of attributes into a deduplicated list of attributes.""" + if not attributes: + return attributes + if not isinstance(attributes, str): + return attributes + normalized_attributes = [] + unique_attributes = set() + for attribute in attributes.split(ATTRIBUTES_SEPARATOR): + attribute = attribute.strip() + if attribute and attribute not in unique_attributes: + unique_attributes.add(attribute) + normalized_attributes.append(attribute) + return normalized_attributes + + +class AgentConfig: + """Storage for the RP agent initialization attributes.""" + + rp_enabled: bool + rp_client_type: ClientType + rp_rerun: Optional[bool] + pconfig: Config + rp_endpoint: str + rp_hierarchy_code: bool + rp_dir_level: int + rp_hierarchy_dirs: bool + rp_hierarchy_test_file: bool + rp_dir_path_separator: str + rp_ignore_attributes: set + rp_is_skipped_an_issue: bool + rp_issue_id_marks: bool + rp_bts_issue_url: str + rp_bts_project: str + rp_bts_url: str + rp_launch: str + rp_launch_uuid: Optional[str] + rp_launch_attributes: Optional[list[str]] + rp_tests_attributes: Optional[list[str]] + rp_launch_description: str + rp_log_batch_size: int + rp_log_batch_payload_limit: int + rp_log_level: Optional[int] + rp_log_format: Optional[str] + rp_mode: str + rp_parent_item_id: Optional[str] + rp_project: str + rp_rerun_of: Optional[str] + rp_api_retries: int + + # API key auth parameter + rp_api_key: Optional[str] + + # OAuth 2.0 parameters + rp_oauth_uri: Optional[str] + rp_oauth_username: Optional[str] + rp_oauth_password: Optional[str] + rp_oauth_client_id: Optional[str] + rp_oauth_client_secret: Optional[str] + rp_oauth_scope: Optional[str] + + rp_verify_ssl: Union[bool, str] + rp_launch_timeout: int + rp_launch_uuid_print: bool + rp_launch_uuid_print_output: OutputType + rp_http_timeout: Optional[Union[tuple[float, float], float]] + rp_report_fixtures: bool + + # Custom log levels and overrides + rp_log_custom_levels: Optional[dict[int, str]] + + def __init__(self, pytest_config: Config) -> None: + """Initialize required attributes.""" + self.rp_enabled = to_bool(self.find_option(pytest_config, "rp_enabled", True)) + self.rp_rerun = self.find_option(pytest_config, "rp_rerun") + self.rp_endpoint = self.find_option(pytest_config, "rp_endpoint") + self.rp_hierarchy_code = to_bool(self.find_option(pytest_config, "rp_hierarchy_code")) + self.rp_dir_level = int(self.find_option(pytest_config, "rp_hierarchy_dirs_level")) + self.rp_hierarchy_dirs = to_bool(self.find_option(pytest_config, "rp_hierarchy_dirs")) + self.rp_dir_path_separator = self.find_option(pytest_config, "rp_hierarchy_dir_path_separator") + self.rp_hierarchy_test_file = to_bool(self.find_option(pytest_config, "rp_hierarchy_test_file")) + self.rp_ignore_attributes = set(self.find_option(pytest_config, "rp_ignore_attributes") or []) + self.rp_is_skipped_an_issue = self.find_option(pytest_config, "rp_is_skipped_an_issue") + self.rp_issue_id_marks = self.find_option(pytest_config, "rp_issue_id_marks") + self.rp_bts_issue_url = self.find_option(pytest_config, "rp_bts_issue_url") + if not self.rp_bts_issue_url: + self.rp_bts_issue_url = self.find_option(pytest_config, "rp_issue_system_url") + if self.rp_bts_issue_url: + warnings.warn( + "Parameter `rp_issue_system_url` is deprecated since 5.4.0 and will be subject for removing" + "in the next major version. Use `rp_bts_issue_url` argument instead.", + DeprecationWarning, + 2, + ) + self.rp_bts_project = self.find_option(pytest_config, "rp_bts_project") + self.rp_bts_url = self.find_option(pytest_config, "rp_bts_url") + self.rp_launch = self.find_option(pytest_config, "rp_launch") + self.rp_launch_uuid = self.find_option(pytest_config, "rp_launch_id") + if self.rp_launch_uuid: + warnings.warn( + "Parameter `rp_launch_id` is deprecated since 5.6.2 and will be subject for removing" + "in the next major version. Use `rp_launch_uuid` argument instead.", + DeprecationWarning, + 2, + ) + self.rp_launch_uuid = self.find_option(pytest_config, "rp_launch_uuid", self.rp_launch_uuid) + + self.rp_launch_attributes = normalize_attributes(self.find_option(pytest_config, "rp_launch_attributes")) + self.rp_tests_attributes = normalize_attributes(self.find_option(pytest_config, "rp_tests_attributes")) + self.rp_launch_description = self.find_option(pytest_config, "rp_launch_description") + self.rp_log_batch_size = int(self.find_option(pytest_config, "rp_log_batch_size")) + batch_payload_size_limit = self.find_option(pytest_config, "rp_log_batch_payload_limit") + batch_payload_size = self.find_option(pytest_config, "rp_log_batch_payload_size") + if batch_payload_size: + warnings.warn( + "Parameter `rp_log_batch_payload_size` is deprecated since 5.5.4 " + "and will be subject for removing in the next major version. Use `rp_log_batch_payload_limit` argument" + " instead.", + DeprecationWarning, + 2, + ) + if not batch_payload_size_limit: + batch_payload_size_limit = batch_payload_size + + if batch_payload_size_limit: + self.rp_log_batch_payload_limit = int(batch_payload_size_limit) + else: + self.rp_log_batch_payload_limit = MAX_LOG_BATCH_PAYLOAD_SIZE + + log_level = self.find_option(pytest_config, "rp_log_level") + if not log_level: + self.rp_log_level = None + elif isinstance(log_level, int): + self.rp_log_level = log_level + else: + self.rp_log_level = int(getattr(logging, str(log_level).upper(), log_level)) + self.rp_log_format = self.find_option(pytest_config, "rp_log_format") + self.rp_thread_logging = to_bool(self.find_option(pytest_config, "rp_thread_logging") or False) + self.rp_mode = self.find_option(pytest_config, "rp_mode") + self.rp_parent_item_id = self.find_option(pytest_config, "rp_parent_item_id") + self.rp_project = self.find_option(pytest_config, "rp_project") + self.rp_rerun_of = self.find_option(pytest_config, "rp_rerun_of") + + rp_api_retries_str = self.find_option(pytest_config, "rp_api_retries", "0") + self.rp_api_retries = rp_api_retries_str and int(rp_api_retries_str) + + # API key auth parameter + self.rp_api_key = self.find_option(pytest_config, "rp_api_key") + + # OAuth 2.0 parameters + self.rp_oauth_uri = self.find_option(pytest_config, "rp_oauth_uri") + self.rp_oauth_username = self.find_option(pytest_config, "rp_oauth_username") + self.rp_oauth_password = self.find_option(pytest_config, "rp_oauth_password") + self.rp_oauth_client_id = self.find_option(pytest_config, "rp_oauth_client_id") + self.rp_oauth_client_secret = self.find_option(pytest_config, "rp_oauth_client_secret") + self.rp_oauth_scope = self.find_option(pytest_config, "rp_oauth_scope") + + rp_verify_ssl = self.find_option(pytest_config, "rp_verify_ssl", True) + try: + self.rp_verify_ssl = to_bool(rp_verify_ssl) + except (ValueError, AttributeError): + self.rp_verify_ssl = rp_verify_ssl + self.rp_launch_timeout = int(self.find_option(pytest_config, "rp_launch_timeout")) + + self.rp_launch_uuid_print = to_bool(self.find_option(pytest_config, "rp_launch_uuid_print") or "False") + print_output = self.find_option(pytest_config, "rp_launch_uuid_print_output") + self.rp_launch_uuid_print_output = OutputType[print_output.upper()] if print_output else OutputType.STDOUT + client_type = self.find_option(pytest_config, "rp_client_type") + self.rp_client_type = ClientType[client_type.upper()] if client_type else ClientType.SYNC + + connect_timeout = self.find_option(pytest_config, "rp_connect_timeout") + connect_timeout = float(connect_timeout) if connect_timeout else None + read_timeout = self.find_option(pytest_config, "rp_read_timeout") + read_timeout = float(read_timeout) if read_timeout else None + if connect_timeout is None and read_timeout is None: + self.rp_http_timeout = None + elif connect_timeout is not None and read_timeout is not None: + self.rp_http_timeout = (connect_timeout, read_timeout) + else: + self.rp_http_timeout = connect_timeout or read_timeout + self.rp_report_fixtures = to_bool(self.find_option(pytest_config, "rp_report_fixtures", False)) + + # Custom log levels and overrides + log_custom_levels = self.find_option(pytest_config, "rp_log_custom_levels") + self.rp_log_custom_levels = None + if log_custom_levels: + levels = {} + for custom_level in log_custom_levels: + level, level_name = str(custom_level).split(":", maxsplit=1) + levels[int(level)] = level_name + self.rp_log_custom_levels = levels + + # noinspection PyMethodMayBeStatic + def find_option(self, pytest_config: Config, option_name: str, default: Any = None) -> Any: + """ + Find a single configuration setting from multiple places. + + The value is retrieved in the following places in priority order: + + 1. From environment variable ``option_name.upper()`` (e.g. ``rp_endpoint`` -> ``RP_ENDPOINT``). + 2. From ``pytest_config.option.``. + 3. From ``pytest_config.getini(option_name)``. + 4. ``default`` value. + + :param pytest_config: config object of PyTest + :param option_name: name of the option + :param default: value to be returned if not found + :return: option value + """ + env_value = getenv(option_name.upper()) + if env_value: + return env_value + value = getattr(pytest_config.option, option_name, None) + if value is not None: + return value + value = pytest_config.getini(option_name) + if isinstance(value, bool): + return value + return value or default diff --git a/pytest_reportportal/errors.py b/pytest_reportportal/errors.py new file mode 100644 index 00000000..00b7f5be --- /dev/null +++ b/pytest_reportportal/errors.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes exceptions used in the package.""" + + +class PytestWarning(UserWarning): + """Pytest warning exception. + + This exception is about to stub absent PytestWarning in Pytest versions + up to 3.8.0. Get rid of this code once we drop support for Pytest versions + below 3.8.0. + """ diff --git a/pytest_reportportal/listener.py b/pytest_reportportal/listener.py deleted file mode 100644 index 60212813..00000000 --- a/pytest_reportportal/listener.py +++ /dev/null @@ -1,46 +0,0 @@ -import cgi -import pytest - -from .service import PyTestService - - -class RPReportListener(object): - def __init__(self): - # Identifier if TestItem is called: - # if setup is failed, pytest will NOT call - # TestItem and Result will not reported! - self.called = False - - # Test Item result - self.result = None - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_protocol(self, item): - PyTestService.start_pytest_item(item) - yield - item_result = self.result if self.called else 'SKIPPED' - PyTestService.finish_pytest_item(item_result) - self.called = False - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_makereport(self): - report = (yield).get_result() - - if report.longrepr: - PyTestService.post_log( - # Used for support python 2.7 - cgi.escape(report.longreprtext), - loglevel='ERROR', - ) - - if report.when == 'call': - self.called = True - - if report.passed: - item_result = 'PASSED' - elif report.failed: - item_result = 'FAILED' - else: - item_result = 'SKIPPED' - - self.result = item_result diff --git a/pytest_reportportal/plugin.py b/pytest_reportportal/plugin.py index b435dd87..6fab6aa1 100644 --- a/pytest_reportportal/plugin.py +++ b/pytest_reportportal/plugin.py @@ -1,103 +1,686 @@ -# This program is free software: you can redistribute it -# and/or modify it under the terms of the GPL licence +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module contains changed pytest for ReportPortal.""" import logging +import os.path +import pickle +import time +from logging import Logger +from typing import Any, Callable, Generator + +import _pytest.logging +import pytest + +# noinspection PyPackageRequirements +from pytest import Item, Session +from reportportal_client import RP, RPLogHandler +from reportportal_client.errors import ResponseError + +from pytest_reportportal import LAUNCH_WAIT_TIMEOUT +from pytest_reportportal.config import AgentConfig +from pytest_reportportal.rp_logging import patching_logger_class, patching_thread_class +from pytest_reportportal.service import PyTestService + +try: + # noinspection PyPackageRequirements + from pytest_bdd.parser import Feature, Scenario, Step + + PYTEST_BDD = True +except ImportError: + Feature = type("dummy", (), {}) + Scenario = type("dummy", (), {}) + Step = type("dummy", (), {}) + PYTEST_BDD = False + +LOGGER: Logger = logging.getLogger(__name__) + +MANDATORY_PARAMETER_MISSED_PATTERN: str = ( + "One of the following mandatory parameters is unset: " + "rp_project: {}, rp_endpoint: {}" +) + +FAILED_LAUNCH_WAIT: str = ( + "Failed to initialize reportportal-client service. " + + "Waiting for Launch start timed out. " + + "Reporting is disabled." +) + + +@pytest.hookimpl(optionalhook=True) +def pytest_configure_node(node: Any) -> None: + """Configure xdist node controller. + + :param node: Object of the xdist WorkerController class + """ + # noinspection PyProtectedMember + if not node.config._rp_enabled: + # Stop now if the plugin is not properly configured + return + node.workerinput["py_test_service"] = pickle.dumps(node.config.py_test_service) + + +# no 'config' type for backward compatibility for older pytest versions +def is_control(config) -> bool: + """Validate workerinput attribute of the Config object. + + True if the code, running the given pytest.config object, + is running as the xdist control node or not running xdist at all. + """ + return not hasattr(config, "workerinput") + + +def wait_launch(rp_client: RP) -> bool: + """Wait for the launch startup. + + :param rp_client: Instance of the ReportPortalService class + """ + timeout = time.time() + LAUNCH_WAIT_TIMEOUT + while not rp_client.launch_uuid: + if time.time() > timeout: + return False + time.sleep(1) + return True + + +# noinspection PyProtectedMember +def pytest_sessionstart(session: Session) -> None: + """Start Report Portal launch. + + This method is called every time on control or worker process start, it + analyses from which process it is called and starts a Report Portal launch + if it's a control process. + :param session: Object of the pytest Session class + """ + config = session.config + if not config._rp_enabled: + return + + try: + config.py_test_service.start() + except ResponseError as response_error: + LOGGER.warning("Failed to initialize reportportal-client service. " "Reporting is disabled.") + LOGGER.debug(str(response_error)) + config.py_test_service.rp = None + config._rp_enabled = False + return + + if is_control(config): + config.py_test_service.start_launch() + if config.pluginmanager.hasplugin("xdist") or config.pluginmanager.hasplugin("pytest-parallel"): + if not wait_launch(session.config.py_test_service.rp): + LOGGER.error(FAILED_LAUNCH_WAIT) + config.py_test_service.rp = None + config._rp_enabled = False + + +def pytest_collection_finish(session: Session) -> None: + """Collect tests if session is configured. + + :param session: Object of the pytest Session class + """ + # noinspection PyProtectedMember + if not session.config._rp_enabled: + # Stop now if the plugin is not properly configured + return + + session.config.py_test_service.collect_tests(session) + + +# noinspection PyProtectedMember +def pytest_sessionfinish(session: Session) -> None: + """Finish current test session. + + :param session: Object of the pytest Session class + """ + config = session.config + if not config._rp_enabled: + # Stop now if the plugin is not properly configured + return + + config.py_test_service.finish_suites() + if is_control(config): + config.py_test_service.finish_launch() + + config.py_test_service.stop() -from .service import PyTestService -from .listener import RPReportListener +# no 'config' type for backward compatibility for older pytest versions +def register_markers(config) -> None: + """Register plugin's markers, to avoid declaring them in `pytest.ini`. -def pytest_sessionstart(session): - PyTestService.init_service( - project=session.config.getini('rp_project'), - endpoint=session.config.getini('rp_endpoint'), - uuid=session.config.getini('rp_uuid'), - log_batch_size=int(session.config.getini('rp_log_batch_size')), - ignore_errors=bool(session.config.getini('rp_ignore_errors')), - ignored_tags=session.config.getini('rp_ignore_tags'), + :param config: Object of the pytest Config class + """ + config.addinivalue_line( + "markers", + "issue(issue_id, reason, issue_type, url): mark test with " "information about skipped or failed result", ) + config.addinivalue_line( + "markers", + "tc_id(id, parameterized, params): report the test" + "case with a custom Test Case ID. Parameters: \n" + "parameterized [True / False] - use parameter values in " + "Test Case ID generation \n" + "params [parameter names as list] - use only specified" + "parameters", + ) + config.addinivalue_line("markers", "name(name): report the test case with a custom Name.") + + +# no 'config' type for backward compatibility for older pytest versions +# noinspection PyProtectedMember +def pytest_configure(config) -> None: + """Update Config object with attributes required for reporting to RP. + + :param config: Object of the pytest Config class + """ + register_markers(config) - PyTestService.start_launch( - session.config.option.rp_launch, - tags=session.config.getini('rp_launch_tags'), - description=session.config.getini('rp_launch_description'), + config._rp_enabled = not ( + config.getoption("--collect-only", default=False) + or config.getoption("--setup-plan", default=False) + or not config.option.rp_enabled ) + if not config._rp_enabled: + LOGGER.debug("Disabling reporting to RP.") + return + agent_config = AgentConfig(config) + cond = (agent_config.rp_project, agent_config.rp_endpoint) + config._rp_enabled = all(cond) + if not config._rp_enabled: + LOGGER.debug(MANDATORY_PARAMETER_MISSED_PATTERN.format(*cond)) + LOGGER.debug("Disabling reporting to RP.") + return -def pytest_sessionfinish(): - # FixMe: currently method of RP api takes the string parameter - # so it is hardcoded - PyTestService.finish_launch(status='RP_Launch') + config._reporter_config = agent_config + if is_control(config): + config.py_test_service = PyTestService(agent_config) + else: + # noinspection PyUnresolvedReferences + config.py_test_service = pickle.loads(config.workerinput["py_test_service"]) -def pytest_configure(config): - if not config.option.rp_launch: - config.option.rp_launch = config.getini('rp_launch') - # set Pytest_Reporter and configure it - config._reporter = RPReportListener() +# noinspection PyProtectedMember +@pytest.hookimpl(hookwrapper=True) +def pytest_runtestloop(session: Session) -> Generator[None, Any, None]: + """ + Control start and finish of all test items in the session. - if hasattr(config, '_reporter'): - config.pluginmanager.register(config._reporter) + :param session: pytest.Session + :return: generator object + """ + config = session.config + if not config._rp_enabled: + yield + return + agent_config = config._reporter_config + with patching_thread_class(agent_config): + yield -def pytest_unconfigure(config): - PyTestService.terminate_service() - if hasattr(config, '_reporter'): - reporter = config._reporter - del config._reporter - config.pluginmanager.unregister(reporter) - logging.debug('RP is unconfigured') +# noinspection PyProtectedMember +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item: Item) -> Generator[None, Any, None]: + """Control start and finish of pytest items. + :param item: Pytest.Item + :return: generator object + """ + config = item.config + if not config._rp_enabled: + yield + return -def pytest_addoption(parser): - group = parser.getgroup('reporting') - group.addoption( - '--rp-launch', - action='store', - dest='rp_launch', - help='Launch name (overrides rp_launch config option)') + service = config.py_test_service + agent_config = config._reporter_config + service.start_pytest_item(item) - parser.addini( - 'rp_uuid', - help='UUID') + log_level = agent_config.rp_log_level or logging.NOTSET + log_handler = RPLogHandler( + level=log_level, + filter_client_logs=True, + endpoint=agent_config.rp_endpoint, + ignored_record_names=("reportportal_client", "pytest_reportportal"), + custom_levels=agent_config.rp_log_custom_levels, + ) + log_format = agent_config.rp_log_format + if log_format: + log_handler.setFormatter(logging.Formatter(log_format)) + with patching_logger_class(): + with _pytest.logging.catching_logs(log_handler, level=log_level): + yield - parser.addini( - 'rp_endpoint', - help='Server endpoint') + service.finish_pytest_item(item) - parser.addini( - 'rp_project', - help='Project name') - parser.addini( - 'rp_launch', - default='Pytest Launch', - help='Launch name') +# noinspection PyProtectedMember +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: Item) -> Generator[None, Any, None]: + """Change runtest_makereport function. - parser.addini( - 'rp_launch_tags', - type='args', - help='Launch tags, i.e Performance Regression') + :param item: pytest.Item + :return: None + """ + result = yield + if not item.config._rp_enabled: + return + report = result.get_result() + service = item.config.py_test_service + service.process_results(item, report) - parser.addini( - 'rp_launch_description', - default='', - help='Launch description') - parser.addini( - 'rp_log_batch_size', - default='20', - help='Size of batch log requests in async mode') +def report_fixture(request, fixturedef, name: str, error_msg: str) -> Generator[None, Any, None]: + """Report fixture setup and teardown. - parser.addini( - 'rp_ignore_errors', + :param request: Object of the FixtureRequest class + :param fixturedef: represents definition of the texture class + :param name: Name of the fixture + :param error_msg: Error message + """ + config = request.config + enabled = getattr(config, "_rp_enabled", False) + service = getattr(config, "py_test_service", None) + agent_config = getattr(config, "_reporter_config", object()) + report_fixtures = getattr(agent_config, "rp_report_fixtures", False) + if not enabled or not service or not report_fixtures: + yield + return + + cached_result = getattr(fixturedef, "cached_result", None) + if cached_result and hasattr(cached_result, "__getitem__"): + result = fixturedef.cached_result[2] + if hasattr(result, "__getitem__"): + result = result[0] + if result and isinstance(result, BaseException): + yield + return + + yield from service.report_fixture(name, error_msg) + + +# no types for backward compatibility for older pytest versions +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef, request) -> Generator[None, Any, None]: + """Report fixture setup. + + :param fixturedef: represents definition of the texture class + :param request: represents fixture execution metadata + """ + yield from report_fixture( + request, + fixturedef, + f"{fixturedef.scope} fixture setup: {fixturedef.argname}", + f"{fixturedef.scope} fixture setup failed: {fixturedef.argname}", + ) + + +# no types for backward compatibility for older pytest versions +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_post_finalizer(fixturedef, request) -> Generator[None, Any, None]: + """Report fixture teardown. + + :param fixturedef: represents definition of the texture class + :param request: represents fixture execution metadata + """ + yield from report_fixture( + request, + fixturedef, + f"{fixturedef.scope} fixture teardown: {fixturedef.argname}", + f"{fixturedef.scope} fixture teardown failed: {fixturedef.argname}", + ) + + +if PYTEST_BDD: + + @pytest.hookimpl(hookwrapper=True) + def pytest_bdd_before_scenario(request, feature: Feature, scenario: Scenario) -> Generator[None, Any, None]: + """Report BDD scenario start. + + :param request: represents item execution metadata + :param feature: represents feature file + :param scenario: represents scenario from feature file + """ + config = request.config + # noinspection PyProtectedMember + if not config._rp_enabled: + yield + return + service = config.py_test_service + service.start_bdd_scenario(feature, scenario) + yield + + @pytest.hookimpl(hookwrapper=True) + def pytest_bdd_after_scenario(request, feature: Feature, scenario: Scenario) -> Generator[None, Any, None]: + """Report BDD scenario finish. + + :param request: represents item execution metadata + :param feature: represents feature file + :param scenario: represents scenario from feature file + """ + config = request.config + # noinspection PyProtectedMember + if not config._rp_enabled: + yield + return + + yield + service = config.py_test_service + service.finish_bdd_scenario(feature, scenario) + + # noinspection PyUnusedLocal + @pytest.hookimpl(hookwrapper=True) + def pytest_bdd_before_step( + request, feature: Feature, scenario: Scenario, step: Step, step_func: Callable[..., Any] + ) -> Generator[None, Any, None]: + """Report BDD step start. + + :param request: represents item execution metadata + :param feature: represents feature file + :param scenario: represents scenario from feature file + :param step: represents step from scenario + :param step_func: represents function for step + """ + config = request.config + # noinspection PyProtectedMember + if not config._rp_enabled: + yield + return + + service = config.py_test_service + service.start_bdd_step(feature, scenario, step) + yield + + # noinspection PyUnusedLocal + @pytest.hookimpl(hookwrapper=True) + def pytest_bdd_after_step( + request, + feature: Feature, + scenario: Scenario, + step: Step, + step_func: Callable[..., Any], + step_func_args: dict[str, Any], + ) -> Generator[None, Any, None]: + """Report BDD step finish. + + :param request: represents item execution metadata + :param feature: represents feature file + :param scenario: represents scenario from feature file + :param step: represents step from scenario + :param step_func: represents function for step + :param step_func_args: represents arguments for step function + """ + config = request.config + # noinspection PyProtectedMember + if not config._rp_enabled: + yield + return + + yield + service = config.py_test_service + service.finish_bdd_step(feature, scenario, step) + + # noinspection PyUnusedLocal + @pytest.hookimpl(hookwrapper=True) + def pytest_bdd_step_error( + request, + feature: Feature, + scenario: Scenario, + step: Step, + step_func: Callable[..., Any], + step_func_args: dict[str, Any], + exception, + ) -> Generator[None, Any, None]: + """Report BDD step error. + + :param request: represents item execution metadata + :param feature: represents feature file + :param scenario: represents scenario from feature file + :param step: represents step from scenario + :param step_func: represents function for step + :param step_func_args: represents arguments for step function + :param exception: represents exception + """ + config = request.config + # noinspection PyProtectedMember + if not config._rp_enabled: + yield + return + + yield + service = config.py_test_service + service.finish_bdd_step_error(feature, scenario, step, exception) + + @pytest.hookimpl(hookwrapper=True) + def pytest_bdd_step_func_lookup_error( + request, feature: Feature, scenario: Scenario, step: Step, exception + ) -> Generator[None, Any, None]: + """Report BDD step lookup error. + + :param request: represents item execution metadata + :param feature: represents feature file + :param scenario: represents scenario from feature file + :param step: represents step from scenario + :param exception: represents exception + """ + config = request.config + # noinspection PyProtectedMember + if not config._rp_enabled: + yield + return + + service = config.py_test_service + service.start_bdd_step(feature, scenario, step) + yield + service.finish_bdd_step_error(feature, scenario, step, exception) + + +# no types for backward compatibility for older pytest versions +def pytest_addoption(parser) -> None: + """Add support for the RP-related options. + + :param parser: Object of the Parser class + """ + group = parser.getgroup("reporting") + + def add_shared_option(name, help_str, default=None, action="store"): + """ + Add an option to both the command line and the .ini file. + + This function modifies `parser` and `group` from the outer scope. + + :param name: name of the option + :param help_str: help message + :param default: default value + :param action: `group.addoption` action + """ + parser.addini( + name=name, + default=default, + help=help_str, + ) + group.addoption( + "--{0}".format(name.replace("_", "-")), + action=action, + dest=name, + help="{help} (overrides {name} config option)".format( + help=help_str, + name=name, + ), + ) + + group.addoption( + "--reportportal", action="store_true", dest="rp_enabled", default=False, help="Enable ReportPortal plugin" + ) + add_shared_option( + name="rp_launch", + help_str="Launch name", + default="Pytest Launch", + ) + add_shared_option( + name="rp_launch_id", + help_str="DEPRECATED: Use already existing launch-id. The plugin won't control the Launch status", + ) + add_shared_option( + name="rp_launch_uuid", + help_str="Use already existing launch UUID. The plugin won't control the Launch status", + ) + add_shared_option( + name="rp_launch_description", + help_str="Launch description", + default="", + ) + add_shared_option(name="rp_project", help_str="Project name") + add_shared_option( + name="rp_log_level", + help_str="Logging level for automated log records reporting", + ) + add_shared_option( + name="rp_log_format", + help_str="Logging format for automated log records reporting", + ) + add_shared_option( + name="rp_rerun", + help_str="Marks the launch as a rerun", + default=False, + action="store_true", + ) + add_shared_option( + name="rp_rerun_of", + help_str="ID of the launch to be marked as a rerun (use only with " "rp_rerun=True)", + default="", + ) + add_shared_option( + name="rp_parent_item_id", + help_str="Create all test item as child items of the given (already " "existing) item.", + ) + add_shared_option(name="rp_api_key", help_str="API key of Report Portal. Usually located on UI profile " "page.") + add_shared_option(name="rp_endpoint", help_str="Server endpoint") + add_shared_option(name="rp_mode", help_str="Visibility of current launch [DEFAULT, DEBUG]", default="DEFAULT") + add_shared_option( + name="rp_thread_logging", + help_str="EXPERIMENTAL: Report logs from threads. " + "This option applies a patch to the builtin Thread class, " + "and so it is turned off by default. Use with caution.", default=False, - type='bool', - help='Ignore Report Portal errors (exit otherwise)') + action="store_true", + ) + add_shared_option( + name="rp_launch_uuid_print", + help_str="Enables printing Launch UUID on test run start. Possible values: [True, False]", + ) + add_shared_option( + name="rp_launch_uuid_print_output", + help_str="Launch UUID print output. Default `stdout`. Possible values: [stderr, stdout]", + ) + + # OAuth 2.0 parameters + parser.addini("rp_oauth_uri", type="args", help="OAuth 2.0 token endpoint URL for password grant authentication") + parser.addini("rp_oauth_username", type="args", help="OAuth 2.0 username for password grant authentication") + parser.addini("rp_oauth_password", type="args", help="OAuth 2.0 password for password grant authentication") + parser.addini("rp_oauth_client_id", type="args", help="OAuth 2.0 client identifier") + parser.addini("rp_oauth_client_secret", type="args", help="OAuth 2.0 client secret") + parser.addini("rp_oauth_scope", type="args", help="OAuth 2.0 access token scope") + + rp_launch_attributes_help_str = "Launch attributes, i.e Performance Regression." + parser.addini("rp_launch_attributes", type="args", help=rp_launch_attributes_help_str) + group.addoption( + "--rp-launch-attributes", dest="rp_launch_attributes", help=rp_launch_attributes_help_str, nargs="+" + ) + + rp_test_attributes_help_str = "Attributes for all tests items, e.g. Smoke." + parser.addini("rp_tests_attributes", type="args", help=rp_test_attributes_help_str) + group.addoption("--rp-tests-attributes", dest="rp_tests_attributes", help=rp_test_attributes_help_str, nargs="+") + parser.addini("rp_log_batch_size", default="20", help="Size of batch log requests in async mode") + parser.addini( + "rp_log_batch_payload_limit", + help="Maximum payload size in bytes of async batch log requests", + ) + parser.addini( + "rp_log_batch_payload_size", + help="DEPRECATED: Maximum payload size in bytes of async batch log requests", + ) + parser.addini( + "rp_log_custom_levels", + type="args", + help="Custom log levels specified as 'int level:string'. E.G.: '35:ASSERTION'. Overrides existing level if int" + " level matches.", + ) + parser.addini("rp_ignore_attributes", type="args", help="Ignore specified pytest markers, i.e parametrize") + parser.addini( + "rp_is_skipped_an_issue", default=True, type="bool", help="Treat skipped tests as required investigation" + ) + parser.addini("rp_hierarchy_code", default=False, type="bool", help="Enables hierarchy for code") + parser.addini("rp_hierarchy_dirs_level", default="0", help="Directory starting hierarchy level") + parser.addini("rp_hierarchy_dirs", default=False, type="bool", help="Enables hierarchy for directories") + parser.addini( + "rp_hierarchy_dir_path_separator", + default=os.path.sep, + help="Path separator to display directories in test hierarchy", + ) + parser.addini("rp_hierarchy_test_file", default=True, type="bool", help="Show file name in hierarchy") parser.addini( - 'rp_ignore_tags', - type='args', - help='Ignore specified pytest markers, i.e parametrize') + "rp_issue_system_url", + default="", + help="URL to get issue description. Issue id from pytest mark will be added to this URL. " + 'Deprecated: use "rp_bts_issue_url".', + ) + parser.addini( + "rp_bts_issue_url", + default="", + help="URL to get issue description. Issue ID from pytest mark will be added to this URL by replacing " + '"{issue_id}" placeholder.', + ) + parser.addini( + "rp_bts_project", + default="", + help="Bug-tracking system project as it configured on Report Portal " + "server. To enable runtime external issue reporting you need to " + 'specify this and "rp_bts_url" property.', + ) + parser.addini( + "rp_bts_url", + default="", + help="URL of bug-tracking system as it configured on Report Portal " + "server. To enable runtime external issue reporting you need to " + 'specify this and "rp_bts_project" property.', + ) + parser.addini( + "rp_verify_ssl", + default="True", + help="True/False - verify HTTPS calls, or path to a CA_BUNDLE or " + "directory with certificates of trusted CAs.", + ) + parser.addini("rp_issue_id_marks", type="bool", default=True, help="Add tag with issue id to the test") + parser.addini("rp_api_retries", default="0", help="Amount of retries for performing REST calls to RP server") + parser.addini( + "rp_launch_timeout", + default=86400, + help="Maximum time to wait for child processes finish, default value: " "86400 seconds (1 day)", + ) + parser.addini( + "rp_client_type", + help="Type of the under-the-hood ReportPortal client implementation. Possible values: [SYNC, ASYNC_THREAD, " + "ASYNC_BATCHED]", + ) + parser.addini("rp_connect_timeout", help="Connection timeout to ReportPortal server") + parser.addini("rp_read_timeout", help="Response read timeout for ReportPortal connection") + parser.addini( + "rp_report_fixtures", + default=False, + type="bool", + help="Enable reporting fixtures as test items. Possible values: [True, False]", + ) diff --git a/pytest_reportportal/rp_logging.py b/pytest_reportportal/rp_logging.py index 7a3a8553..d6ce9a0d 100644 --- a/pytest_reportportal/rp_logging.py +++ b/pytest_reportportal/rp_logging.py @@ -1,72 +1,166 @@ -import sys +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""RPLogger class for low-level logging in tests.""" + import logging +import sys +import threading +from contextlib import contextmanager +from functools import wraps +from typing import Any + +from reportportal_client import RPLogger, current, set_current +from reportportal_client.core.worker import APIWorker + + +def is_api_worker(target): + """Check if target is an RP worker thread.""" + if target: + method_name = getattr(target, "__name__", None) + method_self = getattr(target, "__self__", None) + if method_name == "_monitor" and method_self: + clazz = getattr(method_self, "__class__", None) + if clazz is APIWorker: + return True + return False -from .service import PyTestService - - -class RPLogger(logging.getLoggerClass()): - def __init__(self, name, level=0): - super(RPLogger, self).__init__(name, level=level) - - def _log(self, level, msg, args, - exc_info=None, extra=None, stack_info=False, attachment=None): - """ - Low-level logging routine which creates a LogRecord and then calls - all the handlers of this logger to handle the record. - """ - sinfo = None - if logging._srcfile: - # IronPython doesn't track Python frames, so findCaller raises an - # exception on some versions of IronPython. We trap it here so that - # IronPython can use logging. - try: - fn, lno, func, sinfo = self.findCaller(stack_info) - except ValueError: # pragma: no cover - fn, lno, func = '(unknown file)', 0, '(unknown function)' - else: - fn, lno, func = '(unknown file)', 0, '(unknown function)' - - if exc_info and not isinstance(exc_info, tuple): - exc_info = sys.exc_info() - - record = self.makeRecord( - self.name, level, fn, lno, msg, args, exc_info, func, extra, sinfo - ) - record.attachment = attachment - self.handle(record) - - -class RPLogHandler(logging.Handler): - # Map loglevel codes from `logging` module to ReportPortal text names: - _loglevel_map = { - logging.NOTSET: 'TRACE', - logging.DEBUG: 'DEBUG', - logging.INFO: 'INFO', - logging.WARNING: 'WARN', - logging.ERROR: 'ERROR', - logging.CRITICAL: 'ERROR', - } - _sorted_levelnos = sorted(_loglevel_map.keys(), reverse=True) - - def __init__(self, level=logging.NOTSET): - super(RPLogHandler, self).__init__(level) - - def emit(self, record): - msg = '' +@contextmanager +def patching_thread_class(config): + """ + Add patch for Thread class. + + Set the parent thread client as the child thread's local client + """ + if not config.rp_thread_logging: + # Do nothing + yield + else: + original_start = threading.Thread.start + original_run = threading.Thread.run try: - msg = self.format(record) - except (KeyboardInterrupt, SystemExit): - raise - except: - self.handleError(record) - - for level in self._sorted_levelnos: - if level <= record.levelno: - break - - return PyTestService.post_log( - msg, - loglevel=self._loglevel_map[level], - attachment=record.__dict__.get('attachment', None), - ) + + def wrap_start(original_func): + @wraps(original_func) + def _start(self, *args, **kwargs): + """Save the invoking thread's client if there is one.""" + # Prevent an endless loop of workers being spawned + target = getattr(self, "_target", None) + if not is_api_worker(self) and not is_api_worker(target): + current_client = current() + self.parent_rp_client = current_client + return original_func(self, *args, **kwargs) + + return _start + + def wrap_run(original_func): + @wraps(original_func) + def _run(self, *args, **kwargs): + """Create a new client for the invoked thread.""" + client = None + if hasattr(self, "parent_rp_client") and self.parent_rp_client and not current(): + parent = self.parent_rp_client + client = parent.clone() + try: + return original_func(self, *args, **kwargs) + finally: + if client: + # Stop the client and remove any references + client.close() + self.parent_rp_client = None + del self.parent_rp_client + set_current(None) + + return _run + + if not hasattr(threading.Thread, "patched"): + # patch + threading.Thread.patched = True + threading.Thread.start = wrap_start(original_start) + threading.Thread.run = wrap_run(original_run) + yield + + finally: + if hasattr(threading.Thread, "patched"): + threading.Thread.start = original_start + threading.Thread.run = original_run + del threading.Thread.patched + + +@contextmanager +def patching_logger_class(): + """ + Add patch for RPLogger class. + + Updated attachment in logs + :return: wrapped function + """ + logger_class = logging.getLoggerClass() + original_log = logger_class._log + original_makeRecord = logger_class.makeRecord + + try: + + def wrap_log(original_func): + @wraps(original_func) + def _log(self, *args: list[Any], **kwargs: dict[str, Any]): + my_kwargs = kwargs.copy() + attachment = my_kwargs.pop("attachment", None) + if attachment is not None: + my_kwargs.setdefault("extra", {}).update({"attachment": attachment}) + + # Python 3.11 start catches stack frames in wrappers, + # so add additional stack level skip to not show it + if sys.version_info >= (3, 11): + if "stacklevel" in my_kwargs: + my_kwargs["stacklevel"] = my_kwargs["stacklevel"] + 1 + else: + my_kwargs["stacklevel"] = 2 + return original_func(self, *args, **my_kwargs) + else: + return original_func(self, *args, **my_kwargs) + + return _log + + def wrap_makeRecord(original_func): + @wraps(original_func) + def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None): + if extra is not None: + attachment = extra.pop("attachment", None) + else: + attachment = None + try: + # Python 3.5 + record = original_func( + self, name, level, fn, lno, msg, args, exc_info, func=func, extra=extra, sinfo=sinfo + ) + except TypeError: + # Python 2.7 + record = original_func(self, name, level, fn, lno, msg, args, exc_info, func=func, extra=extra) + record.attachment = attachment + return record + + return makeRecord + + if not issubclass(logger_class, RPLogger): + logger_class._log = wrap_log(logger_class._log) + logger_class.makeRecord = wrap_makeRecord(logger_class.makeRecord) + logging.setLoggerClass(RPLogger) + yield + + finally: + if not issubclass(logger_class, RPLogger): + logger_class._log = original_log + logger_class.makeRecord = original_makeRecord + logging.setLoggerClass(logger_class) diff --git a/pytest_reportportal/service.py b/pytest_reportportal/service.py index 4c87802d..75fa80e0 100644 --- a/pytest_reportportal/service.py +++ b/pytest_reportportal/service.py @@ -1,183 +1,1465 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes Service functions for work with pytest agent.""" + import logging +import os.path +import re import sys +import threading import traceback -import pytest - -from time import time -from six import with_metaclass -from six.moves import queue - -from reportportal_client import ReportPortalServiceAsync - - -def timestamp(): - return str(int(time() * 1000)) - - -class Singleton(type): - _instances = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__( - *args, **kwargs) - return cls._instances[cls] - - -class PyTestServiceClass(with_metaclass(Singleton, object)): - def __init__(self): - self.RP = None - self.ignore_errors = True - self.ignored_tags = [] - - self._errors = queue.Queue() - self._loglevels = ('TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR') - - def init_service(self, endpoint, project, uuid, log_batch_size, - ignore_errors, ignored_tags): - self._errors = queue.Queue() - if self.RP is None: - self.ignore_errors = ignore_errors - self.ignored_tags = ignored_tags - logging.debug( - msg='ReportPortal - Init service: ' - 'endpoint={0}, project={1}, uuid={2}'. - format(endpoint, project, uuid)) - self.RP = ReportPortalServiceAsync( - endpoint=endpoint, - project=project, - token=uuid, - error_handler=self.async_error_handler, - log_batch_size=log_batch_size - ) - else: - logging.debug('The pytest is already initialized') - return self.RP +from collections import OrderedDict +from datetime import datetime, timezone +from enum import Enum +from functools import wraps +from os import curdir +from time import sleep, time +from typing import Any, Callable, Generator, Optional, Union - def async_error_handler(self, exc_info): - self.terminate_service() - self.RP = None - self._errors.put_nowait(exc_info) +from _pytest.doctest import DoctestItem +from py.path import local +from pytest import Class, Function, Item, Module, Package, Session +from reportportal_client.aio import Task +from reportportal_client.core.rp_issues import ExternalIssue, Issue +from reportportal_client.helpers import markdown_helpers - def _stop_if_necessary(self): - try: - exc, msg, tb = self._errors.get(False) - traceback.print_exception(exc, msg, tb) - sys.stderr.flush() - if not self.ignore_errors: - pytest.exit(msg) - except queue.Empty: - pass - - def terminate_service(self): - if self.RP is not None: - self.RP.terminate() - - def start_launch( - self, launch_name, mode=None, tags=None, description=None): - self._stop_if_necessary() - if self.RP is None: - return - - sl_pt = { - 'name': launch_name, - 'start_time': timestamp(), - 'description': description, - 'mode': mode, - 'tags': tags +from .config import AgentConfig + +try: + # noinspection PyProtectedMember + from pytest import Instance +except ImportError: + # in pytest >= 7.0 this type was removed + Instance = type("dummy", (), {}) +try: + from pytest import Dir +except ImportError: + # in pytest < 8.0 there is no such type + Dir = type("dummy", (), {}) +try: + from pytest import Mark +except ImportError: + # in old pytest marks are located in the _pytest.mark module + from _pytest.mark import Mark +try: + # noinspection PyPackageRequirements + from pytest_bdd.parser import Background, Feature, Scenario, ScenarioTemplate, Step + + # noinspection PyPackageRequirements + from pytest_bdd.scenario import make_python_name + + PYTEST_BDD = True +except ImportError: + Background = type("dummy", (), {}) + Feature = type("dummy", (), {}) + Scenario = type("dummy", (), {}) + ScenarioTemplate = type("dummy", (), {}) + Step = type("dummy", (), {}) + make_python_name: Callable[[str], str] = lambda x: x + PYTEST_BDD = False + +try: + # noinspection PyPackageRequirements + from pytest_bdd.parser import Rule +except ImportError: + Rule = type("dummy", (), {}) # Old pytest-bdd versions do not have Rule + +from reportportal_client import RP, OutputType, create_client +from reportportal_client.helpers import dict_to_payload, gen_attributes, get_launch_sys_attrs, get_package_version + +LOGGER = logging.getLogger(__name__) + +KNOWN_LOG_LEVELS = ("TRACE", "DEBUG", "INFO", "WARN", "ERROR") +ROOT_DIR: str = str(os.path.abspath(curdir)) +PYTEST_MARKS_IGNORE: set[str] = {"parametrize", "usefixtures", "filterwarnings"} +NOT_ISSUE: Issue = Issue("NOT_ISSUE") +ISSUE_DESCRIPTION_LINE_TEMPLATE: str = "* {}:{}" +ISSUE_DESCRIPTION_URL_TEMPLATE: str = " [{issue_id}]({url})" +ISSUE_DESCRIPTION_ID_TEMPLATE: str = " {issue_id}" +PYTHON_REPLACE_REGEX = re.compile(r"\W") +ALPHA_REGEX = re.compile(r"^\d+_*") +BACKGROUND_STEP_NAME = "Background" + + +def _is_pytest_bdd_scenario(location_path: str) -> bool: + """ + Return True if the pytest collection path points at pytest-bdd's scenario module. + + ``Item.location[0]`` uses OS-native separators (backslashes on Windows), so a + plain suffix check with ``/`` is not portable. See #418. + """ + return location_path.endswith(os.path.join("pytest_bdd", "scenario.py")) + + +def trim_docstring(docstring: str) -> str: + """ + Convert docstring. + + :param docstring: input docstring + :return: trimmed docstring + """ + if not docstring: + return "" + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = docstring.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxsize + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxsize: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return "\n".join(trimmed) + + +class LeafType(Enum): + """This class stores test item path types.""" + + DIR = 1 + FILE = 2 + CODE = 3 + ROOT = 4 + SUITE = 5 + NESTED = 6 + + +class ExecStatus(Enum): + """This class stores test item path types.""" + + CREATED = 1 + IN_PROGRESS = 2 + FINISHED = 3 + + +def check_rp_enabled(func): + """Verify is RP is enabled in config.""" + + @wraps(func) + def wrap(*args, **kwargs): + if args and isinstance(args[0], PyTestService): + if not args[0].rp: + return + return func(*args, **kwargs) + + return wrap + + +class PyTestService: + """Pytest service class for reporting test results to the Report Portal.""" + + _config: AgentConfig + _issue_types: dict[str, str] + _tree_path: dict[Any, list[dict[str, Any]]] + _bdd_tree: Optional[dict[str, Any]] + _bdd_item_by_name: dict[str, Item] + _bdd_scenario_by_item: dict[Item, Scenario] + _bdd_item_by_scenario: dict[Scenario, Item] + _start_tracker: set[str] + _launch_id: Optional[str] + agent_name: str + agent_version: str + ignored_attributes: list[str] + parent_item_id: Optional[str] + rp: Optional[RP] + project_settings: Union[dict[str, Any], Task] + + def __init__(self, agent_config: AgentConfig) -> None: + """Initialize instance attributes.""" + self._config = agent_config + self._issue_types = {} + self._tree_path = {} + self._bdd_tree = None + self._bdd_item_by_name = OrderedDict() + self._bdd_scenario_by_item = {} + self._bdd_item_by_scenario = {} + self._start_tracker = set() + self._launch_id = None + self.agent_name = "pytest-reportportal" + self.agent_version = get_package_version(self.agent_name) or "None" + self.ignored_attributes = [] + self.parent_item_id = None + self.rp = None + self.project_settings = {} + + @property + def issue_types(self) -> dict[str, str]: + """Issue types for the Report Portal project.""" + if self._issue_types: + return self._issue_types + if not self.project_settings: + return self._issue_types + project_settings = self.project_settings + if not isinstance(self.project_settings, dict): + project_settings = project_settings.blocking_result() + if not project_settings: + return self._issue_types + for values in project_settings["subTypes"].values(): + for item in values: + self._issue_types[item["shortName"]] = item["locator"] + return self._issue_types + + def _get_launch_attributes(self, ini_attrs: Optional[list[dict[str, str]]]) -> list[dict[str, str]]: + """Generate launch attributes in the format supported by the client. + + :param list ini_attrs: List for attributes from the pytest.ini file + """ + attributes = ini_attrs or [] + system_attributes = get_launch_sys_attrs() + system_attributes["agent"] = "{}|{}".format(self.agent_name, self.agent_version) + return attributes + (dict_to_payload(system_attributes) or []) + + def _build_start_launch_rq(self) -> dict[str, Any]: + rp_launch_attributes = self._config.rp_launch_attributes + attributes = gen_attributes(rp_launch_attributes) if rp_launch_attributes else None + + start_rq = { + "attributes": self._get_launch_attributes(attributes), + "name": self._config.rp_launch, + "start_time": datetime.now(tz=timezone.utc), + "description": self._config.rp_launch_description, + "rerun": self._config.rp_rerun, + "rerun_of": self._config.rp_rerun_of, + } + return start_rq + + @check_rp_enabled + def start_launch(self) -> Optional[str]: + """ + Launch test items. + + :return: item ID + """ + sl_pt = self._build_start_launch_rq() + LOGGER.debug("ReportPortal - Start launch: request_body=%s", sl_pt) + self._launch_id = self.rp.start_launch(**sl_pt) + LOGGER.debug("ReportPortal - Launch started: id=%s", self._launch_id) + return self._launch_id + + def _get_item_dirs(self, item: Item) -> list[local]: + """ + Get directory of item. + + :param item: pytest.Item + :return: list of dirs + """ + root_path = item.session.config.rootdir.strpath + dir_path = item.fspath.new(basename="") + rel_dir = dir_path.new(dirname=dir_path.relto(root_path), basename="", drive="") + return [d for d in rel_dir.parts(reverse=False) if d.basename] + + def _get_tree_path(self, item: Item) -> list[Item]: + """Get item of parents. + + :param item: pytest.Item + :return list of parents + """ + path = [item] + parent = item.parent + while parent is not None and not isinstance(parent, Session): + if not isinstance(parent, Instance) and not isinstance(parent, Dir) and not isinstance(parent, Package): + path.append(parent) + parent = parent.parent + + path.reverse() + return path + + def _create_leaf( + self, leaf_type, parent_item: Optional[dict[str, Any]], item: Optional[Any], item_id: Optional[str] = None + ) -> dict[str, Any]: + """Construct a leaf for the itest tree. + + :param leaf_type: the leaf type + :param parent_item: parent pytest.Item of the current leaf + :param item: the leaf's pytest.Item + :return: a leaf + """ + return { + "children": {}, + "type": leaf_type, + "item": item, + "parent": parent_item, + "lock": threading.Lock(), + "exec": ExecStatus.CREATED, + "item_id": item_id, } - logging.debug('ReportPortal - Start launch: ' - 'request_body=%s', sl_pt) - req_data = self.RP.start_launch(**sl_pt) - logging.debug('ReportPortal - Launch started: ' - 'response_body=%s', req_data) - def start_pytest_item(self, test_item=None): - self._stop_if_necessary() - if self.RP is None: + def _build_test_tree(self, session: Session) -> dict[str, Any]: + """Construct a tree of tests and their suites. + + :param session: pytest.Session object of the current execution + :return: a tree of all tests and their suites + """ + test_tree = self._create_leaf(LeafType.ROOT, None, None, item_id=self.parent_item_id) + + for item in session.items: + dir_path = self._get_item_dirs(item) + class_path = self._get_tree_path(item) + + current_leaf = test_tree + for i, leaf in enumerate(dir_path + class_path): + children_leafs = current_leaf["children"] + + leaf_type = LeafType.DIR + if i == len(dir_path): + leaf_type = LeafType.FILE + if i > len(dir_path): + leaf_type = LeafType.CODE + + if leaf not in children_leafs: + children_leafs[leaf] = self._create_leaf(leaf_type, current_leaf, leaf) + current_leaf = children_leafs[leaf] + return test_tree + + def _remove_root_dirs(self, test_tree: dict[str, Any], max_dir_level: int, dir_level: int = 0) -> None: + if test_tree["type"] == LeafType.ROOT: + items = list(test_tree["children"].items()) + for item, child_leaf in items: + self._remove_root_dirs(child_leaf, max_dir_level, 1) return + if test_tree["type"] == LeafType.DIR and dir_level <= max_dir_level: + new_level = dir_level + 1 + parent_leaf = test_tree["parent"] + current_item = test_tree["item"] + del parent_leaf["children"][current_item] + for item, child_leaf in test_tree["children"].items(): + parent_leaf["children"][item] = child_leaf + child_leaf["parent"] = parent_leaf + self._remove_root_dirs(child_leaf, max_dir_level, new_level) - start_rq = { - 'name': self._get_full_name(test_item), - 'description': self._get_description(test_item), - 'tags': self._get_tags(test_item), - 'start_time': timestamp(), - 'item_type': 'STEP' + def _remove_file_names(self, test_tree: dict[str, Any]) -> None: + if test_tree["type"] != LeafType.FILE: + items = list(test_tree["children"].items()) + for item, child_leaf in items: + self._remove_file_names(child_leaf) + return + if not self._config.rp_hierarchy_test_file: + parent_leaf = test_tree["parent"] + current_item = test_tree["item"] + del parent_leaf["children"][current_item] + for item, child_leaf in test_tree["children"].items(): + parent_leaf["children"][item] = child_leaf + child_leaf["parent"] = parent_leaf + self._remove_file_names(child_leaf) + + def _get_scenario_template(self, scenario: Scenario) -> Optional[ScenarioTemplate]: + line_num = scenario.line_number + feature = scenario.feature + scenario_template = None + for template in feature.scenarios.values(): + if template.line_number == line_num: + scenario_template = template + break + if scenario_template and isinstance(scenario_template, ScenarioTemplate): + return scenario_template + return None + + def _get_method_name(self, item: Item) -> str: + """Get the original test method name. + + Returns item.originalname if available, + otherwise strips any trailing @suffix from item.name while + preserving @ inside parameter brackets. + + :param item: pytest.Item + :return: original method name + """ + if hasattr(item, "originalname") and item.originalname is not None: + return item.originalname + + name = item.name + if "@" not in name: + return name + + last_bracket = name.rfind("]") + at_pos = name.rfind("@") + + if at_pos > last_bracket: + return name[:at_pos] + + return name + + def _generate_names(self, test_tree: dict[str, Any]) -> None: + if test_tree["type"] == LeafType.ROOT: + test_tree["name"] = "root" + + if test_tree["type"] == LeafType.DIR: + test_tree["name"] = test_tree["item"].basename + + if test_tree["type"] in {LeafType.CODE, LeafType.FILE}: + item = test_tree["item"] + if isinstance(item, Module): + test_tree["name"] = os.path.split(str(item.fspath))[1] + elif isinstance(item, Feature): + name = item.name if item.name else item.rel_filename + keyword = getattr(item, "keyword", "Feature") + test_tree["name"] = f"{keyword}: {name}" + elif isinstance(item, Scenario): + scenario_template = self._get_scenario_template(item) + if scenario_template and scenario_template.templated: + keyword = getattr(item, "keyword", "Scenario Outline") + else: + keyword = getattr(item, "keyword", "Scenario") + test_tree["name"] = f"{keyword}: {item.name}" + elif isinstance(item, Rule): + keyword = getattr(item, "keyword", "Rule") + test_tree["name"] = f"{keyword}: {item.name}" + else: + test_tree["name"] = item.name + + if test_tree["type"] == LeafType.SUITE: + item = test_tree["item"] + if isinstance(item, Rule): + keyword = getattr(item, "keyword", "Rule") + test_tree["name"] = f"{keyword}: {item.name}" + + for item, child_leaf in test_tree["children"].items(): + self._generate_names(child_leaf) + + def _merge_leaf_types(self, test_tree: dict[str, Any], leaf_types: set, separator: str) -> None: + child_items = list(test_tree["children"].items()) + if test_tree["type"] not in leaf_types: + for item, child_leaf in child_items: + self._merge_leaf_types(child_leaf, leaf_types, separator) + elif len(child_items) > 0: + parent_leaf = test_tree["parent"] + current_item = test_tree["item"] + current_name = test_tree["name"] + child_types = [child_leaf["type"] in leaf_types for _, child_leaf in child_items] + if all(child_types): + del parent_leaf["children"][current_item] + for item, child_leaf in child_items: + if all(child_types): + parent_leaf["children"][item] = child_leaf + child_leaf["parent"] = parent_leaf + child_leaf["name"] = current_name + separator + child_leaf["name"] + self._merge_leaf_types(child_leaf, leaf_types, separator) + + def _merge_dirs(self, test_tree: dict[str, Any]) -> None: + """Merge directory and file leaves using configured separator. + + :param test_tree: Test tree structure to merge + """ + self._merge_leaf_types(test_tree, {LeafType.DIR, LeafType.FILE}, self._config.rp_dir_path_separator) + + def _merge_code_with_separator(self, test_tree: dict[str, Any], separator: str, is_bdd: bool = False) -> None: + """Merge code and suite leaves, respecting hierarchy flags. + + :param test_tree: Test tree structure to merge + :param separator: Separator to use when merging names + :param is_bdd: If True, always merge FILE for BDD scenarios. Otherwise respect rp_hierarchy_test_file + """ + types_to_merge = {LeafType.CODE, LeafType.SUITE} + if is_bdd or not self._config.rp_hierarchy_test_file: + types_to_merge.add(LeafType.FILE) + if not self._config.rp_hierarchy_dirs: + types_to_merge.add(LeafType.DIR) + self._merge_leaf_types(test_tree, types_to_merge, separator) + + def _merge_code(self, test_tree: dict[str, Any]) -> None: + """Merge code and suite leaves using double colon separator. + + :param test_tree: Test tree structure to merge + """ + self._merge_code_with_separator(test_tree, "::") + + def _build_item_paths(self, leaf: dict[str, Any], path: list[dict[str, Any]]) -> None: + children = leaf.get("children", {}) + if PYTEST_BDD: + all_background_steps = all([isinstance(child, Background) for child in children.keys()]) + else: + all_background_steps = False + if len(children) > 0 and not all_background_steps: + path.append(leaf) + for name, child_leaf in leaf["children"].items(): + self._build_item_paths(child_leaf, path) + path.pop() + elif leaf["type"] != LeafType.ROOT: + self._tree_path[leaf["item"]] = path + [leaf] + + @check_rp_enabled + def collect_tests(self, session: Session) -> None: + """Collect all tests. + + :param session: pytest.Session + """ + # Create a test tree to be able to apply mutations + test_tree = self._build_test_tree(session) + self._remove_root_dirs(test_tree, self._config.rp_dir_level) + self._remove_file_names(test_tree) + self._generate_names(test_tree) + if not self._config.rp_hierarchy_dirs: + self._merge_dirs(test_tree) + if not self._config.rp_hierarchy_code: + self._merge_code(test_tree) + self._build_item_paths(test_tree, []) + + def _get_item_description(self, test_item: Any) -> Optional[str]: + """Get description of item. + + :param test_item: pytest.Item + :return string description + """ + if isinstance(test_item, (Class, Function, Module, Item)): + if hasattr(test_item, "obj"): + doc = test_item.obj.__doc__ + if doc is not None: + return trim_docstring(doc) + if isinstance(test_item, DoctestItem): + return test_item.reportinfo()[2] + if isinstance(test_item, (Feature, Rule)): + description = test_item.description + if description: + return description.lstrip() # There is a bug in pytest-bdd that adds an extra space + + def _lock(self, leaf: dict[str, Any], func: Callable[[dict[str, Any]], Any]) -> Any: + """ + Lock test tree leaf and execute a function, bypass the leaf to it. + + :param leaf: a leaf to lock + :param func: a function to execute + :return: the result of the function bypassed + """ + if "lock" in leaf: + with leaf["lock"]: + return func(leaf) + return func(leaf) + + def _process_bdd_attributes(self, item: Union[Feature, Scenario, Rule]) -> list[dict[str, str]]: + tags = [] + tags.extend(item.tags) + if isinstance(item, Scenario): + test_attributes = self._config.rp_tests_attributes + tags.extend(test_attributes if test_attributes else []) + template = self._get_scenario_template(item) + if template and template.templated: + examples = [] + if isinstance(template.examples, list): + examples.extend(template.examples) + else: + examples.append(template.examples) + for example in examples: + tags.extend(getattr(example, "tags", [])) + return gen_attributes(tags) + + def _get_suite_code_ref(self, leaf: dict[str, Any]) -> str: + item = leaf["item"] + if leaf["type"] == LeafType.DIR: + code_ref = str(item) + elif leaf["type"] == LeafType.FILE: + if isinstance(item, Feature): + code_ref = str(item.rel_filename) + else: + code_ref = str(item.fspath) + elif leaf["type"] == LeafType.SUITE: + code_ref = self._get_suite_code_ref(leaf["parent"]) + f"/[{type(item).__name__}:{item.name}]" + else: + code_ref = str(item.fspath) + return code_ref + + def _build_start_suite_rq(self, leaf: dict[str, Any]) -> dict[str, Any]: + code_ref = self._get_suite_code_ref(leaf) + parent_item_id = self._lock(leaf["parent"], lambda p: p.get("item_id")) if "parent" in leaf else None + item = leaf["item"] + payload = { + "name": leaf["name"], + "description": self._get_item_description(item), + "start_time": datetime.now(tz=timezone.utc), + "item_type": "SUITE", + "code_ref": code_ref, + "parent_item_id": parent_item_id, } + if isinstance(item, (Feature, Scenario, Rule)): + payload["attributes"] = self._process_bdd_attributes(item) + return payload + + def _start_suite(self, suite_rq: dict[str, Any]) -> Optional[str]: + LOGGER.debug("ReportPortal - Start Suite: request_body=%s", suite_rq) + return self.rp.start_test_item(**suite_rq) + + def _create_suite(self, leaf: dict[str, Any]) -> None: + if leaf["exec"] != ExecStatus.CREATED: + return + item_id = self._start_suite(self._build_start_suite_rq(leaf)) + leaf["item_id"] = item_id + leaf["exec"] = ExecStatus.IN_PROGRESS + + @check_rp_enabled + def _create_suite_path(self, item: Any) -> None: + path = self._tree_path[item] + for leaf in path[1:-1]: + if leaf["exec"] != ExecStatus.CREATED: + continue + self._lock(leaf, lambda p: self._create_suite(p)) + + def _get_item_name(self, mark) -> Optional[str]: + return mark.kwargs.get("name", mark.args[0] if mark.args else None) + + def _get_code_ref(self, item: Item) -> str: + # Generate script path from work dir, use only backslashes to have the + # same path on different systems and do not affect Test Case ID on + # different systems + path = os.path.relpath(str(item.fspath), ROOT_DIR).replace("\\", "/") + method_name = self._get_method_name(item) + parent = item.parent + classes = [method_name] + while not isinstance(parent, Module): + if not isinstance(parent, Instance) and hasattr(parent, "name"): + classes.append(parent.name) + if hasattr(parent, "parent"): + parent = parent.parent + else: + break + classes.reverse() + class_path = ".".join(classes) + return "{0}:{1}".format(path, class_path) - logging.debug('ReportPortal - Start TestItem: ' - 'request_body=%s', start_rq) - self.RP.start_test_item(**start_rq) - - def _get_tags(self, test_item): - # try to extract names of @pytest.mark.* decorators used for test item - # and exclude those which present in rp_ignore_tags parameter - mark_plugin = test_item.config.pluginmanager.getplugin('mark') - if mark_plugin: - keywords = test_item.keywords - marks = mark_plugin.MarkMapping(keywords)._mymarks - return [m for m in marks if m not in self.ignored_tags] + def _get_test_case_id( + self, + base_name: str, + parameterized: bool, + include_params: Optional[list[str]], + use_index: bool, + parameters: Optional[dict[str, Any]], + parameters_indices: Optional[dict[str, Any]], + ) -> str: + param_str = None + if parameterized and parameters is not None and len(parameters) > 0: + if include_params is not None and len(include_params) > 0: + if use_index: + param_list = [str((param, parameters_indices.get(param, None))) for param in include_params] + else: + param_list = [str(parameters.get(param, None)) for param in include_params] + elif use_index: + param_list = [str(param) for param in parameters_indices.items()] + else: + param_list = [str(param) for param in parameters.values()] + param_str = f"[{','.join(sorted(param_list))}]" + + if param_str is None: + return base_name else: + return base_name + param_str + + def _get_issue_ids(self, mark) -> list[Any]: + issue_ids = mark.kwargs.get("issue_id", []) or [] + if not isinstance(issue_ids, list): + issue_ids = [issue_ids] + return issue_ids + + def _get_issue_urls(self, mark, default_url: str) -> list[Optional[str]]: + issue_ids = self._get_issue_ids(mark) + if not issue_ids: return [] + mark_url = mark.kwargs.get("url", None) or default_url + return [str(mark_url).format(issue_id=str(issue_id)) if mark_url else None for issue_id in issue_ids] - def finish_pytest_item(self, status, issue=None): - self._stop_if_necessary() - if self.RP is None: - return + def _get_issue_description_line(self, mark, default_url: str) -> str: + issue_ids = self._get_issue_ids(mark) + if not issue_ids: + return mark.kwargs["reason"] + + issue_urls = self._get_issue_urls(mark, default_url) + reason = mark.kwargs.get("reason", mark.name) + issues = "" + for i, issue_id in enumerate(issue_ids): + issue_url = issue_urls[i] + template = ISSUE_DESCRIPTION_URL_TEMPLATE if issue_url else ISSUE_DESCRIPTION_ID_TEMPLATE + issues += template.format(issue_id=issue_id, url=issue_url) + return ISSUE_DESCRIPTION_LINE_TEMPLATE.format(reason, issues) + + def _get_issue(self, mark: Mark) -> Optional[Issue]: + """Add issues description and issue_type to the test item. + + :param mark: pytest mark + :return: Issue object + """ + default_url = self._config.rp_bts_issue_url + + issue_description_line = self._get_issue_description_line(mark, default_url) + + # Set issue_type only for first issue mark + issue_short_name = None + if "issue_type" in mark.kwargs: + issue_short_name = mark.kwargs["issue_type"] + + # default value + issue_short_name = "TI" if issue_short_name is None else str(issue_short_name) + + registered_issues = self.issue_types + issue = None + if issue_short_name in registered_issues: + issue = Issue(registered_issues[issue_short_name], issue_description_line) + + if issue and self._config.rp_bts_project and self._config.rp_bts_url: + issue_ids = self._get_issue_ids(mark) + issue_urls = self._get_issue_urls(mark, default_url) + for issue_id, issue_url in zip(issue_ids, issue_urls): + issue.external_issue_add( + ExternalIssue( + bts_url=self._config.rp_bts_url, + bts_project=self._config.rp_bts_project, + ticket_id=issue_id, + url=issue_url, + ) + ) + return issue + + def _to_attribute(self, attribute_tuple): + if attribute_tuple[0]: + return {"key": attribute_tuple[0], "value": attribute_tuple[1]} + else: + return {"value": attribute_tuple[1]} + + def _process_item_name(self, leaf: dict[str, Any]) -> str: + """ + Process Item Name if set. + + :param leaf: item context + :return: Item Name string + """ + item = leaf["item"] + name = leaf["name"] + names = [m for m in item.iter_markers() if m.name == "name"] + if len(names) > 0: + mark_name = self._get_item_name(names[0]) + if mark_name: + name = mark_name + return name - fta_rq = { - 'end_time': timestamp(), - 'status': status, - 'issue': issue + def _get_parameters(self, item) -> Optional[dict[str, Any]]: + """ + Get params of item. + + :param item: Pytest.Item + :return: dict of params + """ + params = item.callspec.params if hasattr(item, "callspec") else None + return params + + def _get_parameters_indices(self, item) -> Optional[dict[str, Any]]: + """ + Get params indices of item. + + :param item: Pytest.Item + :return: dict of params indices + """ + indices = item.callspec.indices if hasattr(item, "callspec") else None + if not indices: + return None + + return indices + + def _process_test_case_id(self, leaf: dict[str, Any]) -> str: + """ + Process Test Case ID if set. + + :param leaf: item context + :return: Test Case ID string + """ + item = leaf["item"] + base_name = leaf["code_ref"] + parameterized = True + include_params = None + use_index = False + parameters: Optional[dict[str, Any]] = leaf.get("parameters", None) + parameters_indices: Optional[dict[str, Any]] = leaf.get("parameters_indices") or {} + + parametrize_markers = [m for m in item.iter_markers() if m.name == "parametrize"] + if parametrize_markers: + mark = parametrize_markers[0] + mark_kwargs = getattr(mark, "kwargs", None) + if mark_kwargs and "ids" in mark_kwargs and mark_kwargs["ids"]: + base_name = item.callspec.id + parameterized = False + + tc_ids = [m for m in item.iter_markers() if m.name == "tc_id"] + if tc_ids: + mark = tc_ids[0] + parameterized = mark.kwargs.get("parameterized", False) + include_params: Optional[Union[str, list[str]]] = mark.kwargs.get("params", None) + use_index = mark.kwargs.get("use_index", False) + + if include_params is not None and not isinstance(include_params, list): + include_params = [include_params] + + if mark.args is not None and len(mark.args) > 0: + base_name = str(mark.args[0]) + else: + base_name = "" + + return self._get_test_case_id( + base_name, parameterized, include_params, use_index, parameters, parameters_indices + ) + + def _process_issue(self, item: Item) -> Optional[Issue]: + """ + Process Issue if set. + + :param item: Pytest.Item + :return: Issue + """ + issues = [m for m in item.iter_markers() if m.name == "issue"] + if len(issues) > 0: + return self._get_issue(issues[0]) + + def _process_attributes(self, item: Item) -> list[dict[str, Any]]: + """ + Process attributes of item. + + :param item: Pytest.Item + :return: a set of attributes + """ + test_attributes = self._config.rp_tests_attributes + if test_attributes: + attributes = { + (attr.get("key", None), attr["value"]) + for attr in gen_attributes(self._config.rp_tests_attributes or []) + } + else: + attributes = set() + for marker in item.iter_markers(): + if marker.name == "issue": + if self._config.rp_issue_id_marks: + for issue_id in self._get_issue_ids(marker): + attributes.add((marker.name, issue_id)) + continue + if marker.name == "name": + continue + if marker.name in self._config.rp_ignore_attributes or marker.name in PYTEST_MARKS_IGNORE: + continue + if len(marker.args) > 0: + attributes.add((marker.name, str(marker.args[0]))) + else: + attributes.add((None, marker.name)) + + return [self._to_attribute(attribute) for attribute in attributes] + + def _process_metadata_item_start(self, leaf: dict[str, Any]) -> None: + """ + Process all types of item metadata for its start event. + + :param leaf: item context + """ + item = leaf["item"] + leaf["name"] = self._process_item_name(leaf) + leaf["description"] = self._get_item_description(item) + leaf["parameters"] = self._get_parameters(item) + leaf["parameters_indices"] = self._get_parameters_indices(item) + leaf["code_ref"] = self._get_code_ref(item) + leaf["test_case_id"] = self._process_test_case_id(leaf) + leaf["issue"] = self._process_issue(item) + leaf["attributes"] = self._process_attributes(item) + + def _process_metadata_item_finish(self, leaf: dict[str, Any]) -> None: + """ + Process all types of item metadata for its finish event. + + :param leaf: item context + """ + item = leaf["item"] + leaf["attributes"] = self._process_attributes(item) + leaf["issue"] = self._process_issue(item) + + def _build_start_step_rq(self, leaf: dict[str, Any]) -> dict[str, Any]: + payload = { + "attributes": leaf.get("attributes", None), + "name": leaf["name"], + "description": leaf["description"], + "start_time": datetime.now(tz=timezone.utc), + "item_type": "STEP", + "code_ref": leaf.get("code_ref", None), + "parameters": leaf.get("parameters", None), + "parent_item_id": self._lock(leaf["parent"], lambda p: p["item_id"]), + "test_case_id": leaf.get("test_case_id", None), } + return payload + + def _start_step(self, step_rq: dict[str, Any]) -> Optional[str]: + LOGGER.debug("ReportPortal - Start TestItem: request_body=%s", step_rq) + return self.rp.start_test_item(**step_rq) + + def __unique_id(self) -> str: + return str(os.getpid()) + "-" + str(threading.current_thread().ident) + + def __started(self) -> bool: + return self.__unique_id() in self._start_tracker - logging.debug('ReportPortal - Finish TestItem: ' - 'request_body=%s', fta_rq) - self.RP.finish_test_item(**fta_rq) + @check_rp_enabled + def start_pytest_item(self, test_item: Optional[Item] = None): + """ + Start pytest_item. - def finish_launch(self, launch=None, status='rp_launch'): - self._stop_if_necessary() - if self.RP is None: + :param test_item: pytest.Item + :return: None + """ + if test_item is None: return - # To finish launch session str parameter is needed - fl_rq = { - 'end_time': timestamp(), - 'status': status + if not self.__started(): + self.start() + + if PYTEST_BDD and _is_pytest_bdd_scenario(test_item.location[0]): + self._bdd_item_by_name[test_item.name] = test_item + return + + self._create_suite_path(test_item) + current_leaf = self._tree_path[test_item][-1] + self._process_metadata_item_start(current_leaf) + item_id = self._start_step(self._build_start_step_rq(current_leaf)) + current_leaf["item_id"] = item_id + current_leaf["exec"] = ExecStatus.IN_PROGRESS + + def process_results(self, test_item: Item, report): + """ + Save test item results after execution. + + :param test_item: pytest.Item + :param report: pytest's result report + """ + if report.longrepr: + self.post_log(test_item, report.longreprtext, log_level="ERROR") + + if PYTEST_BDD and _is_pytest_bdd_scenario(test_item.location[0]): + return + + leaf = self._tree_path[test_item][-1] + # Defining test result + if report.when == "setup": + leaf["status"] = "PASSED" + + if report.failed: + leaf["status"] = "FAILED" + return + + if report.skipped: + if leaf["status"] in (None, "PASSED"): + leaf["status"] = "SKIPPED" + + def _build_finish_step_rq(self, leaf: dict[str, Any]) -> dict[str, Any]: + issue = leaf.get("issue", None) + status = leaf.get("status", "PASSED") + if status == "SKIPPED" and not self._config.rp_is_skipped_an_issue: + issue = NOT_ISSUE + if status == "PASSED": + issue = None + payload = { + "attributes": leaf.get("attributes", None), + "end_time": datetime.now(tz=timezone.utc), + "status": status, + "issue": issue, + "item_id": leaf["item_id"], } - logging.debug('ReportPortal - Finish launch: request_body=%s', fl_rq) - self.RP.finish_launch(**fl_rq) + return payload + + def _finish_step(self, finish_rq: dict[str, Any]) -> None: + LOGGER.debug("ReportPortal - Finish TestItem: request_body=%s", finish_rq) + self.rp.finish_test_item(**finish_rq) + + def _finish_suite(self, finish_rq: dict[str, Any]) -> None: + LOGGER.debug("ReportPortal - End TestSuite: request_body=%s", finish_rq) + self.rp.finish_test_item(**finish_rq) + + def _build_finish_suite_rq(self, leaf) -> dict[str, Any]: + payload = {"end_time": datetime.now(tz=timezone.utc), "item_id": leaf["item_id"]} + return payload + + def _proceed_suite_finish(self, leaf) -> None: + if leaf.get("exec", ExecStatus.FINISHED) == ExecStatus.FINISHED: + return + + self._finish_suite(self._build_finish_suite_rq(leaf)) + leaf["exec"] = ExecStatus.FINISHED + + def _finish_parents(self, leaf: dict[str, Any]) -> None: + if ( + "parent" not in leaf + or leaf["parent"] is None + or leaf["parent"]["type"] is LeafType.ROOT + or leaf["parent"].get("exec", ExecStatus.FINISHED) == ExecStatus.FINISHED + ): + return + + for item, child_leaf in leaf["parent"]["children"].items(): + current_status = child_leaf["exec"] + if current_status != ExecStatus.FINISHED: + current_status = self._lock(child_leaf, lambda p: p["exec"]) + if current_status != ExecStatus.FINISHED: + return + + self._lock(leaf["parent"], lambda p: self._proceed_suite_finish(p)) + self._finish_parents(leaf["parent"]) + + @check_rp_enabled + def finish_pytest_item(self, test_item: Optional[Item] = None) -> None: + """Finish pytest_item. - def post_log(self, message, loglevel='INFO', attachment=None): - self._stop_if_necessary() - if self.RP is None: + :param test_item: pytest.Item + :return: None + """ + if test_item is None: return - if loglevel not in self._loglevels: - logging.warning('Incorrect loglevel = %s. Force set to INFO. ' - 'Available levels: %s.', loglevel, self._loglevels) - loglevel = 'INFO' + leaf = self._tree_path[test_item][-1] + self._process_metadata_item_finish(leaf) + + if PYTEST_BDD and _is_pytest_bdd_scenario(test_item.location[0]): + del self._bdd_item_by_name[test_item.name] + return + + self._finish_step(self._build_finish_step_rq(leaf)) + leaf["exec"] = ExecStatus.FINISHED + self._finish_parents(leaf) + + def _get_items(self, exec_status) -> list[Item]: + return [k for k, v in self._tree_path.items() if v[-1]["exec"] == exec_status] + + def finish_suites(self) -> None: + """ + Finish all suites in run with status calculations. + + If an execution passes in multiprocessing mode we don't know which and + how many items will be passed to our process. Because of that we don't + finish suites until the very last step. And after that we finish them + at once. + """ + # Ensure there is no running items + finish_time = time() + while ( + len(self._get_items(ExecStatus.IN_PROGRESS)) > 0 and time() - finish_time <= self._config.rp_launch_timeout + ): + sleep(0.1) + skipped_items = self._get_items(ExecStatus.CREATED) + for item in skipped_items: + path = list(self._tree_path[item]) + path.reverse() + for leaf in path[1:-1]: + if leaf["exec"] == ExecStatus.IN_PROGRESS: + self._lock(leaf, lambda p: self._proceed_suite_finish(p)) + + def _build_finish_launch_rq(self) -> dict[str, Any]: + finish_rq = {"end_time": datetime.now(tz=timezone.utc)} + return finish_rq + def _finish_launch(self, finish_rq) -> None: + LOGGER.debug("ReportPortal - Finish launch: request_body=%s", finish_rq) + self.rp.finish_launch(**finish_rq) + + @check_rp_enabled + def finish_launch(self) -> None: + """Finish test launch.""" + # To finish launch session str parameter is needed + self._finish_launch(self._build_finish_launch_rq()) + + def _build_log( + self, item_id: str, message: str, log_level: str, attachment: Optional[Any] = None + ) -> dict[str, Any]: sl_rq = { - 'time': timestamp(), - 'message': message, - 'level': loglevel, - 'attachment': attachment, + "item_id": item_id, + "time": datetime.now(tz=timezone.utc), + "message": message, + "level": log_level, } - self.RP.log(**sl_rq) + if attachment: + sl_rq["attachment"] = attachment + return sl_rq + + @check_rp_enabled + def post_log( + self, test_item: Item, message: str, log_level: str = "INFO", attachment: Optional[Any] = None + ) -> None: + """ + Send a log message to the Report Portal. + + :param test_item: pytest.Item + :param message: message in log body + :param log_level: a level of a log entry (ERROR, WARN, INFO, DEBUG, + TRACE, FATAL, UNKNOWN) + :param attachment: attachment file + :return: None + """ + if log_level not in KNOWN_LOG_LEVELS: + LOGGER.warning( + "Incorrect loglevel = %s. Force set to INFO. " "Available levels: %s.", log_level, KNOWN_LOG_LEVELS + ) + item_id = self._tree_path[test_item][-1]["item_id"] + if PYTEST_BDD: + if not item_id: + # Check if we are actually a BDD scenario + scenario = self._bdd_scenario_by_item.get(test_item, None) + if scenario: + # Yes, we are a BDD scenario, report log to the scenario + item_id = self._tree_path[scenario][-1]["item_id"] + + sl_rq = self._build_log(item_id, message, log_level, attachment) + self.rp.log(**sl_rq) - @staticmethod - def _get_full_name(test_item): - return test_item.nodeid + def report_fixture(self, name: str, error_msg: str) -> Generator[None, Any, None]: + """Report fixture setup and teardown. + + :param name: Name of the fixture + :param error_msg: Error message + """ + if not self.rp: + yield + return + + reporter = self.rp.step_reporter + item_id = reporter.start_nested_step(name, datetime.now(tz=timezone.utc)) - @staticmethod - def _get_description(test_item): try: - # for common items - return test_item.function.__doc__ - except AttributeError: - # doctest has no `function` attribute - return test_item.reportinfo()[2] + outcome = yield + exc_info = outcome.excinfo + exception = exc_info[1] if exc_info else None + status = "PASSED" + if exception: + if type(exception).__name__ != "Skipped": + status = "FAILED" + error_log = self._build_log(item_id, error_msg, log_level="ERROR") + self.rp.log(**error_log) + traceback_str = "\n".join( + traceback.format_exception(outcome.excinfo[0], value=exception, tb=exc_info[2]) + ) + exception_log = self._build_log(item_id, traceback_str, log_level="ERROR") + self.rp.log(**exception_log) + reporter.finish_nested_step(item_id, datetime.now(tz=timezone.utc), status) + except Exception as e: + LOGGER.error("Failed to report fixture: %s", name) + LOGGER.exception(e) + reporter.finish_nested_step(item_id, datetime.now(tz=timezone.utc), "FAILED") + + def _get_python_name(self, scenario: Scenario) -> str: + python_name = f"test_{make_python_name(self._get_scenario_template(scenario).name)}" + same_item_names = [name for name in self._bdd_item_by_name.keys() if name.startswith(python_name)] + if len(same_item_names) < 1: + return python_name + else: + return same_item_names[-1] # Should work fine, since we use OrderedDict + + def start_bdd_scenario(self, feature: Feature, scenario: Scenario) -> None: + """Save BDD scenario and Feature to test tree. The scenario will be started later if a step will be reported. + + :param feature: pytest_bdd.Feature + :param scenario: pytest_bdd.Scenario + """ + if not PYTEST_BDD: + return + item_name = self._get_python_name(scenario) + test_item = self._bdd_item_by_name.get(item_name, None) + self._bdd_scenario_by_item[test_item] = scenario + self._bdd_item_by_scenario[scenario] = test_item + + root_leaf = self._bdd_tree + if not root_leaf: + self._bdd_tree = root_leaf = self._create_leaf(LeafType.ROOT, None, None, item_id=self.parent_item_id) + # noinspection PyTypeChecker + children_leafs: dict[Any, Any] = root_leaf["children"] + if feature in children_leafs: + feature_leaf = children_leafs[feature] + else: + feature_leaf = self._create_leaf(LeafType.FILE, root_leaf, feature) + children_leafs[feature] = feature_leaf + children_leafs = feature_leaf["children"] + rule = getattr(scenario, "rule", None) + if rule: + if rule in children_leafs: + rule_leaf = children_leafs[rule] + else: + rule_leaf = self._create_leaf(LeafType.SUITE, feature_leaf, rule) + children_leafs[rule] = rule_leaf + else: + rule_leaf = feature_leaf + children_leafs = rule_leaf["children"] + scenario_leaf = self._create_leaf(LeafType.CODE, rule_leaf, scenario) + children_leafs[scenario] = scenario_leaf + children_leafs = scenario_leaf["children"] + background = feature.background + if background: + if background not in children_leafs: + background_leaf = self._create_leaf(LeafType.NESTED, rule_leaf, background) + children_leafs[background] = background_leaf + + self._remove_file_names(root_leaf) + self._generate_names(root_leaf) + if not self._config.rp_hierarchy_code: + try: + self._merge_code_with_separator(root_leaf, " - ", is_bdd=True) + except Exception as e: + LOGGER.exception(e) + self._build_item_paths(root_leaf, []) + + def finish_bdd_scenario(self, feature: Feature, scenario: Scenario) -> None: + """Finish BDD scenario. Skip if it was not started. + + :param feature: pytest_bdd.Feature + :param scenario: pytest_bdd.Scenario + """ + if not PYTEST_BDD: + return + + leaf = self._tree_path[scenario][-1] + if leaf["exec"] != ExecStatus.IN_PROGRESS: + return + self._finish_step(self._build_finish_step_rq(leaf)) + leaf["exec"] = ExecStatus.FINISHED + self._finish_parents(leaf) + + def _get_scenario_parameters_from_template(self, scenario: Scenario) -> Optional[dict[str, str]]: + """Get scenario parameters from its template by comparing steps. + + :param scenario: The scenario instance + :return: A dictionary with parameter names and values, or None if no parameters found + """ + item = self._bdd_item_by_scenario.get(scenario, None) + if not item: + return None + item_params = item.callspec.params if hasattr(item, "callspec") else None + if not item_params: + return None + if "_pytest_bdd_example" in item_params: + return OrderedDict(item_params["_pytest_bdd_example"]) + return None + + def _get_scenario_code_ref(self, scenario: Scenario, scenario_template: Optional[ScenarioTemplate]) -> str: + code_ref = scenario.feature.rel_filename + "/" + rule = getattr(scenario, "rule", None) + if rule: + code_ref += f"[RULE:{rule.name}]/" + if scenario_template and scenario_template.templated and scenario_template.examples: + parameters = self._get_scenario_parameters_from_template(scenario) + if parameters: + parameters_str = ";".join([f"{k}:{v}" for k, v in sorted(parameters.items())]) + parameters_str = f"[{parameters_str}]" if parameters_str else "" + else: + parameters_str = "" + code_ref += f"[EXAMPLE:{scenario.name}{parameters_str}]" + else: + keyword = getattr(scenario, "keyword", "Scenario").upper() + code_ref += f"[{keyword}:{scenario.name}]" + + return code_ref + + def _get_scenario_test_case_id(self, leaf: dict[str, Any]) -> str: + attributes = leaf.get("attributes", []) + params: Optional[dict[str, str]] = leaf.get("parameters", None) + for attribute in attributes: + if attribute.get("key", None) == "tc_id": + tc_id = attribute["value"] + params_str = "" + if params: + params_str = ";".join([f"{k}:{v}" for k, v in sorted(params.items())]) + params_str = f"[{params_str}]" + return f"{tc_id}{params_str}" + return leaf["code_ref"] + + def _process_scenario_metadata(self, leaf: dict[str, Any]) -> None: + """ + Process all types of scenario metadata for its start event. + + :param leaf: item context + """ + scenario = leaf["item"] + description = ( + "\n".join(scenario.description) if isinstance(scenario.description, list) else scenario.description + ).rstrip("\n") + leaf["description"] = description if description else None + scenario_template = self._get_scenario_template(scenario) + if scenario_template and scenario_template.templated: + parameters = self._get_scenario_parameters_from_template(scenario) + leaf["parameters"] = parameters + if parameters: + parameters_str = f"Parameters:\n\n{markdown_helpers.format_data_table_dict(parameters)}" + if leaf["description"]: + leaf["description"] = markdown_helpers.as_two_parts(leaf["description"], parameters_str) + else: + leaf["description"] = parameters_str + leaf["code_ref"] = self._get_scenario_code_ref(scenario, scenario_template) + leaf["attributes"] = self._process_bdd_attributes(scenario) + leaf["test_case_id"] = self._get_scenario_test_case_id(leaf) + + def _finish_bdd_step(self, leaf: dict[str, Any], status: str) -> None: + if leaf["exec"] != ExecStatus.IN_PROGRESS: + return + + reporter = self.rp.step_reporter + item_id = leaf["item_id"] + reporter.finish_nested_step(item_id, datetime.now(tz=timezone.utc), status) + leaf["exec"] = ExecStatus.FINISHED + + def _is_background_step(self, step: Step, feature: Feature) -> bool: + """Check if step belongs to feature background. + + :param step: Current step + :param feature: Current feature + :return: True if step is from background, False otherwise + """ + if not feature.background: + return False + + background_steps = feature.background.steps + return any( + s.name == step.name and s.keyword == step.keyword and s.line_number == step.line_number + for s in background_steps + ) + + @check_rp_enabled + def start_bdd_step(self, feature: Feature, scenario: Scenario, step: Step) -> None: + """Start BDD step. + + :param feature: pytest_bdd.Feature + :param scenario: pytest_bdd.Scenario + :param step: pytest_bdd.Step + """ + if not PYTEST_BDD: + return + + self._create_suite_path(scenario) + scenario_leaf = self._tree_path[scenario][-1] + if scenario_leaf["exec"] != ExecStatus.IN_PROGRESS: + self._process_scenario_metadata(scenario_leaf) + scenario_leaf["item_id"] = self._start_step(self._build_start_step_rq(scenario_leaf)) + scenario_leaf["exec"] = ExecStatus.IN_PROGRESS + reporter = self.rp.step_reporter + step_leaf = self._create_leaf(LeafType.NESTED, scenario_leaf, step) + if self._is_background_step(step, feature): + background_leaf = scenario_leaf["children"][feature.background] + background_leaf["children"][step] = step_leaf + if background_leaf["exec"] != ExecStatus.IN_PROGRESS: + item_id = reporter.start_nested_step(BACKGROUND_STEP_NAME, datetime.now(tz=timezone.utc)) + background_leaf["item_id"] = item_id + background_leaf["exec"] = ExecStatus.IN_PROGRESS + else: + scenario_leaf["children"][step] = step_leaf + if feature.background: + background_leaf = scenario_leaf["children"][feature.background] + self._finish_bdd_step(background_leaf, "PASSED") + item_id = reporter.start_nested_step(f"{step.keyword} {step.name}", datetime.now(tz=timezone.utc)) + step_leaf["item_id"] = item_id + step_leaf["exec"] = ExecStatus.IN_PROGRESS + + @check_rp_enabled + def finish_bdd_step(self, feature: Feature, scenario: Scenario, step: Step) -> None: + """Finish BDD step. + + :param feature: pytest_bdd.Feature + :param scenario: pytest_bdd.Scenario + :param step: pytest_bdd.Step + """ + if not PYTEST_BDD: + return + + scenario_leaf = self._tree_path[scenario][-1] + background_steps = [] + if feature.background: + background_steps = feature.background.steps + if next( + filter( + lambda s: s.name == step.name and s.keyword == step.keyword and s.line_number == step.line_number, + background_steps, + ), + None, + ): + parent_leaf = scenario_leaf["children"][feature.background] + else: + parent_leaf = scenario_leaf + step_leaf = parent_leaf["children"][step] + self._finish_bdd_step(step_leaf, "PASSED") + + @check_rp_enabled + def finish_bdd_step_error(self, feature: Feature, scenario: Scenario, step: Step, exception: Exception) -> None: + """Report BDD step error. + + :param feature: pytest_bdd.Feature + :param scenario: pytest_bdd.Scenario + :param step: pytest_bdd.Step + :param exception: Exception + """ + if not PYTEST_BDD: + return + + scenario_leaf = self._tree_path[scenario][-1] + scenario_leaf["status"] = "FAILED" + if step.background: + step_leaf = scenario_leaf["children"][step.background]["children"][step] + else: + step_leaf = scenario_leaf["children"][step] + item_id = step_leaf["item_id"] + traceback_str = "\n".join( + traceback.format_exception(type(exception), value=exception, tb=exception.__traceback__) + ) + exception_log = self._build_log(item_id, traceback_str, log_level="ERROR") + client = self.rp.step_reporter.client + client.log(**exception_log) + + self._finish_bdd_step(step_leaf, "FAILED") + if step.background: + background_leaf = scenario_leaf["children"][step.background] + self._finish_bdd_step(background_leaf, "FAILED") + def start(self) -> None: + """Start servicing Report Portal requests.""" + self.parent_item_id = self._config.rp_parent_item_id + self.ignored_attributes = list(set(self._config.rp_ignore_attributes or []).union({"parametrize"})) + LOGGER.debug( + "ReportPortal - Init service: endpoint=%s, " "project=%s, api_key=%s", + self._config.rp_endpoint, + self._config.rp_project, + self._config.rp_api_key, + ) + launch_id = self._launch_id + if self._config.rp_launch_uuid: + launch_id = self._config.rp_launch_uuid + self.rp = create_client( + client_type=self._config.rp_client_type, + endpoint=self._config.rp_endpoint, + project=self._config.rp_project, + api_key=self._config.rp_api_key, + is_skipped_an_issue=self._config.rp_is_skipped_an_issue, + log_batch_size=self._config.rp_log_batch_size, + retries=self._config.rp_api_retries, + verify_ssl=self._config.rp_verify_ssl, + launch_uuid=launch_id, + log_batch_payload_limit=self._config.rp_log_batch_payload_limit, + launch_uuid_print=self._config.rp_launch_uuid_print, + print_output=self._config.rp_launch_uuid_print_output or OutputType.STDOUT, + http_timeout=self._config.rp_http_timeout, + mode=self._config.rp_mode, + # OAuth 2.0 parameters + oauth_uri=self._config.rp_oauth_uri, + oauth_username=self._config.rp_oauth_username, + oauth_password=self._config.rp_oauth_password, + oauth_client_id=self._config.rp_oauth_client_id, + oauth_client_secret=self._config.rp_oauth_client_secret, + oauth_scope=self._config.rp_oauth_scope, + ) + if hasattr(self.rp, "get_project_settings"): + self.project_settings = self.rp.get_project_settings() + # noinspection PyUnresolvedReferences + self._start_tracker.add(self.__unique_id()) -PyTestService = PyTestServiceClass() + def stop(self) -> None: + """Finish servicing Report Portal requests.""" + self.rp.close() + self.rp = None + self._start_tracker.remove(self.__unique_id()) diff --git a/requirements-dev-bdd.txt b/requirements-dev-bdd.txt new file mode 100644 index 00000000..472e2bd1 --- /dev/null +++ b/requirements-dev-bdd.txt @@ -0,0 +1 @@ +pytest-bdd>=7.2.0 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..76a8e254 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +pytest>=4.6.10 +delayed-assert +pytest-cov +pytest-parallel +black +isort +mypy==1.19.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..b1139e69 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +reportportal-client==5.7.10 diff --git a/setup.cfg b/setup.cfg index 11d16761..4a307609 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,14 @@ +[aliases] +test=pytest + [metadata] -description-file = README.rst +description_file = README.md [sdist] formats=gztar [bdist_wheel] universal = 1 + +[tool:pytest] +addopts=-vv diff --git a/setup.py b/setup.py index cf13c891..96a55314 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,61 @@ +# Copyright (c) 2023 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""Config for setup package pytest agent.""" + import os from setuptools import setup - -def read_file(fname): - return open(os.path.join(os.path.dirname(__file__), fname)).read() +__version__ = "5.6.12" -version = '1.0.1' -tar_url = 'https://github.com/reportportal/agent-python-pytest/tarball/1.0.1' - +def read_file(fname): + """Read the given file. -requirements = [ - 'reportportal-client>=3.0.0', - 'pytest>=3.0.7', - 'six>=1.10.0', -] + :param fname: Filename to be read + :return: File content + """ + with open(os.path.join(os.path.dirname(__file__), fname)) as f: + return f.read() setup( - name='pytest-reportportal', - version=version, - description='Agent for Reporting results of tests to the Report Portal', - long_description=read_file('README.rst') + '\n\n', - author='Pavel Papou', - author_email='SupportEPMC-TSTReportPortal@epam.com', - url='https://github.com/reportportal/agent-python-pytest', - download_url=tar_url, - packages=['pytest_reportportal'], - install_requires=requirements, - license='GNU General Public License v3', - keywords=['testing', 'reporting', 'reportportal', 'pytest'], + name="pytest-reportportal", + version=__version__, + description="Agent for Reporting results of tests to the Report Portal", + long_description=read_file("README.md"), + long_description_content_type="text/markdown", + author="Report Portal Team", + author_email="support@reportportal.io", + url="https://github.com/reportportal/agent-python-pytest", + packages=["pytest_reportportal"], + package_data={"pytest_reportportal": ["*.pyi"]}, + install_requires=read_file("requirements.txt").splitlines(), + license="Apache 2.0", + keywords=["testing", "reporting", "reportportal", "pytest", "agent"], classifiers=[ - 'Framework :: Pytest', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6' - ], + "Framework :: Pytest", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + ], entry_points={ - 'pytest11': [ - 'pytest_reportportal = pytest_reportportal.plugin', + "pytest11": [ + "pytest_reportportal = pytest_reportportal.plugin", ] }, ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..cc2c4a7f --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This package contains tests for the project.""" + +REPORT_PORTAL_SERVICE = "reportportal_client.RPClient" diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..ec72f65b --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This package contains integration tests for the project.""" diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py new file mode 100644 index 00000000..f492da75 --- /dev/null +++ b/tests/helpers/utils.py @@ -0,0 +1,149 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module contains utility code for unit tests.""" + +import os +import random +import time +from multiprocessing.pool import ThreadPool + +import pytest + +DEFAULT_VARIABLES = { + "rp_launch": "Pytest", + "rp_endpoint": "http://localhost:8080", + "rp_project": "default_personal", + "rp_api_key": "test_api_key", + "rp_skip_connection_test": "True", +} + +DEFAULT_PROJECT_SETTINGS = { + "project": 2, + "subTypes": { + "NO_DEFECT": [ + { + "id": 4, + "locator": "nd001", + "typeRef": "NO_DEFECT", + "longName": "No Defect", + "shortName": "ND", + "color": "#777777", + } + ], + "TO_INVESTIGATE": [ + { + "id": 1, + "locator": "ti001", + "typeRef": "TO_INVESTIGATE", + "longName": "To Investigate", + "shortName": "TI", + "color": "#ffb743", + } + ], + "AUTOMATION_BUG": [ + { + "id": 2, + "locator": "ab001", + "typeRef": "AUTOMATION_BUG", + "longName": "Automation Bug", + "shortName": "AB", + "color": "#f7d63e", + } + ], + "PRODUCT_BUG": [ + { + "id": 3, + "locator": "pb001", + "typeRef": "PRODUCT_BUG", + "longName": "Product Bug", + "shortName": "PB", + "color": "#ec3900", + } + ], + "SYSTEM_ISSUE": [ + { + "id": 5, + "locator": "si001", + "typeRef": "SYSTEM_ISSUE", + "longName": "System Issue", + "shortName": "SI", + "color": "#0274d1", + } + ], + }, +} + + +def run_pytest_tests(tests, args=None, variables=None): + """Run specific pytest tests. + + :param tests: a list of tests to run + :param args: command line arguments which will be passed to pytest + :param variables: parameter variables which will be passed to pytest + :return: exit code + """ + if args is None: + args = [] + if variables is None: + variables = DEFAULT_VARIABLES + + arguments = ["--reportportal"] + args + for k, v in variables.items(): + arguments.append("-o") + arguments.append("{0}={1}".format(k, str(v))) + + if tests is not None: + for t in tests: + arguments.append(t) + + # Workaround collisions with parent test + current_test = os.environ["PYTEST_CURRENT_TEST"] + del os.environ["PYTEST_CURRENT_TEST"] + result = pytest.main(arguments) + os.environ["PYTEST_CURRENT_TEST"] = current_test + + return result + + +def item_id_gen(**kwargs): + return "{}-{}-{}".format(kwargs["name"], str(round(time.time() * 1000)), random.randint(0, 9999)) + + +def project_settings(**_): + return DEFAULT_PROJECT_SETTINGS + + +def attributes_to_tuples(attributes): + result = set() + for attribute in attributes: + if "key" in attribute: + result.add((attribute["key"], attribute["value"])) + else: + result.add((None, attribute["value"])) + return result + + +# noinspection PyProtectedMember +def run_tests_with_client(client, tests, args=None, variables=None): + def test_func(): + from reportportal_client import set_current + + set_current(client) + return run_pytest_tests(tests, args, variables) + + pool = ThreadPool(processes=1) + async_result = pool.apply_async(test_func) + result = async_result.get() + pool.terminate() + return result diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..3d59848e --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,336 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This package contains integration tests for the project.""" +from collections import defaultdict +from typing import Optional + +from reportportal_client import set_current +from reportportal_client.steps import StepReporter + +from tests.helpers import utils + +HIERARCHY_TESTS = ( + [ + ["examples/test_simple.py"], + ["examples/hierarchy/inner/test_inner_simple.py"], + ["examples/hierarchy/test_in_class.py"], + ["examples/hierarchy/test_in_class_in_class.py"], + [ + "examples/hierarchy/another_inner/test_another_inner_simple.py", + "examples/hierarchy/inner/test_inner_simple.py", + ], + ] + + [["examples/params/test_in_class_parameterized.py"]] * 5 + + [["examples/hierarchy/inner/test_inner_simple.py"]] * 7 + + [["examples/hierarchy/test_in_class_in_class.py"]] + + [["examples/test_simple.py"]] * 2 + + [["examples/hierarchy/inner/test_inner_simple.py"]] +) + +# noinspection PyTypeChecker +HIERARCHY_TEST_VARIABLES = [ + dict({"rp_hierarchy_dirs": True, "rp_hierarchy_code": True}, **utils.DEFAULT_VARIABLES) +] * 6 + [ + dict( + {"rp_hierarchy_dirs": True, "rp_hierarchy_code": True, "rp_hierarchy_dirs_level": 1}, **utils.DEFAULT_VARIABLES + ), + dict( + {"rp_hierarchy_dirs": True, "rp_hierarchy_code": True, "rp_hierarchy_dirs_level": 2}, **utils.DEFAULT_VARIABLES + ), + dict( + {"rp_hierarchy_dirs": True, "rp_hierarchy_code": True, "rp_hierarchy_dirs_level": 999}, + **utils.DEFAULT_VARIABLES, + ), + dict( + {"rp_hierarchy_dirs": True, "rp_hierarchy_code": True, "rp_hierarchy_dirs_level": -1}, + **utils.DEFAULT_VARIABLES, + ), + dict({"rp_hierarchy_dir_path_separator": "/", "rp_hierarchy_code": True}, **utils.DEFAULT_VARIABLES), + dict({"rp_hierarchy_dir_path_separator": "\\", "rp_hierarchy_code": True}, **utils.DEFAULT_VARIABLES), + dict({"rp_hierarchy_dirs_level": 1, "rp_hierarchy_code": True}, **utils.DEFAULT_VARIABLES), + dict({"rp_hierarchy_dirs_level": 2, "rp_hierarchy_code": True}, **utils.DEFAULT_VARIABLES), + dict({"rp_hierarchy_dirs_level": 999, "rp_hierarchy_code": True}, **utils.DEFAULT_VARIABLES), + dict({"rp_hierarchy_dirs_level": -1, "rp_hierarchy_code": True}, **utils.DEFAULT_VARIABLES), + dict(**utils.DEFAULT_VARIABLES), + dict(**utils.DEFAULT_VARIABLES), + dict({"rp_hierarchy_test_file": False}, **utils.DEFAULT_VARIABLES), + dict({"rp_hierarchy_test_file": False, "rp_hierarchy_dirs_level": 1}, **utils.DEFAULT_VARIABLES), + dict( + {"rp_hierarchy_dirs": True, "rp_hierarchy_test_file": True, "rp_hierarchy_code": False}, + **utils.DEFAULT_VARIABLES, + ), +] + +HIERARCHY_TEST_EXPECTED_ITEMS = [ + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "test_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + {"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("test_simple.py")}, + ], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "hierarchy", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + {"name": "inner", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("hierarchy")}, + {"name": "test_inner_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("inner")}, + {"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("test_inner_simple.py")}, + ], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "hierarchy", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + {"name": "test_in_class.py", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("hierarchy")}, + {"name": "Tests", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("test_in_class.py")}, + {"name": "test_in_class", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("Tests")}, + ], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "hierarchy", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + { + "name": "test_in_class_in_class.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("hierarchy"), + }, + {"name": "Tests", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("test_in_class_in_class.py")}, + {"name": "Test", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("Tests")}, + {"name": "test_in_class_in_class", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("Test")}, + ], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "hierarchy", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + {"name": "another_inner", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("hierarchy")}, + { + "name": "test_another_inner_simple.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("another_inner"), + }, + { + "name": "test_simple", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("test_another_inner_simple.py"), + }, + {"name": "inner", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("hierarchy")}, + {"name": "test_inner_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("inner")}, + {"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("test_inner_simple.py")}, + ], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "params", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + { + "name": "test_in_class_parameterized.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("params"), + }, + { + "name": "Tests", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("test_in_class_parameterized.py"), + }, + { + "name": "test_in_class_parameterized[param]", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("Tests"), + }, + ], + [ + {"name": "params", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + { + "name": "test_in_class_parameterized.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("params"), + }, + { + "name": "Tests", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("test_in_class_parameterized.py"), + }, + { + "name": "test_in_class_parameterized[param]", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("Tests"), + }, + ], + [ + {"name": "test_in_class_parameterized.py", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + { + "name": "Tests", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("test_in_class_parameterized.py"), + }, + { + "name": "test_in_class_parameterized[param]", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("Tests"), + }, + ], + [ + {"name": "test_in_class_parameterized.py", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + { + "name": "Tests", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("test_in_class_parameterized.py"), + }, + { + "name": "test_in_class_parameterized[param]", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("Tests"), + }, + ], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "params", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + { + "name": "test_in_class_parameterized.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("params"), + }, + { + "name": "Tests", + "item_type": "SUITE", + "parent_item_id": lambda x: x.startswith("test_in_class_parameterized.py"), + }, + { + "name": "test_in_class_parameterized[param]", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("Tests"), + }, + ], + [ + { + "name": "examples/hierarchy/inner/test_inner_simple.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x is None, + }, + { + "name": "test_simple", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("examples/hierarchy/inner/test_inner_simple.py"), + }, + ], + [ + { + "name": "examples\\hierarchy\\inner\\test_inner_simple.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x is None, + }, + { + "name": "test_simple", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("examples\\hierarchy\\inner\\test_inner_simple.py"), + }, + ], + [ + {"name": "hierarchy/inner/test_inner_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + { + "name": "test_simple", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("hierarchy/inner/test_inner_simple.py"), + }, + ], + [ + {"name": "inner/test_inner_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + { + "name": "test_simple", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("inner/test_inner_simple.py"), + }, + ], + [ + {"name": "test_inner_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("test_inner_simple.py")}, + ], + [ + { + "name": "examples/hierarchy/inner/test_inner_simple.py", + "item_type": "SUITE", + "parent_item_id": lambda x: x is None, + }, + { + "name": "test_simple", + "item_type": "STEP", + "parent_item_id": lambda x: x.startswith("examples/hierarchy/inner/test_inner_simple.py"), + }, + ], + [ + { + "name": "examples/hierarchy/inner/test_inner_simple.py::test_simple", + "item_type": "STEP", + "parent_item_id": lambda x: x is None, + } + ], + [ + { + "name": "examples/hierarchy/test_in_class_in_class.py::Tests::Test::test_in_class_in_class", + "item_type": "STEP", + "parent_item_id": lambda x: x is None, + } + ], + [{"name": "examples::test_simple", "item_type": "STEP", "parent_item_id": lambda x: x is None}], + [{"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x is None}], + [ + {"name": "examples", "item_type": "SUITE", "parent_item_id": lambda x: x is None}, + {"name": "hierarchy", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("examples")}, + {"name": "inner", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("hierarchy")}, + {"name": "test_inner_simple.py", "item_type": "SUITE", "parent_item_id": lambda x: x.startswith("inner")}, + {"name": "test_simple", "item_type": "STEP", "parent_item_id": lambda x: x.startswith("test_inner_simple.py")}, + ], +] + +HIERARCHY_TEST_PARAMETERS = [ + (test, HIERARCHY_TEST_VARIABLES[idx], HIERARCHY_TEST_EXPECTED_ITEMS[idx]) + for idx, test in enumerate(HIERARCHY_TESTS) +] + +ITEM_ID_DICT = defaultdict(lambda: 0) +ITEM_ID_LIST = [] + + +def generate_item_id(*args, **kwargs) -> str: + global ITEM_ID_DICT + global ITEM_ID_LIST + if args: + name = args[0] + else: + name = kwargs["name"] + count = ITEM_ID_DICT[name] + count += 1 + ITEM_ID_DICT[name] = count + item_id = f"{name}_{count}" + ITEM_ID_LIST.append(item_id) + return item_id + + +def get_last_item_id() -> Optional[str]: + global ITEM_ID_LIST + if len(ITEM_ID_LIST) > 0: + return ITEM_ID_LIST[-1] + + +def remove_last_item_id(*_, **__) -> Optional[str]: + global ITEM_ID_LIST + if len(ITEM_ID_LIST) > 0: + return ITEM_ID_LIST.pop() + + +def setup_mock(mock_client_init): + mock_client = mock_client_init.return_value + mock_client.step_reporter = StepReporter(mock_client) + set_current(mock_client) + return mock_client + + +def setup_mock_for_logging(mock_client_init): + mock_client = setup_mock(mock_client_init) + mock_client.start_test_item.side_effect = generate_item_id + mock_client.finish_test_item.side_effect = remove_last_item_id + mock_client.current_item.side_effect = get_last_item_id + return mock_client diff --git a/tests/integration/test_attributes.py b/tests/integration/test_attributes.py new file mode 100644 index 00000000..cff940c6 --- /dev/null +++ b/tests/integration/test_attributes.py @@ -0,0 +1,207 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for different attribute reporting.""" + +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_custom_attribute_report(mock_client_init): + """Verify custom attribute is reported. + + :param mock_client_init: Pytest fixture + """ + variables = {"markers": "scope: to which test scope a test relates"} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/attributes/test_one_attribute.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + assert step_call_args["attributes"] == [{"key": "scope", "value": "smoke"}] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_custom_attribute_not_reported_if_skip_configured(mock_client_init): + """Verify custom attribute is not reported if it's configured as skipped. + + :param mock_client_init: Pytest fixture + """ + variables = {"markers": "scope: to which test scope a test relates", "rp_ignore_attributes": "scope"} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/attributes/test_one_attribute.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + assert step_call_args["attributes"] == [] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_two_attributes_different_values_report(mock_client_init): + """Verify two attributes with different values is reported. + + :param mock_client_init: Pytest fixture + """ + variables = {"markers": "scope: to which test scope a test relates"} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests( + tests=["examples/attributes/test_two_attributes_with_same_key.py"], variables=variables + ) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + actual_attributes = step_call_args["attributes"] + + assert utils.attributes_to_tuples(actual_attributes) == {("scope", "smoke"), ("scope", "regression")} + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_skip_attribute(mock_client_init): + """Skip attribute is reported as tag. + + :param mock_client_init: Pytest fixture + """ + result = utils.run_pytest_tests(tests=["examples/skip/test_simple_skip.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + actual_attributes = step_call_args["attributes"] + + assert utils.attributes_to_tuples(actual_attributes) == {(None, "skip")} + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_custom_runtime_attribute_report(mock_client_init): + """Verify custom attribute is reported. + + :param mock_client_init: Pytest fixture + """ + variables = {"markers": "scope: to which test scope a test relates\nruntime: runtime attribute mark"} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/attributes/test_runtime_attribute.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + assert mock_client.finish_test_item.call_count > 0, '"finish_test_item" called incorrect number of times' + + start_call_args = mock_client.start_test_item.call_args_list + start_step_call_args = start_call_args[-1][1] + assert start_step_call_args["attributes"] == [{"key": "scope", "value": "smoke"}] + + finish_call_args = mock_client.finish_test_item.call_args_list + finish_step_call_args = finish_call_args[-1][1] + actual_attributes = finish_step_call_args["attributes"] + attribute_tuple_list = [(kv.get("key"), kv["value"]) for kv in actual_attributes] + + assert set(attribute_tuple_list) == {("scope", "smoke"), (None, "runtime")} + + +@pytest.mark.parametrize("rp_hierarchy_code", [True, False]) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_tests_attributes(mock_client_init, rp_hierarchy_code): + """Verify configuration attributes are reported. + + :param mock_client_init: Pytest fixture + """ + variables = {"rp_tests_attributes": "test_key:test_value", "rp_hierarchy_code": rp_hierarchy_code} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_simple.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + assert step_call_args["attributes"] == [{"key": "test_key", "value": "test_value"}] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_tests_attributes_add(mock_client_init): + """Verify configuration attributes are reported along with custom attribute. + + :param mock_client_init: Pytest fixture + """ + variables = {"markers": "scope: to which test scope a test relates", "rp_tests_attributes": "test_key:test_value"} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/attributes/test_one_attribute.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + attributes = step_call_args["attributes"] + assert len(attributes) == 2 + assert {"key": "scope", "value": "smoke"} in attributes + assert {"key": "test_key", "value": "test_value"} in attributes + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_tests_attributes_string_split_and_deduplicated(mock_client_init, monkeypatch): + """Verify string `rp_tests_attributes` are split and deduplicated.""" + monkeypatch.setenv("RP_TESTS_ATTRIBUTES", " test_key:test_value ; smoke ; test_key:test_value ") + variables = {} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_simple.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + actual_attributes = step_call_args["attributes"] + + assert utils.attributes_to_tuples(actual_attributes) == {("test_key", "test_value"), (None, "smoke")} + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_launch_attributes_string_split_and_deduplicated(mock_client_init, monkeypatch): + """Verify string `rp_launch_attributes` are split and deduplicated.""" + monkeypatch.setenv("RP_LAUNCH_ATTRIBUTES", " launch_key:launch_value ; smoke ; launch_key:launch_value ") + variables = {} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_simple.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_launch.call_count > 0, '"start_launch" called incorrect number of times' + + launch_call_args = mock_client.start_launch.call_args_list + launch_attributes = launch_call_args[0][1]["attributes"] + + assert {("launch_key", "launch_value"), (None, "smoke")} <= utils.attributes_to_tuples(launch_attributes) diff --git a/tests/integration/test_bdd.py b/tests/integration/test_bdd.py new file mode 100644 index 00000000..1b10c5b1 --- /dev/null +++ b/tests/integration/test_bdd.py @@ -0,0 +1,1083 @@ +# Copyright 2025 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils +from tests.integration import setup_mock, setup_mock_for_logging + +pytest_bdd_version = [int(p) for p in importlib.metadata.version("pytest-bdd").split(".")] + +STEP_NAMES = [ + "Given there are 5 cucumbers", + "When I eat 3 cucumbers", + "And I eat 2 cucumbers", + "Then I should have 0 cucumbers", +] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_basic(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/test_arguments.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + assert mock_client.start_test_item.call_count == 5, 'There should be exactly five "start_test_item" calls' + assert ( + mock_client.start_test_item.call_count == mock_client.finish_test_item.call_count + ), '"start_test_item" and "finish_test_item" should be called the same number of times' + + scenario_call = mock_client.start_test_item.call_args_list[0] + code_ref = "features/arguments_four_steps.feature/[SCENARIO:Arguments for given, when, and, then]" + assert scenario_call[1]["item_type"] == "STEP", "First call should be a Scenario" + assert scenario_call[1].get("has_stats", True) is True, "First call should have stats" + assert scenario_call[1]["code_ref"] == code_ref + assert scenario_call[1]["test_case_id"] == code_ref + assert scenario_call[1]["name"] == "Feature: Four step arguments - Scenario: Arguments for given, when, and, then" + assert scenario_call[1]["parent_item_id"] is None + assert scenario_call[1]["parameters"] is None + assert scenario_call[1]["description"] is None + + step_calls = mock_client.start_test_item.call_args_list[1:] + for i, call in enumerate(step_calls): + assert call[0][0] == STEP_NAMES[i] + assert call[0][2] == "step", "All other calls should be Steps" + assert call[1]["has_stats"] is False, "All other calls should not have stats" + + finish_calls = mock_client.finish_test_item.call_args_list + finish_step_calls = finish_calls[:-1] + for i, call in enumerate(finish_step_calls): + assert call[0][0] == f"{STEP_NAMES[i]}_1" + assert call[1]["status"] == "PASSED" + finish_scenario_call = finish_calls[-1] + assert finish_scenario_call[1]["status"] == "PASSED" + assert ( + finish_scenario_call[1]["item_id"] + == "Feature: Four step arguments - Scenario: Arguments for given, when, and, then_1" + ) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_basic_with_feature_suite(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + variables = {"rp_hierarchy_code": True} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/test_arguments.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + assert mock_client.start_test_item.call_count == 6, 'There should be exactly six "start_test_item" calls' + assert ( + mock_client.start_test_item.call_count == mock_client.finish_test_item.call_count + ), '"start_test_item" and "finish_test_item" should be called the same number of times' + + suite_call = mock_client.start_test_item.call_args_list[0] + assert suite_call[1]["item_type"] == "SUITE", "First call should be a Feature" + assert suite_call[1].get("has_stats", True) is True, "First call should have stats" + assert suite_call[1]["parent_item_id"] is None + assert suite_call[1]["name"] == "Feature: Four step arguments" + + scenario_call = mock_client.start_test_item.call_args_list[1] + code_ref = "features/arguments_four_steps.feature/[SCENARIO:Arguments for given, when, and, then]" + assert scenario_call[1]["item_type"] == "STEP", "First call should be a Scenario" + assert scenario_call[1].get("has_stats", True) is True, "First call should have stats" + assert scenario_call[1]["code_ref"] == code_ref + assert scenario_call[1]["test_case_id"] == code_ref + assert scenario_call[1]["name"] == "Scenario: Arguments for given, when, and, then" + assert scenario_call[1]["parent_item_id"] == "Feature: Four step arguments_1" + assert scenario_call[1]["parameters"] is None + assert scenario_call[1]["description"] is None + + step_calls = mock_client.start_test_item.call_args_list[2:] + for call in step_calls: + assert call[0][2] == "step", "All other calls should be Steps" + assert call[1]["has_stats"] is False, "All other calls should not have stats" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_descriptions(mock_client_init): + mock_client = setup_mock(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/test_arguments_description.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + code_ref = "features/arguments_four_steps_description.feature/[SCENARIO:Arguments for given, when, and, then]" + scenario_call = mock_client.start_test_item.call_args_list[0] + assert scenario_call[1]["code_ref"] == code_ref + assert scenario_call[1]["test_case_id"] == code_ref + description = scenario_call[1]["description"] + assert description == "Description for the scenario" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_feature_descriptions(mock_client_init): + mock_client = setup_mock(mock_client_init) + variables = {"rp_hierarchy_code": True} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests( + tests=["examples/bdd/step_defs/test_arguments_description.py"], variables=variables + ) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + feature_call = mock_client.start_test_item.call_args_list[0] + assert feature_call[1]["description"] == "Description for the feature" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_failed_feature(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/test_failed_step.py"]) + assert int(result) == 1, "Exit code should be 1 (test error)" + + assert mock_client.start_test_item.call_count == 2, 'There should be exactly two "start_test_item" calls' + assert ( + mock_client.start_test_item.call_count == mock_client.finish_test_item.call_count + ), '"start_test_item" and "finish_test_item" should be called the same number of times' + + finish_calls = mock_client.finish_test_item.call_args_list + finish_step_call = finish_calls[0] + finish_scenario_call = finish_calls[1] + + assert finish_step_call[0][0] == "Given I have a failed step_1" + assert finish_step_call[1]["status"] == "FAILED" + assert finish_scenario_call[1]["item_id"] == "Feature: Test failed scenario - Scenario: The scenario_1" + assert finish_scenario_call[1]["status"] == "FAILED" + + log_count = mock_client.log.call_count + # 1 - debug log from pytest-bdd's scenario module; 2 - traceback log from the agent; 3 - error log from pytest + assert log_count == 1 + 1 + 1, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list[1:] + assert log_call_args_list[0][1]["level"] == "ERROR" + assert log_call_args_list[0][1]["message"].endswith("AssertionError: assert False\n") + assert log_call_args_list[0][1]["item_id"] == "Given I have a failed step_1" + + assert log_call_args_list[1][1]["level"] == "ERROR" + assert log_call_args_list[1][1]["message"].endswith("AssertionError") + assert log_call_args_list[1][1]["item_id"] == "Feature: Test failed scenario - Scenario: The scenario_1" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_attributes(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_file = "examples/bdd/step_defs/test_belly.py" + result = utils.run_pytest_tests(tests=[test_file]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + scenario_call = mock_client.start_test_item.call_args_list[0] + scenario_attrs = scenario_call[1].get("attributes", None) + assert scenario_attrs is not None + assert len(scenario_attrs) == 2 + assert {"value": "ok"} in scenario_attrs + assert {"key": "key", "value": "value"} in scenario_attrs + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_feature_attributes(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + variables = {"rp_hierarchy_code": True} + variables.update(utils.DEFAULT_VARIABLES.items()) + test_file = "examples/bdd/step_defs/test_belly.py" + result = utils.run_pytest_tests(tests=[test_file], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + feature_call = mock_client.start_test_item.call_args_list[0] + feature_attrs = feature_call[1].get("attributes", []) + assert feature_attrs is not None + assert len(feature_attrs) == 3 + assert {"value": "smoke"} in feature_attrs + assert {"value": "test"} in feature_attrs + assert {"key": "feature", "value": "belly"} in feature_attrs + + scenario_call = mock_client.start_test_item.call_args_list[1] + scenario_attrs = scenario_call[1].get("attributes", []) + assert scenario_attrs is not None + assert len(scenario_attrs) == 2 + assert {"value": "ok"} in scenario_attrs + assert {"key": "key", "value": "value"} in scenario_attrs + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_background_step(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_file = "examples/bdd/step_defs/test_background.py" + result = utils.run_pytest_tests(tests=[test_file]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify the first scenario + scenario_call_1 = mock_client.start_test_item.call_args_list[0] + assert scenario_call_1[1]["name"] == "Feature: Test scenario with a background - Scenario: The first scenario" + assert scenario_call_1[1]["item_type"] == "STEP" + assert scenario_call_1[1].get("has_stats", True) + + # Verify the Background step for the first scenario + background_call_1 = mock_client.start_test_item.call_args_list[1] + assert background_call_1[0][0] == "Background" + assert background_call_1[0][2] == "step" + assert background_call_1[1]["has_stats"] is False + assert background_call_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + + # Verify the nested steps within the Background for the first scenario + nested_step_call_1 = mock_client.start_test_item.call_args_list[2] + assert nested_step_call_1[0][0] == "Given I have empty step" + assert nested_step_call_1[0][2] == "step" + assert nested_step_call_1[1]["parent_item_id"] == background_call_1[0][0] + "_1" + assert nested_step_call_1[1]["has_stats"] is False + + # Verify the step within the first scenario + scenario_step_call_1 = mock_client.start_test_item.call_args_list[3] + assert scenario_step_call_1[0][0] == "Then I have another empty step" + assert scenario_step_call_1[0][2] == "step" + assert scenario_step_call_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + assert scenario_step_call_1[1]["has_stats"] is False + + # Verify the second scenario + scenario_call_2 = mock_client.start_test_item.call_args_list[4] + assert scenario_call_2[1]["name"] == "Feature: Test scenario with a background - Scenario: The second scenario" + assert scenario_call_2[1]["item_type"] == "STEP" + assert scenario_call_1[1].get("has_stats", True) + + # Verify the Background step for the second scenario + background_call_2 = mock_client.start_test_item.call_args_list[5] + assert background_call_2[0][0] == "Background" + assert background_call_2[0][2] == "step" + assert background_call_2[1]["has_stats"] is False + assert background_call_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_1" + + # Verify the nested steps within the Background for the second scenario + nested_step_call_2 = mock_client.start_test_item.call_args_list[6] + assert nested_step_call_2[0][0] == "Given I have empty step" + assert nested_step_call_2[0][2] == "step" + assert nested_step_call_2[1]["parent_item_id"] == background_call_2[0][0] + "_2" + assert nested_step_call_2[1]["has_stats"] is False + + # Verify the step within the second scenario + scenario_step_call_2 = mock_client.start_test_item.call_args_list[7] + assert scenario_step_call_2[0][0] == "Then I have one more empty step" + assert scenario_step_call_2[0][2] == "step" + assert scenario_step_call_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_1" + assert scenario_step_call_2[1]["has_stats"] is False + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_background_two_steps(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_file = "examples/bdd/step_defs/test_background_two_steps.py" + result = utils.run_pytest_tests(tests=[test_file]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify the scenario + scenario_call = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call[1]["name"] == "Feature: Test scenario with a background with two steps - Scenario: The scenario" + ) + assert scenario_call[1]["item_type"] == "STEP" + assert scenario_call[1].get("has_stats", True) + + # Verify the Background step + background_call = mock_client.start_test_item.call_args_list[1] + assert background_call[0][0] == "Background" + assert background_call[0][2] == "step" + assert background_call[1]["has_stats"] is False + assert background_call[1]["parent_item_id"] == scenario_call[1]["name"] + "_1" + + # Verify the first nested step within the Background + nested_step_call_1 = mock_client.start_test_item.call_args_list[2] + assert nested_step_call_1[0][0] == "Given I have first empty step" + assert nested_step_call_1[0][2] == "step" + assert nested_step_call_1[1]["parent_item_id"] == background_call[0][0] + "_3" + assert nested_step_call_1[1]["has_stats"] is False + + # Verify the second nested step within the Background + nested_step_call_2 = mock_client.start_test_item.call_args_list[3] + assert nested_step_call_2[0][0] == "And I have second empty step" + assert nested_step_call_2[0][2] == "step" + assert nested_step_call_2[1]["parent_item_id"] == background_call[0][0] + "_3" + assert nested_step_call_2[1]["has_stats"] is False + + # Verify the scenario step + scenario_step_call = mock_client.start_test_item.call_args_list[4] + assert scenario_step_call[0][0] == "Then I have main step" + assert scenario_step_call[0][2] == "step" + assert scenario_step_call[1]["parent_item_id"] == scenario_call[1]["name"] + "_1" + assert scenario_step_call[1]["has_stats"] is False + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rule(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/test_rule_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify first scenario from first rule + scenario_1_call = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_1_call[1]["name"] + == "Feature: Test rule keyword - Rule: The first rule - Scenario: The first scenario" + ) + assert scenario_1_call[1]["item_type"] == "STEP" + assert scenario_1_call[1].get("has_stats", True) is True + assert scenario_1_call[1]["parent_item_id"] is None + assert ( + scenario_1_call[1]["code_ref"] + == "features/rule_keyword.feature/[RULE:The first rule]/[SCENARIO:The first scenario]" + ) + + # Verify first scenario steps + step_1_given = mock_client.start_test_item.call_args_list[1] + assert step_1_given[0][0] == "Given I have empty step" + assert step_1_given[0][2] == "step" + assert step_1_given[1]["parent_item_id"] == scenario_1_call[1]["name"] + "_1" + assert step_1_given[1]["has_stats"] is False + + step_1_then = mock_client.start_test_item.call_args_list[2] + assert step_1_then[0][0] == "Then I have another empty step" + assert step_1_then[0][2] == "step" + assert step_1_then[1]["parent_item_id"] == scenario_1_call[1]["name"] + "_1" + assert step_1_then[1]["has_stats"] is False + + # Verify second scenario from first rule + scenario_2_call = mock_client.start_test_item.call_args_list[3] + assert ( + scenario_2_call[1]["name"] + == "Feature: Test rule keyword - Rule: The first rule - Scenario: The second scenario" + ) + assert scenario_2_call[1]["item_type"] == "STEP" + assert scenario_2_call[1].get("has_stats", True) is True + assert scenario_2_call[1]["parent_item_id"] is None + assert ( + scenario_2_call[1]["code_ref"] + == "features/rule_keyword.feature/[RULE:The first rule]/[SCENARIO:The second scenario]" + ) + + # Verify second scenario steps + step_2_given = mock_client.start_test_item.call_args_list[4] + assert step_2_given[0][0] == "Given I have empty step" + assert step_2_given[0][2] == "step" + assert step_2_given[1]["parent_item_id"] == scenario_2_call[1]["name"] + "_1" + assert step_2_given[1]["has_stats"] is False + + step_2_then = mock_client.start_test_item.call_args_list[5] + assert step_2_then[0][0] == "Then I have one more empty step" + assert step_2_then[0][2] == "step" + assert step_2_then[1]["parent_item_id"] == scenario_2_call[1]["name"] + "_1" + assert step_2_then[1]["has_stats"] is False + + # Verify third scenario from second rule + scenario_3_call = mock_client.start_test_item.call_args_list[6] + assert ( + scenario_3_call[1]["name"] + == "Feature: Test rule keyword - Rule: The second rule - Scenario: The third scenario" + ) + assert scenario_3_call[1]["item_type"] == "STEP" + assert scenario_3_call[1].get("has_stats", True) is True + assert scenario_3_call[1]["parent_item_id"] is None + assert ( + scenario_3_call[1]["code_ref"] + == "features/rule_keyword.feature/[RULE:The second rule]/[SCENARIO:The third scenario]" + ) + + # Verify third scenario steps + step_3_given = mock_client.start_test_item.call_args_list[7] + assert step_3_given[0][0] == "Given I have empty step" + assert step_3_given[0][2] == "step" + assert step_3_given[1]["parent_item_id"] == scenario_3_call[1]["name"] + "_1" + assert step_3_given[1]["has_stats"] is False + + step_3_then = mock_client.start_test_item.call_args_list[8] + assert step_3_then[0][0] == "Then I have one more else empty step" + assert step_3_then[0][2] == "step" + assert step_3_then[1]["parent_item_id"] == scenario_3_call[1]["name"] + "_1" + assert step_3_then[1]["has_stats"] is False + + # Verify all steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rule_hierarchy(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + variables = {"rp_hierarchy_code": True} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/test_rule_steps.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify Feature + feature_call = mock_client.start_test_item.call_args_list[0] + assert feature_call[1]["name"] == "Feature: Test rule keyword" + assert feature_call[1]["item_type"] == "SUITE" + assert feature_call[1].get("has_stats", True) is True + assert feature_call[1]["parent_item_id"] is None + feature_id = "Feature: Test rule keyword_1" + + # Verify first Rule + rule_1_call = mock_client.start_test_item.call_args_list[1] + assert rule_1_call[1]["name"] == "Rule: The first rule" + assert rule_1_call[1]["item_type"] == "SUITE" + assert rule_1_call[1].get("has_stats", True) is True + assert rule_1_call[1]["parent_item_id"] == feature_id + rule_1_id = "Rule: The first rule_1" + + # Verify first scenario under first rule + scenario_1_call = mock_client.start_test_item.call_args_list[2] + assert scenario_1_call[1]["name"] == "Scenario: The first scenario" + assert scenario_1_call[1]["item_type"] == "STEP" + assert scenario_1_call[1].get("has_stats", True) is True + assert scenario_1_call[1]["parent_item_id"] == rule_1_id + assert ( + scenario_1_call[1]["code_ref"] + == "features/rule_keyword.feature/[RULE:The first rule]/[SCENARIO:The first scenario]" + ) + + # Verify second scenario under first rule + scenario_2_call = mock_client.start_test_item.call_args_list[5] + assert scenario_2_call[1]["name"] == "Scenario: The second scenario" + assert scenario_2_call[1]["item_type"] == "STEP" + assert scenario_2_call[1].get("has_stats", True) is True + assert scenario_2_call[1]["parent_item_id"] == rule_1_id + assert ( + scenario_2_call[1]["code_ref"] + == "features/rule_keyword.feature/[RULE:The first rule]/[SCENARIO:The second scenario]" + ) + + # Verify second Rule + rule_2_call = mock_client.start_test_item.call_args_list[8] + assert rule_2_call[1]["name"] == "Rule: The second rule" + assert rule_2_call[1]["item_type"] == "SUITE" + assert rule_2_call[1].get("has_stats", True) is True + assert rule_2_call[1]["parent_item_id"] == feature_id + rule_2_id = "Rule: The second rule_1" + + # Verify third scenario under second rule + scenario_3_call = mock_client.start_test_item.call_args_list[9] + assert scenario_3_call[1]["name"] == "Scenario: The third scenario" + assert scenario_3_call[1]["item_type"] == "STEP" + assert scenario_3_call[1].get("has_stats", True) is True + assert scenario_3_call[1]["parent_item_id"] == rule_2_id + assert ( + scenario_3_call[1]["code_ref"] + == "features/rule_keyword.feature/[RULE:The second rule]/[SCENARIO:The third scenario]" + ) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_outline_parameters(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/scenario_outline_parameters_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify first scenario with parameters + scenario_call_1 = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call_1[1]["name"] + == "Feature: Basic test with parameters - Scenario Outline: Test with different parameters" + ) + assert scenario_call_1[1]["item_type"] == "STEP" + assert scenario_call_1[1].get("has_stats", True) + assert ( + scenario_call_1[1]["code_ref"] + == "features/scenario_outline_parameters.feature/[EXAMPLE:Test with different parameters" + '[parameters:123;str:"first"]]' + ) + parameters = scenario_call_1[1]["parameters"].items() + assert len(parameters) == 2 + assert ("str", '"first"') in parameters + assert ("parameters", "123") in parameters + assert scenario_call_1[1]["description"] is not None + assert scenario_call_1[1]["description"].endswith('|\xa0"first"\xa0|\xa0\xa0\xa0\xa0123\xa0\xa0\xa0\xa0\xa0|') + + # Verify steps for first scenario + given_step_1 = mock_client.start_test_item.call_args_list[1] + assert given_step_1[0][0] == "Given It is test with parameters" + assert given_step_1[0][2] == "step" + assert given_step_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + assert given_step_1[1]["has_stats"] is False + + when_step_1 = mock_client.start_test_item.call_args_list[2] + assert when_step_1[0][0] == 'When I have parameter "first"' + assert when_step_1[0][2] == "step" + assert when_step_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + assert when_step_1[1]["has_stats"] is False + + then_step_1 = mock_client.start_test_item.call_args_list[3] + assert then_step_1[0][0] == "Then I emit number 123 on level info" + assert then_step_1[0][2] == "step" + assert then_step_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + assert then_step_1[1]["has_stats"] is False + + # Verify second scenario with parameters + scenario_call_2 = mock_client.start_test_item.call_args_list[4] + assert ( + scenario_call_2[1]["name"] + == "Feature: Basic test with parameters - Scenario Outline: Test with different parameters" + ) + assert scenario_call_2[1]["item_type"] == "STEP" + assert scenario_call_2[1].get("has_stats", True) + assert ( + scenario_call_2[1]["code_ref"] + == "features/scenario_outline_parameters.feature/[EXAMPLE:Test with different parameters" + '[parameters:12345;str:"second"]]' + ) + parameters = scenario_call_2[1]["parameters"].items() + assert len(parameters) == 2 + assert ("str", '"second"') in parameters + assert ("parameters", "12345") in parameters + assert scenario_call_2[1]["description"] is not None + assert scenario_call_2[1]["description"].endswith('|\xa0"second"\xa0|\xa0\xa0\xa012345\xa0\xa0\xa0\xa0|') + + # Verify steps for second scenario + given_step_2 = mock_client.start_test_item.call_args_list[5] + assert given_step_2[0][0] == "Given It is test with parameters" + assert given_step_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_2" + assert given_step_2[1]["has_stats"] is False + + when_step_2 = mock_client.start_test_item.call_args_list[6] + assert when_step_2[0][0] == 'When I have parameter "second"' + assert when_step_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_2" + assert when_step_2[1]["has_stats"] is False + + then_step_2 = mock_client.start_test_item.call_args_list[7] + assert then_step_2[0][0] == "Then I emit number 12345 on level info" + assert then_step_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_2" + assert then_step_2[1]["has_stats"] is False + + # Verify third scenario with parameters + scenario_call_3 = mock_client.start_test_item.call_args_list[8] + assert ( + scenario_call_3[1]["name"] + == "Feature: Basic test with parameters - Scenario Outline: Test with different parameters" + ) + assert scenario_call_3[1]["item_type"] == "STEP" + assert scenario_call_3[1].get("has_stats", True) + assert ( + scenario_call_3[1]["code_ref"] + == "features/scenario_outline_parameters.feature/[EXAMPLE:Test with different parameters" + '[parameters:12345678;str:"third"]]' + ) + parameters = scenario_call_3[1]["parameters"].items() + assert len(parameters) == 2 + assert ("str", '"third"') in parameters + assert ("parameters", "12345678") in parameters + assert scenario_call_3[1]["description"] is not None + assert scenario_call_3[1]["description"].endswith('|\xa0"third"\xa0|\xa0\xa012345678\xa0\xa0|') + + # Verify steps for third scenario + given_step_3 = mock_client.start_test_item.call_args_list[9] + assert given_step_3[0][0] == "Given It is test with parameters" + assert given_step_3[1]["parent_item_id"] == scenario_call_3[1]["name"] + "_3" + assert given_step_3[1]["has_stats"] is False + + when_step_3 = mock_client.start_test_item.call_args_list[10] + assert when_step_3[0][0] == 'When I have parameter "third"' + assert when_step_3[1]["parent_item_id"] == scenario_call_3[1]["name"] + "_3" + assert when_step_3[1]["has_stats"] is False + + then_step_3 = mock_client.start_test_item.call_args_list[11] + assert then_step_3[0][0] == "Then I emit number 12345678 on level info" + assert then_step_3[1]["parent_item_id"] == scenario_call_3[1]["name"] + "_3" + assert then_step_3[1]["has_stats"] is False + + # Verify all steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@mock.patch(REPORT_PORTAL_SERVICE) +def test_examples_tags(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/example_tags_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify first scenario + scenario_call = mock_client.start_test_item.call_args_list[0] + scenario_attrs = scenario_call[1].get("attributes", None) + assert scenario_attrs is not None + assert len(scenario_attrs) == 1 + assert {"value": "test"} in scenario_attrs + + # Verify second scenario + scenario_call = mock_client.start_test_item.call_args_list[4] + scenario_attrs = scenario_call[1].get("attributes", None) + assert scenario_attrs is not None + assert len(scenario_attrs) == 1 + assert {"value": "test"} in scenario_attrs + + # Verify third scenario + scenario_call = mock_client.start_test_item.call_args_list[8] + scenario_attrs = scenario_call[1].get("attributes", None) + assert scenario_attrs is not None + assert len(scenario_attrs) == 1 + assert {"value": "test"} in scenario_attrs + + # Verify all steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_outline_background_steps(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/scenario_outline_background_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify first scenario with background + scenario_call_1 = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call_1[1]["name"] + == "Feature: Basic test with parameters and background - Scenario Outline: Test with different parameters" + ) + assert scenario_call_1[1]["item_type"] == "STEP" + assert scenario_call_1[1].get("has_stats", True) + assert ( + scenario_call_1[1]["code_ref"] + == "features/scenario_outline_background.feature/[EXAMPLE:Test with different parameters" + '[parameters:123;str:"first"]]' + ) + parameters = scenario_call_1[1]["parameters"].items() + assert len(parameters) == 2 + assert ("str", '"first"') in parameters + assert ("parameters", "123") in parameters + + # Verify the Background step + background_call = mock_client.start_test_item.call_args_list[1] + assert background_call[0][0] == "Background" + assert background_call[0][2] == "step" + assert background_call[1]["has_stats"] is False + assert background_call[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + + # Verify background step for first scenario + background_step_1 = mock_client.start_test_item.call_args_list[2] + assert background_step_1[0][0] == "Given I have empty step in background" + assert background_step_1[1]["parent_item_id"] == background_call[0][0] + "_4" + assert background_step_1[1]["has_stats"] is False + + # Verify regular steps for first scenario + given_step_1 = mock_client.start_test_item.call_args_list[3] + assert given_step_1[0][0] == "Given It is test with parameters" + assert given_step_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + assert given_step_1[1]["has_stats"] is False + + when_step_1 = mock_client.start_test_item.call_args_list[4] + assert when_step_1[0][0] == 'When I have parameter "first"' + assert when_step_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + assert when_step_1[1]["has_stats"] is False + + then_step_1 = mock_client.start_test_item.call_args_list[5] + assert then_step_1[0][0] == "Then I emit number 123 on level info" + assert then_step_1[1]["parent_item_id"] == scenario_call_1[1]["name"] + "_1" + assert then_step_1[1]["has_stats"] is False + + # Verify second scenario with background + scenario_call_2 = mock_client.start_test_item.call_args_list[6] + assert ( + scenario_call_2[1]["name"] + == "Feature: Basic test with parameters and background - Scenario Outline: Test with different parameters" + ) + assert scenario_call_2[1]["item_type"] == "STEP" + assert scenario_call_2[1].get("has_stats", True) + assert ( + scenario_call_2[1]["code_ref"] + == "features/scenario_outline_background.feature/[EXAMPLE:Test with different parameters" + '[parameters:12345;str:"second"]]' + ) + parameters = scenario_call_2[1]["parameters"].items() + assert len(parameters) == 2 + assert ("str", '"second"') in parameters + assert ("parameters", "12345") in parameters + + # Verify the Background step + background_call = mock_client.start_test_item.call_args_list[7] + assert background_call[0][0] == "Background" + assert background_call[0][2] == "step" + assert background_call[1]["has_stats"] is False + assert background_call[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_2" + + # Verify background step for second scenario + background_step_2 = mock_client.start_test_item.call_args_list[8] + assert background_step_2[0][0] == "Given I have empty step in background" + assert background_step_2[1]["parent_item_id"] == background_call[0][0] + "_5" + assert background_step_2[1]["has_stats"] is False + + # Verify steps for second scenario + given_step_2 = mock_client.start_test_item.call_args_list[9] + assert given_step_2[0][0] == "Given It is test with parameters" + assert given_step_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_2" + assert given_step_2[1]["has_stats"] is False + + when_step_2 = mock_client.start_test_item.call_args_list[10] + assert when_step_2[0][0] == 'When I have parameter "second"' + assert when_step_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_2" + assert when_step_2[1]["has_stats"] is False + + then_step_2 = mock_client.start_test_item.call_args_list[11] + assert then_step_2[0][0] == "Then I emit number 12345 on level info" + assert then_step_2[1]["parent_item_id"] == scenario_call_2[1]["name"] + "_2" + assert then_step_2[1]["has_stats"] is False + + # Verify all steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_outline_description(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/scenario_outline_description_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify first scenario with description + scenario_call_1 = mock_client.start_test_item.call_args_list[0] + description_1 = scenario_call_1[1]["description"] + assert description_1.startswith("The description for the scenario outline") + assert "Parameters:\n\n" in description_1 + assert "|\xa0\xa0\xa0str\xa0\xa0\xa0|\xa0parameters\xa0|" in description_1 + assert '|\xa0"first"\xa0|\xa0\xa0\xa0\xa0123\xa0\xa0\xa0\xa0\xa0|' in description_1 + + scenario_call_2 = mock_client.start_test_item.call_args_list[4] + description_2 = scenario_call_2[1]["description"] + assert description_2.startswith("The description for the scenario outline") + assert "Parameters:\n\n" in description_2 + assert "|\xa0\xa0\xa0str\xa0\xa0\xa0\xa0|\xa0parameters\xa0|" in description_2 + assert '|\xa0"second"\xa0|\xa0\xa0\xa012345\xa0\xa0\xa0\xa0|' in description_2 + + # Verify the steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rule_description(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + variables = {"rp_hierarchy_code": True} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests( + tests=["examples/bdd/step_defs/test_rule_description_steps.py"], variables=variables + ) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify Feature call + feature_call = mock_client.start_test_item.call_args_list[0] + assert feature_call[1]["name"] == "Feature: Test rule keyword" + assert feature_call[1]["code_ref"].endswith("rule_description.feature") + + # Verify Rule call + rule_call = mock_client.start_test_item.call_args_list[1] + assert rule_call[1]["name"] == "Rule: The first rule" + assert rule_call[1]["description"] == "Description for the Rule" + assert rule_call[1]["item_type"] == "SUITE" + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_outline_dynamic_name(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/scenario_outline_name_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + scenario_call_1 = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call_1[1]["name"] + == 'Feature: Dynamic scenario outline names - Scenario Outline: Test with the parameter "first"' + ) + assert scenario_call_1[1]["item_type"] == "STEP" + assert ( + scenario_call_1[1]["code_ref"] == "features/dynamic_scenario_outline_names.feature/" + '[EXAMPLE:Test with the parameter "first"[parameters:123;str:"first"]]' + ) + assert scenario_call_1[1]["parameters"] == {"str": '"first"', "parameters": "123"} + assert scenario_call_1[1]["description"] == ( + "Parameters:\n\n" + "\xa0\xa0\xa0\xa0|\xa0\xa0\xa0str\xa0\xa0\xa0|\xa0parameters\xa0|\n" + "\xa0\xa0\xa0\xa0|---------|------------|\n" + '\xa0\xa0\xa0\xa0|\xa0"first"\xa0|\xa0\xa0\xa0\xa0123\xa0\xa0\xa0\xa0\xa0|' + ) + + scenario_call_2 = mock_client.start_test_item.call_args_list[4] + assert ( + scenario_call_2[1]["name"] + == 'Feature: Dynamic scenario outline names - Scenario Outline: Test with the parameter "second"' + ) + assert scenario_call_2[1]["item_type"] == "STEP" + assert ( + scenario_call_2[1]["code_ref"] == "features/dynamic_scenario_outline_names.feature/" + '[EXAMPLE:Test with the parameter "second"[parameters:12345;str:"second"]]' + ) + assert scenario_call_2[1]["parameters"] == {"str": '"second"', "parameters": "12345"} + assert scenario_call_2[1]["description"] == ( + "Parameters:\n\n" + "\xa0\xa0\xa0\xa0|\xa0\xa0\xa0str\xa0\xa0\xa0\xa0|\xa0parameters\xa0|\n" + "\xa0\xa0\xa0\xa0|----------|------------|\n" + '\xa0\xa0\xa0\xa0|\xa0"second"\xa0|\xa0\xa0\xa012345\xa0\xa0\xa0\xa0|' + ) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_outline_fail(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/scenario_outline_fail_steps.py"]) + assert int(result) == 1, "Exit code should be 1 (test error)" + + # Verify first scenario with parameters + scenario_call_1 = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call_1[1]["name"] + == "Feature: Basic test with parameters which fails - Scenario Outline: Test with different parameters failing" + ) + assert scenario_call_1[1]["item_type"] == "STEP" + assert scenario_call_1[1].get("has_stats", True) + assert ( + scenario_call_1[1]["code_ref"] + == "features/scenario_outline_fail.feature/[EXAMPLE:Test with different parameters failing" + '[parameters:123;str:"first"]]' + ) + + # Check failure logging for first scenario + finish_step_call_1 = mock_client.finish_test_item.call_args_list[3] + assert finish_step_call_1[1]["status"] == "FAILED" + assert finish_step_call_1[0][0].startswith("Then I fail") + + finish_scenario_call_1 = mock_client.finish_test_item.call_args_list[4] + assert finish_scenario_call_1[1]["status"] == "FAILED" + assert finish_scenario_call_1[1]["item_id"] == scenario_call_1[1]["name"] + "_1" + + log_calls = [ + log_call + for log_call in mock_client.log.call_args_list + if "level" in log_call[1] and log_call[1]["level"] == "ERROR" + ] + assert len(log_calls) >= 2, "Should have at least 2 error log calls" + + error_log = log_calls[0][1] + assert "AssertionError: This step always fails" in error_log["message"] + assert error_log["item_id"].startswith("Then I fail") + + final_error_log = log_calls[1][1] + assert final_error_log["level"] == "ERROR" + assert final_error_log["message"].endswith("AssertionError") + assert final_error_log["item_id"] == scenario_call_1[1]["name"] + "_1" + + # Verify first scenario with parameters + scenario_call_2 = mock_client.start_test_item.call_args_list[5] + assert ( + scenario_call_2[1]["name"] + == "Feature: Basic test with parameters which fails - Scenario Outline: Test with different parameters failing" + ) + assert scenario_call_2[1]["item_type"] == "STEP" + assert scenario_call_2[1].get("has_stats", True) + assert ( + scenario_call_2[1]["code_ref"] + == "features/scenario_outline_fail.feature/[EXAMPLE:Test with different parameters failing" + '[parameters:12345;str:"second"]]' + ) + + # Check failure logging for first scenario + finish_step_call_2 = mock_client.finish_test_item.call_args_list[5 + 3] + assert finish_step_call_2[1]["status"] == "FAILED" + assert finish_step_call_2[0][0].startswith("Then I fail") + + finish_scenario_call_2 = mock_client.finish_test_item.call_args_list[5 + 4] + assert finish_scenario_call_2[1]["status"] == "FAILED" + assert finish_scenario_call_2[1]["item_id"] == scenario_call_2[1]["name"] + "_2" + + error_log = log_calls[2][1] + assert "AssertionError: This step always fails" in error_log["message"] + assert error_log["item_id"].startswith("Then I fail") + + final_error_log = log_calls[3][1] + assert final_error_log["level"] == "ERROR" + assert final_error_log["message"].endswith("AssertionError") + assert final_error_log["item_id"] == scenario_call_1[1]["name"] + "_2" + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@mock.patch(REPORT_PORTAL_SERVICE) +def test_doc_string_parameters(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/doc_string_parameters_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify scenario + scenario_call = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call[1]["name"] + == "Feature: Basic test with a docstring parameter - Scenario: Test with a docstring parameter" + ) + assert scenario_call[1]["item_type"] == "STEP" + assert scenario_call[1].get("has_stats", True) + assert ( + scenario_call[1]["code_ref"] + == "features/doc_string_parameters.feature/[SCENARIO:Test with a docstring parameter]" + ) + + # Verify step + given_step = mock_client.start_test_item.call_args_list[1] + assert given_step[0][0] == "Given I have a docstring parameter:" + assert given_step[0][2] == "step" + assert given_step[1]["parent_item_id"] == scenario_call[1]["name"] + "_1" + assert given_step[1]["has_stats"] is False + + # Verify steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@mock.patch(REPORT_PORTAL_SERVICE) +def test_data_table_parameter_steps(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/data_table_parameter_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify scenario + scenario_call = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call[1]["name"] + == "Feature: A basic test with a Data Table parameter - Scenario: Test with Data Table" + ) + assert scenario_call[1]["item_type"] == "STEP" + assert scenario_call[1].get("has_stats", True) + assert scenario_call[1]["code_ref"] == "features/data_table_parameter.feature/[SCENARIO:Test with Data Table]" + + # Verify step + given_step = mock_client.start_test_item.call_args_list[1] + assert given_step[0][0] == "Given a step with a data table:" + assert given_step[0][2] == "step" + assert given_step[1]["parent_item_id"] == scenario_call[1]["name"] + "_1" + assert given_step[1]["has_stats"] is False + + # Verify steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_scenario_outline_test_case_id(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/scenario_outline_test_case_id_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify first scenario with parameters includes the test case ID + scenario_call = mock_client.start_test_item.call_args_list[0] + assert ( + scenario_call[1]["name"] + == "Feature: Basic test with parameters - Scenario Outline: Test with different parameters" + ) + assert scenario_call[1]["item_type"] == "STEP" + assert ( + scenario_call[1]["code_ref"] + == "features/scenario_outline_test_case_id.feature/[EXAMPLE:Test with different parameters" + '[parameters:123;str:"first"]]' + ) + assert scenario_call[1]["parameters"] == {"str": '"first"', "parameters": "123"} + + # Verify the test case ID is correctly reported using the tag instead of code_ref + assert scenario_call[1]["test_case_id"] == 'outline_tc_id[parameters:123;str:"first"]' + + # Verify all steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_custom_test_case_id(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/custom_test_case_id_steps.py"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + # Verify scenario includes the test case ID + scenario_call = mock_client.start_test_item.call_args_list[0] + assert scenario_call[1]["name"] == "Feature: Test dummy scenario - Scenario: The scenario" + assert scenario_call[1]["item_type"] == "STEP" + assert scenario_call[1]["code_ref"] == "features/custom_test_case_id.feature/[SCENARIO:The scenario]" + + # Verify the test case ID is correctly reported using the tag instead of code_ref + assert scenario_call[1]["test_case_id"] == "my_tc_id" + + # Verify step info + step_call = mock_client.start_test_item.call_args_list[1] + assert step_call[0][0] == "Given I have empty step" + assert step_call[0][2] == "step" + assert step_call[1]["parent_item_id"] == scenario_call[1]["name"] + "_1" + assert step_call[1]["has_stats"] is False + + # Verify all steps pass + finish_calls = mock_client.finish_test_item.call_args_list + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_tests_attributes_rule(mock_client_init): + mock_client = setup_mock(mock_client_init) + variables = {"rp_tests_attributes": "test_key:test_value"} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests( + tests=["examples/bdd/step_defs/test_rule_description_steps.py"], variables=variables + ) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + scenario_call = mock_client.start_test_item.call_args_list[0] + assert scenario_call[1]["attributes"] == [{"key": "test_key", "value": "test_value"}] + + +@pytest.mark.skipif(pytest_bdd_version[0] < 8, reason="Only for pytest-bdd 8+") +@pytest.mark.parametrize("rp_hierarchy_code, scenario_idx", [(True, 2), (False, 0)]) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_tests_attributes_rule_hierarchy(mock_client_init, rp_hierarchy_code, scenario_idx): + mock_client = setup_mock(mock_client_init) + variables = {"rp_tests_attributes": "test_key:test_value", "rp_hierarchy_code": rp_hierarchy_code} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests( + tests=["examples/bdd/step_defs/test_rule_description_steps.py"], variables=variables + ) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + scenario_call = mock_client.start_test_item.call_args_list[scenario_idx] + assert scenario_call[1]["attributes"] == [{"key": "test_key", "value": "test_value"}] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_tests_attributes_bdd_tags(mock_client_init): + mock_client = setup_mock(mock_client_init) + variables = {"rp_tests_attributes": "test_key:test_value"} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/bdd/step_defs/test_belly.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + scenario_call = mock_client.start_test_item.call_args_list[0] + attributes = scenario_call[1]["attributes"] + assert len(attributes) == 3 + assert {"key": "test_key", "value": "test_value"} in attributes + assert {"value": "ok"} in attributes + assert {"key": "key", "value": "value"} in attributes diff --git a/tests/integration/test_case_id_report.py b/tests/integration/test_case_id_report.py new file mode 100644 index 00000000..eb94cb8e --- /dev/null +++ b/tests/integration/test_case_id_report.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for Test Case ID report.""" + +from unittest import mock + +import pytest + +from examples.test_case_id import ( + test_case_id_decorator, + test_case_id_decorator_params_false, + test_case_id_decorator_params_mark, + test_case_id_decorator_params_no, + test_case_id_decorator_params_partially, + test_case_id_decorator_params_true, +) +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +@pytest.mark.parametrize( + ["test", "expected_id"], + [ + ("examples/test_simple.py", "examples/test_simple.py:test_simple"), + ( + "examples/params/test_in_class_parameterized.py", + "examples/params/test_in_class_parameterized.py:Tests.test_in_class_parameterized[param]", + ), + ("examples/test_case_id/test_case_id_decorator.py", test_case_id_decorator.TEST_CASE_ID), + ( + "examples/test_case_id/test_case_id_decorator_params_false.py", + test_case_id_decorator_params_false.TEST_CASE_ID, + ), + ("examples/test_case_id/test_case_id_decorator_params_no.py", test_case_id_decorator_params_no.TEST_CASE_ID), + ( + "examples/test_case_id/test_case_id_decorator_params_partially.py", + test_case_id_decorator_params_partially.TEST_CASE_ID + "[value1]", + ), + ( + "examples/test_case_id/test_case_id_decorator_params_true.py", + test_case_id_decorator_params_true.TEST_CASE_ID + "[value1,value2]", + ), + ("examples/test_case_id/test_case_id_decorator_no_id.py", ""), + ("examples/test_case_id/test_case_id_decorator_no_id_params_false.py", ""), + ("examples/test_case_id/test_case_id_decorator_no_id_params_true.py", "[value1,value2]"), + ("examples/test_case_id/test_case_id_decorator_no_id_partial_params_true.py", "[value2]"), + ( + "examples/test_case_id/test_case_id_decorator_params_mark.py", + test_case_id_decorator_params_mark.TEST_CASE_ID, + ), + ], +) +def test_parameters(mock_client_init, test, expected_id): + """Verify different tests have correct Test Case IDs. + + :param mock_client_init: Pytest fixture + :param test: a test to run + :param expected_id: an expected Test Case ID + """ + result = utils.run_pytest_tests(tests=[test]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + assert step_call_args["test_case_id"] == expected_id diff --git a/tests/integration/test_case_id_use_index_report.py b/tests/integration/test_case_id_use_index_report.py new file mode 100644 index 00000000..5e6a3590 --- /dev/null +++ b/tests/integration/test_case_id_use_index_report.py @@ -0,0 +1,77 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for Test Case ID report with use_index parameter.""" + +from unittest import mock + +import pytest + +from examples.test_case_id import ( + test_case_id_decorator_use_index_false, + test_case_id_decorator_use_index_partial_params, + test_case_id_decorator_use_index_true, +) +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +@pytest.mark.parametrize( + ["test", "expected_id"], + [ + # Test use_index=True: should use parameter indices instead of values + # Format should be: ('param1', 0),('param2', 0) instead of value1,value2 + ( + "examples/test_case_id/test_case_id_decorator_use_index_true.py", + test_case_id_decorator_use_index_true.TEST_CASE_ID + "[('param1', 0),('param2', 0)]", + ), + # Test use_index=False: should use parameter values (default behavior) + ( + "examples/test_case_id/test_case_id_decorator_use_index_false.py", + test_case_id_decorator_use_index_false.TEST_CASE_ID + "[value1,value2]", + ), + # Test use_index=True with selected params: should use index for selected param only + # Should be ('param1', 0) instead of value1 + ( + "examples/test_case_id/test_case_id_decorator_use_index_partial_params.py", + test_case_id_decorator_use_index_partial_params.TEST_CASE_ID + "[('param1', 0)]", + ), + ], +) +def test_use_index_parameters(mock_client_init, test, expected_id): + """Verify the use_index parameter in Test Case ID functionality. + + This test verifies that the new use_index parameter works correctly for parameterized tests. + When use_index=True, the test case ID should include parameter indices instead of parameter values. + This is useful for scenarios where parameter values might change between test runs but the indices + remain constant, ensuring that retry groups are properly maintained. + + The test covers: + 1. use_index=True with all parameters - should use (param_name, index) format + 2. use_index=False - should use parameter values (default behavior) + 3. use_index=True with selected parameters - should use indices only for selected params + + :param mock_client_init: Pytest fixture for mocking the ReportPortal client + :param test: a test file path to run + :param expected_id: the expected Test Case ID format + """ + result = utils.run_pytest_tests(tests=[test]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + assert step_call_args["test_case_id"] == expected_id diff --git a/tests/integration/test_code_reference.py b/tests/integration/test_code_reference.py new file mode 100644 index 00000000..b197e25e --- /dev/null +++ b/tests/integration/test_code_reference.py @@ -0,0 +1,55 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for code references generation.""" + +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +@pytest.mark.parametrize( + ["test", "code_ref"], + [ + ("examples/test_simple.py", "examples/test_simple.py:test_simple"), + ( + "examples/params/test_in_class_parameterized.py", + "examples/params/test_in_class_parameterized.py:Tests.test_in_class_parameterized", + ), + ("examples/hierarchy/test_in_class.py", "examples/hierarchy/test_in_class.py:Tests.test_in_class"), + ( + "examples/hierarchy/test_in_class_in_class.py", + "examples/hierarchy/test_in_class_in_class.py:Tests.Test.test_in_class_in_class", + ), + ], +) +def test_code_reference(mock_client_init, test, code_ref): + """Verify different tests have correct code reference. + + :param mock_client_init: Pytest fixture + :param test: a test to run + :param code_ref: an expected code reference value + """ + result = utils.run_pytest_tests(tests=[test]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + assert step_call_args["code_ref"] == code_ref diff --git a/tests/integration/test_config_handling.py b/tests/integration/test_config_handling.py new file mode 100644 index 00000000..64372119 --- /dev/null +++ b/tests/integration/test_config_handling.py @@ -0,0 +1,292 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for configuration parameters.""" + +import warnings +from unittest import mock + +import pytest +from delayed_assert import assert_expectations, expect +from reportportal_client import OutputType + +from examples import test_rp_custom_logging, test_rp_logging +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils +from tests.integration import setup_mock_for_logging + +TEST_LAUNCH_ID = "test_launch_id" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_launch_uuid(mock_client_init): + """Verify that RP plugin does not start/stop launch if 'rp_launch_id' set. + + :param mock_client_init: mocked client class + """ + variables = dict() + variables["rp_launch_id"] = TEST_LAUNCH_ID + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_simple.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + assert mock_client_init.call_count == 1 + constructor_call = mock_client_init.call_args_list[0] + assert "launch_uuid" in constructor_call[1] + assert constructor_call[1]["launch_uuid"] == TEST_LAUNCH_ID + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_parent_item_id(mock_client_init): + """Verify RP set Parent ID for root items if 'rp_parent_item_id' set. + + :param mock_client_init: Pytest fixture + """ + parent_id = "parent_id" + variables = dict() + variables["rp_parent_item_id"] = parent_id + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_simple.py"], variables=variables) + + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + expect(mock_client.start_launch.call_count == 1, '"start_launch" method was not called') + expect(mock_client.finish_launch.call_count == 1, '"finish_launch" method was not called') + + start_call_args = mock_client.start_test_item.call_args_list + finish_call_args = mock_client.finish_test_item.call_args_list + + expect(len(start_call_args) == len(finish_call_args)) + expect(start_call_args[0][1]["parent_item_id"] == parent_id) + assert_expectations() + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_parent_item_id_and_rp_launch_id(mock_client_init): + """Verify RP handles both conf props 'rp_parent_item_id' & 'rp_launch_id'. + + :param mock_client_init: mocked client class + """ + parent_id = "parent_id" + variables = dict() + variables["rp_parent_item_id"] = parent_id + variables["rp_launch_id"] = TEST_LAUNCH_ID + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_simple.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + assert mock_client_init.call_count == 1 + constructor_call = mock_client_init.call_args_list[0] + assert "launch_uuid" in constructor_call[1] + assert constructor_call[1]["launch_uuid"] == TEST_LAUNCH_ID + + mock_client = mock_client_init.return_value + + assert mock_client.start_test_item.call_count > 0 + item_create = mock_client.start_test_item.call_args_list[0] + call_kwargs = item_create[1] + assert "parent_item_id" in call_kwargs + assert call_kwargs["parent_item_id"] == parent_id + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_log_format(mock_client_init): + log_format = "(%(name)s) %(message)s (%(filename)s:%(lineno)s)" + variables = {"rp_log_format": log_format} + variables.update(utils.DEFAULT_VARIABLES.items()) + + mock_client = mock_client_init.return_value + result = utils.run_tests_with_client(mock_client, ["examples/test_rp_logging.py"], variables=variables) + + assert int(result) == 0, "Exit code should be 0 (no errors)" + + expect(mock_client.log.call_count == 1) + message = mock_client.log.call_args_list[0][0][1] + expect(len(message) > 0) + expect(message == f"(test_rp_logging) {test_rp_logging.LOG_MESSAGE} (test_rp_logging.py:24)") + assert_expectations() + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_log_batch_payload_limit(mock_client_init): + log_size = 123456 + variables = {"rp_log_batch_payload_limit": log_size} + variables.update(utils.DEFAULT_VARIABLES.items()) + + result = utils.run_pytest_tests(["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + expect(mock_client_init.call_count == 1) + + constructor_args = mock_client_init.call_args_list[0][1] + expect(constructor_args["log_batch_payload_limit"] == log_size) + assert_expectations() + + +def filter_agent_call(warn): + category = getattr(warn, "category", None) + if category: + return category.__name__ == "DeprecationWarning" or category.__name__ == "RuntimeWarning" + return False + + +def filter_agent_calls(warning_list): + return list(filter(lambda call: filter_agent_call(call), warning_list)) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_api_key(mock_client_init): + api_key = "rp_api_key" + variables = dict(utils.DEFAULT_VARIABLES) + variables.update({"rp_api_key": api_key}.items()) + + with warnings.catch_warnings(record=True) as w: + result = utils.run_pytest_tests(["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + expect(mock_client_init.call_count == 1) + + constructor_args = mock_client_init.call_args_list[0][1] + expect(constructor_args["api_key"] == api_key) + expect(len(filter_agent_calls(w)) == 0) + assert_expectations() + + +def test_rp_api_key_empty(): + api_key = "" + variables = dict(utils.DEFAULT_VARIABLES) + variables.update({"rp_api_key": api_key}.items()) + + result = utils.run_pytest_tests(["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 3, "Exit code should be 3 (exited with internal error)" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_api_retries(mock_client_init): + retries = 5 + variables = dict(utils.DEFAULT_VARIABLES) + variables.update({"rp_api_retries": str(retries)}.items()) + mock_client = mock_client_init.return_value + + with warnings.catch_warnings(record=True) as w: + result = utils.run_tests_with_client(mock_client, ["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + expect(mock_client_init.call_count == 1) + + constructor_args = mock_client_init.call_args_list[0][1] + expect(constructor_args["retries"] == retries) + expect(len(filter_agent_calls(w)) == 0) + assert_expectations() + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_issue_system_url_warning(mock_client_init): + url = "https://bugzilla.some.com/show_bug.cgi?id={issue_id}" + variables = utils.DEFAULT_VARIABLES.copy() + variables.update({"rp_issue_system_url": str(url)}.items()) + + with warnings.catch_warnings(record=True) as w: + result = utils.run_pytest_tests(["examples/test_issue_id.py"], variables=variables) + assert int(result) == 1, "Exit code should be 1 (test failure)" + + expect(mock_client_init.call_count == 1) + expect(len(filter_agent_calls(w)) == 1) + assert_expectations() + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_launch_uuid_print(mock_client_init): + print_uuid = True + variables = utils.DEFAULT_VARIABLES.copy() + variables.update({"rp_launch_uuid_print": str(print_uuid)}.items()) + mock_client = mock_client_init.return_value + result = utils.run_tests_with_client(mock_client, ["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + expect(mock_client_init.call_count == 1) + expect(mock_client_init.call_args_list[0][1]["launch_uuid_print"] == print_uuid) + expect(mock_client_init.call_args_list[0][1]["print_output"] is OutputType.STDOUT) + assert_expectations() + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_launch_uuid_print_stderr(mock_client_init): + print_uuid = True + variables = utils.DEFAULT_VARIABLES.copy() + variables.update({"rp_launch_uuid_print": str(print_uuid), "rp_launch_uuid_print_output": "stderr"}.items()) + mock_client = mock_client_init.return_value + result = utils.run_tests_with_client(mock_client, ["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + expect(mock_client_init.call_count == 1) + expect(mock_client_init.call_args_list[0][1]["launch_uuid_print"] == print_uuid) + expect(mock_client_init.call_args_list[0][1]["print_output"] is OutputType.STDERR) + assert_expectations() + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_launch_uuid_print_invalid_output(mock_client_init): + print_uuid = True + variables = utils.DEFAULT_VARIABLES.copy() + variables.update({"rp_launch_uuid_print": str(print_uuid), "rp_launch_uuid_print_output": "something"}.items()) + result = utils.run_pytest_tests(["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 3, "Exit code should be 3 (INTERNALERROR)" + assert mock_client_init.call_count == 0 + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_no_launch_uuid_print(mock_client_init): + variables = utils.DEFAULT_VARIABLES.copy() + mock_client = mock_client_init.return_value + result = utils.run_tests_with_client(mock_client, ["examples/test_rp_logging.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + expect(mock_client_init.call_count == 1) + expect(mock_client_init.call_args_list[0][1]["launch_uuid_print"] is False) + expect(mock_client_init.call_args_list[0][1]["print_output"] is OutputType.STDOUT) + assert_expectations() + + +@pytest.mark.parametrize( + "connect_value, read_value, expected_result", + [("5", "15", (5.0, 15.0)), ("5.5", "15.5", (5.5, 15.5)), (None, None, None), (None, "5", 5), ("5", None, 5)], +) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_client_timeouts(mock_client_init, connect_value, read_value, expected_result): + variables = utils.DEFAULT_VARIABLES.copy() + if connect_value: + variables["rp_connect_timeout"] = connect_value + if read_value: + variables["rp_read_timeout"] = read_value + + mock_client = mock_client_init.return_value + result = utils.run_tests_with_client(mock_client, ["examples/test_rp_logging.py"], variables=variables) + + assert int(result) == 0, "Exit code should be 0 (no errors)" + assert mock_client_init.call_count == 1 + assert mock_client_init.call_args_list[0][1]["http_timeout"] == expected_result + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_log_custom_levels(mock_client_init): + setup_mock_for_logging(mock_client_init) + custom_log_level = test_rp_custom_logging.LOG_LEVEL + custom_log_name = "ASSERTION" + variables = dict(utils.DEFAULT_VARIABLES) + variables.update({"rp_log_custom_levels": str(custom_log_level) + ":" + custom_log_name}) + + result = utils.run_pytest_tests(["examples/test_rp_custom_logging.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.log.call_count == 1 + assert mock_client.log.call_args_list[0][1]["level"] == custom_log_name diff --git a/tests/integration/test_connection_close.py b/tests/integration/test_connection_close.py new file mode 100644 index 00000000..bf6cf9c0 --- /dev/null +++ b/tests/integration/test_connection_close.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +from unittest import mock + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_connection_close(mock_client_init): + mock_client = mock_client_init.return_value + + result = utils.run_tests_with_client(mock_client, ["examples/test_rp_logging.py"]) + + assert int(result) == 0, "Exit code should be 0 (no errors)" + assert mock_client.close.call_count == 1, '"close" method was not called at the end of the test' diff --git a/tests/integration/test_custom_name.py b/tests/integration/test_custom_name.py new file mode 100644 index 00000000..16699fd4 --- /dev/null +++ b/tests/integration/test_custom_name.py @@ -0,0 +1,47 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + +from examples.custom_name.test_custom_name_args import TEST_NAME_ARGS +from examples.custom_name.test_custom_name_empty import TEST_NAME_EMPTY +from examples.custom_name.test_custom_name_kwargs import TEST_NAME_KWARGS +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@pytest.mark.parametrize( + "test, expected", + [ + ("examples/custom_name/test_custom_name_args.py", TEST_NAME_ARGS), + ("examples/custom_name/test_custom_name_kwargs.py", TEST_NAME_KWARGS), + ("examples/custom_name/test_custom_name_empty.py", TEST_NAME_EMPTY), + ], +) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_custom_attribute_report(mock_client_init, test, expected): + result = utils.run_pytest_tests(tests=[test], variables=utils.DEFAULT_VARIABLES) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == finish_count == 1, 'Incorrect number of "start_test_item" or "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[0][1] + assert step_call_args["name"] == expected, "Incorrect item name" + assert step_call_args["attributes"] == [], "No attributes should be added for the test item" diff --git a/tests/integration/test_debug_mode.py b/tests/integration/test_debug_mode.py new file mode 100644 index 00000000..c7b65633 --- /dev/null +++ b/tests/integration/test_debug_mode.py @@ -0,0 +1,45 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for the debug mode switch.""" + +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +@pytest.mark.parametrize(["mode", "expected_mode"], [("DEFAULT", "DEFAULT"), ("DEBUG", "DEBUG"), (None, "DEFAULT")]) +def test_launch_mode(mock_client_init, mode, expected_mode): + """Verify different launch modes are passed to `start_launch` method. + + :param mock_client_init: Pytest fixture + :param mode: a variable to be passed to pytest + :param expected_mode: a value which should be passed to + ReportPortalService + """ + variables = dict() + if mode is not None: + variables["rp_mode"] = mode + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_simple.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + assert mock_client_init.call_count == 1, "client wasn't initialized" + + init_kwargs = mock_client_init.call_args_list[0][1] + assert "mode" in init_kwargs + assert init_kwargs["mode"] == expected_mode diff --git a/tests/integration/test_empty_run.py b/tests/integration/test_empty_run.py new file mode 100644 index 00000000..e3bda689 --- /dev/null +++ b/tests/integration/test_empty_run.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for the empty run.""" + +from datetime import datetime, timezone +from unittest import mock + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_empty_run(mock_client_init): + """Verify that RP plugin does not fail if there is not tests in run. + + :param mock_client_init: Pytest fixture + """ + test_start_time = datetime.now(timezone.utc) + result = utils.run_pytest_tests(tests=["examples/empty/"]) + + assert int(result) == 5, "Exit code should be 5 (no tests)" + + mock_client = mock_client_init.return_value + assert mock_client.start_launch.call_count == 1, '"start_launch" method was not called' + assert mock_client.finish_launch.call_count == 1, '"finish_launch" method was not called' + + finish_args = mock_client.finish_launch.call_args_list + assert "status" not in finish_args[0][1], "Launch status should not be defined" + launch_end_time = finish_args[0][1]["end_time"] + assert launch_end_time is not None and launch_end_time > test_start_time, "Launch end time is empty" diff --git a/tests/integration/test_fixtures.py b/tests/integration/test_fixtures.py new file mode 100644 index 00000000..f685f0f9 --- /dev/null +++ b/tests/integration/test_fixtures.py @@ -0,0 +1,606 @@ +# Copyright 2024 EPAM Systems +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict +from typing import Optional +from unittest import mock + +import pytest +from reportportal_client import set_current +from reportportal_client.steps import StepReporter + +from examples.fixtures.test_failure_fixture_teardown.conftest import ( + LOG_MESSAGE_BEFORE_YIELD as LOG_MESSAGE_BEFORE_YIELD_TEST_FAILURE, +) +from examples.fixtures.test_failure_fixture_teardown.conftest import ( + LOG_MESSAGE_TEARDOWN as LOG_MESSAGE_TEARDOWN_TEST_FAILURE, +) +from examples.fixtures.test_fixture_return_none.conftest import LOG_MESSAGE_SETUP as LOG_MESSAGE_BEFORE_RETURN_NONE +from examples.fixtures.test_fixture_setup.conftest import LOG_MESSAGE_SETUP as SINGLE_SETUP_MESSAGE +from examples.fixtures.test_fixture_setup_failure.conftest import LOG_MESSAGE_SETUP as LOG_MESSAGE_SETUP_FAILURE +from examples.fixtures.test_fixture_teardown.conftest import LOG_MESSAGE_BEFORE_YIELD, LOG_MESSAGE_TEARDOWN +from examples.fixtures.test_fixture_teardown_failure.conftest import ( + LOG_MESSAGE_BEFORE_YIELD as LOG_MESSAGE_BEFORE_YIELD_FAILURE, +) +from examples.fixtures.test_fixture_teardown_failure.conftest import ( + LOG_MESSAGE_TEARDOWN as LOG_MESSAGE_TEARDOWN_FAILURE, +) +from examples.fixtures.test_fixture_yield_none.conftest import LOG_MESSAGE_SETUP as LOG_MESSAGE_BEFORE_YIELD_NONE +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + +ITEM_ID_DICT = defaultdict(lambda: 0) +ITEM_ID_LIST = [] + + +def generate_item_id(*args, **kwargs) -> str: + if args: + name = args[0] + else: + name = kwargs["name"] + ITEM_ID_DICT[name] += 1 + item_id = f"{name}_{ITEM_ID_DICT[name]}" + ITEM_ID_LIST.append(item_id) + return item_id + + +def get_last_item_id() -> str: + return ITEM_ID_LIST[-1] + + +def remove_last_item_id(*_, **__) -> Optional[str]: + if len(ITEM_ID_LIST) > 0: + return ITEM_ID_LIST.pop() + + +def setup_mock(mock_client_init): + mock_client = mock_client_init.return_value + mock_client.step_reporter = StepReporter(mock_client) + return mock_client + + +def setup_mock_for_logging(mock_client_init): + mock_client = setup_mock(mock_client_init) + set_current(mock_client) + mock_client.start_test_item.side_effect = generate_item_id + mock_client.finish_test_item.side_effect = remove_last_item_id + mock_client.current_item.side_effect = get_last_item_id + return mock_client + + +@pytest.mark.parametrize("switch", [True, False]) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_on_off(mock_client_init, switch): + mock_client = setup_mock(mock_client_init) + + variables = dict(utils.DEFAULT_VARIABLES) + variables["rp_report_fixtures"] = switch + result = utils.run_pytest_tests(tests=["examples/fixtures/test_fixture_teardown"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + start_count = mock_client.start_test_item.call_count + finish_calls = mock_client.finish_test_item.call_args_list + finish_count = mock_client.finish_test_item.call_count + expected_count = 3 if switch else 1 + assert start_count == expected_count, 'Incorrect number of "start_test_item" calls' + assert finish_count == expected_count, 'Incorrect number of "finish_test_item" calls' + for call in finish_calls: + assert call[1]["status"] == "PASSED" + + +def run_tests(test_path, should_fail=False): + variables = dict(utils.DEFAULT_VARIABLES) + variables["rp_report_fixtures"] = True + result = utils.run_pytest_tests(tests=[test_path], variables=variables) + if should_fail: + assert int(result) == 1, "Exit code should be 1 (test failure)" + else: + assert int(result) == 0, "Exit code should be 0 (no errors)" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_setup(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_setup" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 3, 'Incorrect number of "start_test_item" calls' + assert finish_count == 3, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + assert teardown_call_args[0] == f"function fixture teardown: {fixture_name}" + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 1, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list + log_call_args = log_call_args_list[0][0] + log_call_kwargs = log_call_args_list[0][1] + + assert log_call_args[1] == SINGLE_SETUP_MESSAGE + assert log_call_kwargs["item_id"] == f"{step_name}_1" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_teardown(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_teardown" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 3, 'Incorrect number of "start_test_item" calls' + assert finish_count == 3, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + setup_step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == setup_step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + teardown_step_name = f"function fixture teardown: {fixture_name}" + assert teardown_call_args[0] == teardown_step_name + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 2, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list + log_call_args = log_call_args_list[0][0] + log_call_kwargs = log_call_args_list[0][1] + + assert log_call_args[1] == LOG_MESSAGE_BEFORE_YIELD + assert log_call_kwargs["item_id"] == f"{setup_step_name}_1" + + log_call_args = log_call_args_list[-1][0] + log_call_kwargs = log_call_args_list[-1][1] + + assert log_call_args[1] == LOG_MESSAGE_TEARDOWN + assert ( + log_call_kwargs["item_id"] + == "examples/fixtures/test_fixture_teardown/test_fixture_teardown.py::test_fixture_teardown_1" + ) + + +FIXTURE_FAILED_MESSAGE = "function fixture setup failed: test_fixture_setup_failure_config" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_setup_failure(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_setup_failure" + run_tests(test_path, True) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 2, 'Incorrect number of "start_test_item" calls' + assert finish_count == 2, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 4, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list + log_call_args = log_call_args_list[0][0] + log_call_kwargs = log_call_args_list[0][1] + + assert log_call_args[1] == LOG_MESSAGE_SETUP_FAILURE + assert log_call_kwargs["item_id"] == f"{step_name}_1" + + log_call_kwargs = log_call_args_list[1][1] + assert log_call_kwargs["message"] == FIXTURE_FAILED_MESSAGE + assert log_call_kwargs["item_id"] == f"{step_name}_1" + + log_call_kwargs = log_call_args_list[2][1] + assert log_call_kwargs["message"].startswith("Traceback (most recent call last):") + assert log_call_kwargs["item_id"] == f"{step_name}_1" + + log_call_kwargs = log_call_args_list[3][1] + + assert log_call_kwargs["message"].endswith( + "examples/fixtures/test_fixture_setup_failure/conftest.py:30: Exception" + ) + assert ( + log_call_kwargs["item_id"] + == "examples/fixtures/test_fixture_setup_failure/test_fixture_setup_failure.py::test_fixture_setup_failure_1" + ) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_teardown_failure(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_teardown_failure" + run_tests(test_path, True) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 3, 'Incorrect number of "start_test_item" calls' + assert finish_count == 3, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + setup_step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == setup_step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + teardown_step_name = f"function fixture teardown: {fixture_name}" + assert teardown_call_args[0] == teardown_step_name + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 3, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list + log_call_args = log_call_args_list[0][0] + log_call_kwargs = log_call_args_list[0][1] + + assert log_call_args[1] == LOG_MESSAGE_BEFORE_YIELD_FAILURE + assert log_call_kwargs["item_id"] == f"{setup_step_name}_1" + + log_call_args = log_call_args_list[1][0] + log_call_kwargs = log_call_args_list[1][1] + + assert log_call_args[1] == LOG_MESSAGE_TEARDOWN_FAILURE + assert log_call_kwargs["item_id"] == ( + "examples/fixtures/test_fixture_teardown_failure/test_fixture_teardown_failure.py::" + "test_fixture_teardown_failure_1" + ) + + log_call_kwargs = log_call_args_list[2][1] + + assert log_call_kwargs["message"].endswith( + "examples/fixtures/test_fixture_teardown_failure/conftest.py:34: Exception" + ) + assert log_call_kwargs["item_id"] == ( + "examples/fixtures/test_fixture_teardown_failure/test_fixture_teardown_failure.py::" + "test_fixture_teardown_failure_1" + ) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_yield_none(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_yield_none" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 3, 'Incorrect number of "start_test_item" calls' + assert finish_count == 3, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + setup_step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == setup_step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + teardown_step_name = f"function fixture teardown: {fixture_name}" + assert teardown_call_args[0] == teardown_step_name + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 1, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list + log_call_args = log_call_args_list[0][0] + log_call_kwargs = log_call_args_list[0][1] + + assert log_call_args[1] == LOG_MESSAGE_BEFORE_YIELD_NONE + assert log_call_kwargs["item_id"] == f"{setup_step_name}_1" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_return_none(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_return_none" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 3, 'Incorrect number of "start_test_item" calls' + assert finish_count == 3, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + setup_step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == setup_step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + teardown_step_name = f"function fixture teardown: {fixture_name}" + assert teardown_call_args[0] == teardown_step_name + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 1, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list + log_call_args = log_call_args_list[0][0] + log_call_kwargs = log_call_args_list[0][1] + + assert log_call_args[1] == LOG_MESSAGE_BEFORE_RETURN_NONE + assert log_call_kwargs["item_id"] == f"{setup_step_name}_1" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_failure_fixture_teardown(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_failure_fixture_teardown" + run_tests(test_path, True) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 3, 'Incorrect number of "start_test_item" calls' + assert finish_count == 3, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + setup_step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == setup_step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + teardown_step_name = f"function fixture teardown: {fixture_name}" + assert teardown_call_args[0] == teardown_step_name + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 3, 'Incorrect number of "log" calls' + + log_call_args_list = mock_client.log.call_args_list + log_call_args = log_call_args_list[0][0] + log_call_kwargs = log_call_args_list[0][1] + + assert log_call_args[1] == LOG_MESSAGE_BEFORE_YIELD_TEST_FAILURE + assert log_call_kwargs["item_id"] == f"{setup_step_name}_1" + + log_call_args = log_call_args_list[2][0] + log_call_kwargs = log_call_args_list[2][1] + + assert log_call_args[1] == LOG_MESSAGE_TEARDOWN_TEST_FAILURE + assert log_call_kwargs["item_id"] == ( + "examples/fixtures/test_failure_fixture_teardown/test_failure_fixture_teardown.py::" + "test_failure_fixture_teardown_1" + ) + + log_call_kwargs = log_call_args_list[1][1] + + assert log_call_kwargs["message"].endswith( + "examples/fixtures/test_failure_fixture_teardown/test_failure_fixture_teardown.py:29: AssertionError" + ) + assert log_call_kwargs["item_id"] == ( + "examples/fixtures/test_failure_fixture_teardown/test_failure_fixture_teardown.py::" + "test_failure_fixture_teardown_1" + ) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_session_fixture_setup(mock_client_init): + mock_client = setup_mock(mock_client_init) + + test_path = "examples/fixtures/session_fixture_return" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 4, 'Incorrect number of "start_test_item" calls' + assert finish_count == 4, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + step_name = f"session fixture setup: {fixture_name}" + assert setup_call_args[0] == step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + assert teardown_call_args[0] == f"session fixture teardown: {fixture_name}" + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_package_fixture_setup(mock_client_init): + mock_client = setup_mock(mock_client_init) + + test_path = "examples/fixtures/package_fixture_return" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == finish_count == 4, 'Incorrect number of "start_test_item" or "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + step_name = f"package fixture setup: {fixture_name}" + assert setup_call_args[0] == step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + assert teardown_call_args[0] == f"package fixture teardown: {fixture_name}" + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_module_fixture_setup(mock_client_init): + mock_client = setup_mock(mock_client_init) + + test_path = "examples/fixtures/module_fixture_return" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 4, 'Incorrect number of "start_test_item" calls' + assert finish_count == 4, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + step_name = f"module fixture setup: {fixture_name}" + assert setup_call_args[0] == step_name + + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + assert teardown_call_args[0] == f"module fixture teardown: {fixture_name}" + + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_class_fixture_setup(mock_client_init): + mock_client = setup_mock(mock_client_init) + + test_path = "examples/fixtures/class_fixture_return" + run_tests(test_path) + + start_count = mock_client.start_test_item.call_count + finish_count = mock_client.finish_test_item.call_count + assert start_count == 8, 'Incorrect number of "start_test_item" calls' + assert finish_count == 8, 'Incorrect number of "finish_test_item" calls' + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[1][0] + fixture_name = f'{test_path.split("/")[-1]}_config' + step_name = f"class fixture setup: {fixture_name}" + assert setup_call_args[0] == step_name + setup_call_kwargs = call_args[1][1] + assert not setup_call_kwargs["has_stats"] + + setup_call_args = call_args[-3][0] + assert setup_call_args[0] == step_name + setup_call_kwargs = call_args[-3][1] + assert not setup_call_kwargs["has_stats"] + + teardown_step_name = f"class fixture teardown: {fixture_name}" + teardown_call_args = call_args[-5][0] + assert teardown_call_args[0] == teardown_step_name + setup_call_kwargs = call_args[-5][1] + assert not setup_call_kwargs["has_stats"] + + teardown_call_args = call_args[-1][0] + assert teardown_call_args[0] == teardown_step_name + setup_call_kwargs = call_args[-1][1] + assert not setup_call_kwargs["has_stats"] + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_setup_skip(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_skipped/test_fixture_skipped.py" + run_tests(test_path, False) + + call_args = mock_client.start_test_item.call_args_list + setup_call_args = call_args[2][0] + fixture_name = "skip_fixture" + step_name = f"function fixture setup: {fixture_name}" + assert setup_call_args[0] == step_name + + setup_call_kwargs = call_args[2][1] + assert not setup_call_kwargs["has_stats"] + + log_count = mock_client.log.call_count + assert log_count == 1, 'Incorrect number of "log" calls' + + call_args = mock_client.finish_test_item.call_args_list + finish_call_kwargs = call_args[1][1] + assert finish_call_kwargs["status"] == "PASSED" + + finish_call_kwargs = call_args[-1][1] + assert finish_call_kwargs["status"] == "SKIPPED" + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_fixture_exit(mock_client_init): + mock_client = setup_mock_for_logging(mock_client_init) + + test_path = "examples/fixtures/test_fixture_exit/test_fixture_exit.py" + variables = dict(utils.DEFAULT_VARIABLES) + variables["rp_report_fixtures"] = True + result = utils.run_pytest_tests(tests=[test_path], variables=variables) + assert int(result) == 2, "Exit code should be 2 (unexpected exit)" + + call_args = mock_client.start_test_item.call_args_list + assert len(call_args) == 2, 'Incorrect number of "start_test_item" calls' + + call_args = mock_client.finish_test_item.call_args_list + assert len(call_args) == 2, 'Incorrect number of "finish_test_item" calls' diff --git a/tests/integration/test_issue_report.py b/tests/integration/test_issue_report.py new file mode 100644 index 00000000..bdf04760 --- /dev/null +++ b/tests/integration/test_issue_report.py @@ -0,0 +1,198 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration test for issue type reporting.""" + +from unittest import mock + +import pytest +from delayed_assert import assert_expectations, expect +from reportportal_client.core.rp_issues import Issue + +from examples import test_issue_id +from pytest_reportportal.service import NOT_ISSUE +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + +ISSUE_PLACEHOLDER = "{issue_id}" +ISSUE_URL_PATTERN = f"https://bugzilla.some.com/show_bug.cgi?id={ISSUE_PLACEHOLDER}" +BTS_PROJECT = "RP-TEST" +BTS_URL = "https://bugzilla.some.com" + + +@mock.patch(REPORT_PORTAL_SERVICE) +@pytest.mark.parametrize("issue_id_mark", [True, False]) +def test_issue_id_attribute(mock_client_init, issue_id_mark): + """Verify agent reports issue attribute if configured. + + :param mock_client_init: Pytest fixture + :param issue_id_mark: Attribute report configuration + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + mock_client.get_project_settings.side_effect = utils.project_settings + + variables = {"rp_issue_id_marks": issue_id_mark} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_issue_id.py"], variables=variables) + assert int(result) == 1, "Exit code should be 1 (test failed)" + + call_args = mock_client.start_test_item.call_args_list + finish_test_step = call_args[-1][1] + attributes = finish_test_step["attributes"] + + if issue_id_mark: + assert len(attributes) == 1 + issue_attribute = attributes[0] + expect(issue_attribute["key"] == "issue") + expect(issue_attribute["value"] == test_issue_id.ID) + assert_expectations() + else: + assert len(attributes) == 0 + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_issue_report(mock_client_init): + """Verify agent reports issue ids and defect type. + + :param mock_client_init: Pytest fixture + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + mock_client.get_project_settings.side_effect = utils.project_settings + + variables = {"rp_issue_system_url": ISSUE_URL_PATTERN} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_issue_id.py"], variables=variables) + assert int(result) == 1, "Exit code should be 1 (test failed)" + + call_args = mock_client.finish_test_item.call_args_list + finish_test_step = call_args[0][1] + issue = finish_test_step["issue"] + + assert isinstance(issue, Issue) + expect(issue.issue_type == "pb001") + expect(issue.comment is not None) + assert_expectations() + comments = issue.comment.split("\n") + assert len(comments) == 1 + comment = comments[0] + assert comment == "* {}: [{}]({})".format( + test_issue_id.REASON, test_issue_id.ID, ISSUE_URL_PATTERN.replace(ISSUE_PLACEHOLDER, test_issue_id.ID) + ) + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_passed_no_issue_report(mock_client_init): + """Verify agent do not report issue if test passed. + + :param mock_client_init: Pytest fixture + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + mock_client.get_project_settings.side_effect = utils.project_settings + + variables = {"rp_issue_system_url": ISSUE_URL_PATTERN} + variables.update(utils.DEFAULT_VARIABLES.items()) + result = utils.run_pytest_tests(tests=["examples/test_issue_id_pass.py"], variables=variables) + assert int(result) == 0, "Exit code should be 0 (no failures)" + + call_args = mock_client.finish_test_item.call_args_list + finish_test_step = call_args[0][1] + assert "issue" not in finish_test_step or finish_test_step["issue"] is None + + +@pytest.mark.parametrize(("flag_value", "expected_issue"), [(True, None), (False, NOT_ISSUE), (None, None)]) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_skipped_not_issue(mock_client_init, flag_value, expected_issue): + """Verify 'rp_is_skipped_an_issue' option handling. + + :param mock_client_init: mocked Report Portal client Pytest fixture + :param flag_value: option value to set during the test + :param expected_issue: result issue value to verify + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + + variables = dict() + if flag_value is not None: + variables["rp_is_skipped_an_issue"] = flag_value + variables.update(utils.DEFAULT_VARIABLES.items()) + + result = utils.run_pytest_tests(tests=["examples/skip/test_simple_skip.py"], variables=variables) + + assert int(result) == 0, "Exit code should be 0 (no failures)" + call_args = mock_client.finish_test_item.call_args_list + finish_test_step = call_args[0][1] + actual_issue = finish_test_step.get("issue", None) + assert actual_issue == expected_issue + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_skipped_custom_issue(mock_client_init): + """Verify skipped test with issue decorator handling. + + :param mock_client_init: mocked Report Portal client Pytest fixture + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + mock_client.get_project_settings.side_effect = utils.project_settings + + variables = dict() + variables["rp_is_skipped_an_issue"] = True + variables["rp_issue_system_url"] = ISSUE_URL_PATTERN + variables.update(utils.DEFAULT_VARIABLES.items()) + + result = utils.run_pytest_tests(tests=["examples/skip/test_skip_issue.py"], variables=variables) + + assert int(result) == 0, "Exit code should be 0 (no failures)" + call_args = mock_client.finish_test_item.call_args_list + finish_test_step = call_args[0][1] + actual_issue = finish_test_step.get("issue", None) + assert isinstance(actual_issue, Issue) + expect(actual_issue.issue_type == "pb001") + expect(actual_issue.comment is not None) + assert_expectations() + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_external_issue(mock_client_init): + """Verify skipped test with issue decorator handling. + + :param mock_client_init: mocked Report Portal client Pytest fixture + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + mock_client.get_project_settings.side_effect = utils.project_settings + + variables = {"rp_bts_project": BTS_PROJECT, "rp_bts_url": BTS_URL, "rp_bts_issue_url": ISSUE_URL_PATTERN} + variables.update(utils.DEFAULT_VARIABLES.items()) + + result = utils.run_pytest_tests(tests=["examples/test_issue_id.py"], variables=variables) + + assert int(result) == 1, "Exit code should be 1 (test failed)" + call_args = mock_client.finish_test_item.call_args_list + finish_test_step = call_args[0][1] + actual_issue = finish_test_step.get("issue", None) + assert isinstance(actual_issue, Issue) + expect(actual_issue.issue_type == "pb001") + expect(actual_issue.comment is not None) + external_issues = actual_issue._external_issues + expect(len(external_issues) == 1) + assert_expectations() + external_issue = external_issues[0] + expect(external_issue["btsUrl"] == BTS_URL) + expect(external_issue["btsProject"] == BTS_PROJECT) + expect(external_issue["ticketId"] == test_issue_id.ID) + expect(external_issue["url"] == ISSUE_URL_PATTERN.replace(ISSUE_PLACEHOLDER, test_issue_id.ID)) + assert_expectations() diff --git a/tests/integration/test_parameters_report.py b/tests/integration/test_parameters_report.py new file mode 100644 index 00000000..62c5d9e8 --- /dev/null +++ b/tests/integration/test_parameters_report.py @@ -0,0 +1,51 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for parameters report.""" + +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +@pytest.mark.parametrize( + ["test", "expected_params"], + [ + ("examples/test_simple.py", None), + ("examples/params/test_in_class_parameterized.py", {"param": "param"}), + ( + "examples/params/test_different_parameter_types.py", + {"integer": 1, "floating_point": 1.5, "boolean": True, "none": None}, + ), + ], +) +def test_parameters(mock_client_init, test, expected_params): + """Verify different tests have correct parameters. + + :param mock_client_init: Pytest fixture + :param test: a test to run + :param expected_params: an expected parameter dictionary + """ + result = utils.run_pytest_tests(tests=[test]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + assert mock_client.start_test_item.call_count > 0, '"start_test_item" called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + step_call_args = call_args[-1][1] + assert step_call_args["parameters"] == expected_params diff --git a/tests/integration/test_pass_failed_skipped.py b/tests/integration/test_pass_failed_skipped.py new file mode 100644 index 00000000..e67abbca --- /dev/null +++ b/tests/integration/test_pass_failed_skipped.py @@ -0,0 +1,68 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for item statuses report.""" + +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@pytest.mark.parametrize( + ("test", "expected_run_status", "expected_item_status"), + [ + ("examples/test_simple.py", 0, "PASSED"), + ("examples/test_simple_fail.py", 1, "FAILED"), + ("examples/skip/test_simple_skip.py", 0, "SKIPPED"), + ], +) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_simple_tests(mock_client_init, test, expected_run_status, expected_item_status): + """Verify a simple test creates correct structure and finishes all items. + + Report 'None' for suites and launch due to possible parallel execution. + Leave status calculation on Server. + :param mock_client_init: mocked Report Portal client Pytest fixture + :param test: a test to run as use case + :param expected_run_status: expected pytest run status + :param expected_item_status: expected result test item status + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + + result = utils.run_pytest_tests(tests=[test]) + assert int(result) == expected_run_status, "Exit code should be " + str(expected_run_status) + + start_call_args = mock_client.start_test_item.call_args_list + finish_call_args = mock_client.finish_test_item.call_args_list + assert len(start_call_args) == len(finish_call_args), "Number of started items should be equal to finished items" + + for i in range(len(start_call_args)): + start_test_step = start_call_args[-1 - i][1] + finish_test_step = finish_call_args[i][1] + + assert finish_test_step["item_id"].startswith(start_test_step["name"]) + if i == 0: + actual_status = finish_test_step["status"] + assert ( + actual_status == expected_item_status + ), f'Invalid item status, actual "{actual_status}", expected: "{expected_item_status}"' + + finish_launch_call_args = mock_client.finish_launch.call_args_list + assert len(finish_launch_call_args) == 1 + assert "end_time" in finish_launch_call_args[0][1] + assert finish_launch_call_args[0][1]["end_time"] is not None + assert "status" not in finish_launch_call_args[0][1] diff --git a/tests/integration/test_pytest_parallel.py b/tests/integration/test_pytest_parallel.py new file mode 100644 index 00000000..1c6cdfc5 --- /dev/null +++ b/tests/integration/test_pytest_parallel.py @@ -0,0 +1,46 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for "pytest_parallel" plugin.""" + +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils +from tests.helpers.utils import item_id_gen + + +@mock.patch(REPORT_PORTAL_SERVICE) +@pytest.mark.skip(reason="This test breaks all other tests, so only for local execution") +def test_pytest_parallel_threads(mock_client_init): + """Verify "pytest_parallel" plugin run tests in two threads. + + :param mock_client_init: Pytest fixture + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = item_id_gen + + result = utils.run_pytest_tests(tests=["examples/hierarchy"], args=["--tests-per-worker", "2"]) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + mock_client = mock_client_init.return_value + + assert mock_client.start_launch.call_count == 1, '"start_launch" method was not called' + assert mock_client.finish_launch.call_count == 1, '"finish_launch" method was not called' + + finish_args = mock_client.finish_launch.call_args_list + assert finish_args[0][1]["status"] in ("PASSED", None), "Launch failed" + launch_end_time = finish_args[0][1]["end_time"] + assert launch_end_time is not None and int(launch_end_time) > 0, "Launch end time is empty" diff --git a/tests/integration/test_suite_hierarchy.py b/tests/integration/test_suite_hierarchy.py new file mode 100644 index 00000000..6ac14988 --- /dev/null +++ b/tests/integration/test_suite_hierarchy.py @@ -0,0 +1,52 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes integration tests for different suite hierarchy.""" + +from unittest import mock + +import pytest + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils +from tests.integration import HIERARCHY_TEST_PARAMETERS + + +def verify_start_item_parameters(mock_client, expected_items): + assert mock_client.start_test_item.call_count == len( + expected_items + ), '"start_test_item" method was called incorrect number of times' + + call_args = mock_client.start_test_item.call_args_list + for i, call in enumerate(call_args): + start_kwargs = call[1] + assert start_kwargs["name"] == expected_items[i]["name"] + assert start_kwargs["item_type"] == expected_items[i]["item_type"] + verification = expected_items[i]["parent_item_id"] + assert verification(start_kwargs["parent_item_id"]) + + +@pytest.mark.parametrize(("test", "variables", "expected_items"), HIERARCHY_TEST_PARAMETERS) +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_hierarchy_parameters(mock_client_init, test, variables, expected_items): + """Verify suite hierarchy with `rp_hierarchy_dirs=True`. + + :param mock_client_init: Pytest fixture + """ + mock_client = mock_client_init.return_value + mock_client.start_test_item.side_effect = utils.item_id_gen + + result = utils.run_pytest_tests(tests=test, variables=variables) + assert int(result) == 0, "Exit code should be 0 (no errors)" + + verify_start_item_parameters(mock_client, expected_items) diff --git a/tests/integration/test_threads_logs.py b/tests/integration/test_threads_logs.py new file mode 100644 index 00000000..ae191610 --- /dev/null +++ b/tests/integration/test_threads_logs.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +from unittest import mock + +from tests import REPORT_PORTAL_SERVICE +from tests.helpers import utils + + +@mock.patch(REPORT_PORTAL_SERVICE) +def test_rp_thread_logs_reporting(mock_client_init): + """Verify logs from threads are sent to correct items`. + + :param mock_client_init: Pytest fixture + """ + mock_client = mock_client_init.return_value + mock_thread_client = mock_client.clone() + + def init_thread_client(*_, **__): + from reportportal_client import set_current + + set_current(mock_thread_client) + return mock_thread_client + + mock_client.clone.side_effect = init_thread_client + result = utils.run_tests_with_client(mock_client, ["examples/threads/"], args=["--rp-thread-logging"]) + + assert int(result) == 0, "Exit code should be 0 (no errors)" + assert mock_client.start_launch.call_count == 1, '"start_launch" method was not called' + assert mock_client.log.call_count == 1 + assert mock_thread_client.log.call_count == 2 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..678f062f --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This package contains unit tests for the project.""" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 00000000..4f66484b --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,115 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module contains common Pytest fixtures and hooks for unit tests.""" + +# noinspection PyUnresolvedReferences +from unittest import mock + +import py +from _pytest.config import Config +from _pytest.main import Session +from pluggy._tracing import TagTracer +from pytest import Module, fixture +from reportportal_client import RPLogger + +from pytest_reportportal.config import AgentConfig +from pytest_reportportal.service import PyTestService +from tests import REPORT_PORTAL_SERVICE + +ITEM_PATH = py.path.local("examples/test_simple.py") + + +@fixture +def logger(): + """Prepare instance of the RPLogger for testing.""" + return RPLogger("pytest_reportportal.test") + + +@fixture() +def mocked_config(): + """Mock Pytest config for testing.""" + mocked_config = mock.create_autospec(Config) + + mocked_config.getoption_side_effects = {"--collect-only": False, "--setup-plan": False, "rp_log_level": "debug"} + + def getoption_side_effect(name, default=None): + return mocked_config.getoption_side_effects.get(name, default if default else mock.Mock()) + + mocked_config._reporter_config = mock.Mock() + mocked_config.getoption.side_effect = getoption_side_effect + mocked_config._rp_enabled = True + mocked_config.rootdir = py.path.local("/path/to") + mocked_config.trace = TagTracer().get("root") + mocked_config.pluginmanager = mock.Mock() + mocked_config.option = mock.create_autospec(Config) + mocked_config.option.rp_project = "default_personal" + mocked_config.option.rp_endpoint = "http://docker.local:8080/" + mocked_config.option.rp_api_key = mock.sentinel.rp_api_key + mocked_config.option.rp_log_batch_size = -1 + mocked_config.option.retries = -1 + mocked_config.option.rp_hierarchy_dirs_level = "0" + mocked_config.option.rp_rerun = False + mocked_config.option.rp_launch_timeout = -1 + mocked_config.option.rp_thread_logging = True + mocked_config.option.rp_launch_uuid_print = "False" + mocked_config.option.rp_launch_uuid_print_output = "STDOUT" + mocked_config.option.rp_client_type = "SYNC" + mocked_config.option.rp_report_fixtures = "False" + mocked_config.option.rp_hierarchy_code = "False" + mocked_config.option.rp_hierarchy_dirs = "False" + mocked_config.option.rp_hierarchy_test_file = "True" + mocked_config.option.rp_skip_connection_test = "False" + mocked_config.option.rp_enabled = True + mocked_config.option.rp_log_level = "debug" + return mocked_config + + +@fixture() +def mocked_session(mocked_config): + """Mock Pytest session for testing.""" + mocked_session = mock.create_autospec(Session) + mocked_session.config = mocked_config + return mocked_session + + +@fixture() +def mocked_module(mocked_session): + """Mock Pytest Module for testing.""" + mocked_module = mock.create_autospec(Module) + mocked_module.parent = mocked_session + mocked_module.name = "module" + mocked_module.fspath = ITEM_PATH + return mocked_module + + +@fixture() +def mocked_item(mocked_session, mocked_module): + """Mock Pytest item for testing.""" + test_item = mock.Mock() + test_item.session = mocked_session + test_item.fspath = ITEM_PATH + name = "test_item" + test_item.name = name + test_item.originalname = name + test_item.parent = mocked_module + return test_item + + +@fixture() +def rp_service(mocked_config): + """Prepare instance of the PyTestService for testing.""" + service = PyTestService(AgentConfig(mocked_config)) + with mock.patch(REPORT_PORTAL_SERVICE + ".get_project_settings"): + service.start() + return service diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 00000000..f29c8e29 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,115 @@ +# Copyright (c) 2022 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +import pytest + +from pytest_reportportal.config import AgentConfig + + +@pytest.mark.parametrize( + ["verify_ssl", "expected_result"], + [ + ("True", True), + ("False", False), + ("true", True), + ("false", False), + (True, True), + (False, False), + ("path/to/certificate", "path/to/certificate"), + (None, True), + ], +) +def test_verify_ssl_true(mocked_config, verify_ssl, expected_result): + mocked_config.getini.side_effect = lambda x: verify_ssl if x == "rp_verify_ssl" else None + config = AgentConfig(mocked_config) + + assert config.rp_verify_ssl == expected_result + + +@pytest.mark.parametrize( + ["env_var", "env_value", "config_attr", "expected"], + [ + ("RP_ENDPOINT", "http://env.example.com", "rp_endpoint", "http://env.example.com"), + ("RP_PROJECT", "env_project", "rp_project", "env_project"), + ("RP_LAUNCH", "env_launch", "rp_launch", "env_launch"), + ("RP_API_KEY", "env_api_key", "rp_api_key", "env_api_key"), + ("RP_LAUNCH_UUID", "env_launch_id", "rp_launch_uuid", "env_launch_id"), + ("RP_MODE", "DEBUG", "rp_mode", "DEBUG"), + ("RP_PARENT_ITEM_ID", "env_parent_id", "rp_parent_item_id", "env_parent_id"), + ("RP_RERUN_OF", "env_rerun_of", "rp_rerun_of", "env_rerun_of"), + ], +) +def test_env_var_overrides_config(monkeypatch, mocked_config, env_var, env_value, config_attr, expected): + monkeypatch.setenv(env_var, env_value) + config = AgentConfig(mocked_config) + assert getattr(config, config_attr) == expected + + +def test_env_var_overrides_verify_ssl(monkeypatch, mocked_config): + monkeypatch.setenv("RP_VERIFY_SSL", "False") + config = AgentConfig(mocked_config) + assert config.rp_verify_ssl is False + + +def test_env_var_overrides_enabled(monkeypatch, mocked_config): + mocked_config.option.rp_enabled = False + monkeypatch.setenv("RP_ENABLED", "True") + config = AgentConfig(mocked_config) + assert config.rp_enabled is True + + +def test_env_var_overrides_log_level(monkeypatch, mocked_config): + import logging + + monkeypatch.setenv("RP_LOG_LEVEL", "ERROR") + config = AgentConfig(mocked_config) + assert config.rp_log_level == logging.ERROR + + +def test_env_var_not_set_falls_back_to_config(mocked_config): + config = AgentConfig(mocked_config) + assert config.rp_endpoint == "http://docker.local:8080/" + + +@pytest.mark.parametrize( + ["option_name", "option_value", "expected_result"], + [ + ("rp_launch_attributes", " smoke ; launch:demo ; smoke ; launch:demo ", ["smoke", "launch:demo"]), + ("rp_tests_attributes", " test:key ; smoke ; test:key ", ["test:key", "smoke"]), + ], +) +def test_string_attributes_are_split_and_deduplicated(mocked_config, option_name, option_value, expected_result): + mocked_config.option.rp_launch_attributes = None + mocked_config.option.rp_tests_attributes = None + mocked_config.getini.side_effect = lambda x: option_value if x == option_name else None + + config = AgentConfig(mocked_config) + + assert getattr(config, option_name) == expected_result + + +@pytest.mark.parametrize( + ["option_name", "option_value"], + [ + ("rp_launch_attributes", ["smoke", "smoke"]), + ("rp_tests_attributes", ["test:key", "test:key"]), + ], +) +def test_attributes_not_split_if_not_string(mocked_config, option_name, option_value): + mocked_config.option.rp_launch_attributes = None + mocked_config.option.rp_tests_attributes = None + mocked_config.getini.side_effect = lambda x: option_value if x == option_name else None + + config = AgentConfig(mocked_config) + + assert getattr(config, option_name) == option_value diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py new file mode 100644 index 00000000..2b0c0385 --- /dev/null +++ b/tests/unit/test_plugin.py @@ -0,0 +1,325 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes unit tests for the plugin.""" + +# noinspection PyUnresolvedReferences +from unittest import mock + +import pytest +from _pytest.config.argparsing import Parser +from delayed_assert import assert_expectations, expect + +from pytest_reportportal.config import AgentConfig +from pytest_reportportal.plugin import ( + FAILED_LAUNCH_WAIT, + LOGGER, + MANDATORY_PARAMETER_MISSED_PATTERN, + is_control, + pytest_addoption, + pytest_collection_finish, + pytest_configure, + pytest_sessionfinish, + pytest_sessionstart, + wait_launch, +) +from pytest_reportportal.service import PyTestService + + +def test_is_control(mocked_config): + """Test is_master() function for the correct responses.""" + mocked_config.workerinput = None + expect(is_control(mocked_config) is False) + delattr(mocked_config, "workerinput") + expect(is_control(mocked_config) is True) + assert_expectations() + + +@mock.patch("reportportal_client.logs.RPLogger.handle") +@pytest.mark.parametrize("log_level", ("info", "debug", "warning", "error")) +def test_logger_handle_attachment(mock_handler, logger, log_level): + """Test logger call for different log levels with some text attachment.""" + log_call = getattr(logger, log_level) + attachment = "Some {} attachment".format(log_level) + log_call("Some {} message".format(log_level), attachment=attachment) + expect(mock_handler.call_count == 1, "logger.handle called more than 1 time") + expect( + getattr(mock_handler.call_args[0][0], "attachment") == attachment, + "record.attachment in args doesn't match real value", + ) + assert_expectations() + + +@mock.patch("reportportal_client.logs.RPLogger.handle") +@pytest.mark.parametrize("log_level", ("info", "debug", "warning", "error")) +def test_logger_handle_no_attachment(mock_handler, logger, log_level): + """Test logger call for different log levels without any attachment.""" + log_call = getattr(logger, log_level) + log_call("Some {} message".format(log_level)) + expect(mock_handler.call_count == 1, "logger.handle called more than 1 time") + expect(getattr(mock_handler.call_args[0][0], "attachment") is None, "record.attachment in args is not None") + assert_expectations() + + +def test_pytest_configure(mocked_config): + """Test plugin successful configuration. + + :param mocked_config: Pytest fixture + """ + mocked_config.option.rp_enabled = True + mocked_config.option.rp_project = None + pytest_configure(mocked_config) + expect(mocked_config._rp_enabled is True) + expect(lambda: isinstance(mocked_config.py_test_service, PyTestService)) + assert_expectations() + mocked_config.getoption.assert_has_calls( + [mock.call("--collect-only", default=False), mock.call("--setup-plan", default=False)] + ) + + +def test_pytest_configure_dry_run(mocked_config): + """Test plugin configuration in case of dry-run execution.""" + mocked_config.getoption.side_effect = lambda opt, default: True + pytest_configure(mocked_config) + assert mocked_config._rp_enabled is False + + +@mock.patch("pytest_reportportal.plugin.LOGGER", wraps=LOGGER) +def test_pytest_configure_misssing_rp_endpoint(mocked_log, mocked_config): + """Test plugin configuration in case of missing rp_endpoint. + + The value of the _reportportal_configured attribute of the pytest Config + object should be changed to False, stopping plugin configuration, if + rp_endpoint is not set. + + :param mocked_config: Pytest fixture + """ + mocked_config.option.rp_enabled = True + mocked_config.option.rp_endpoint = None + mocked_config.getini.return_value = 0 + pytest_configure(mocked_config) + assert mocked_config._rp_enabled is False + mocked_log.debug.assert_has_calls( + [ + mock.call(MANDATORY_PARAMETER_MISSED_PATTERN.format(mocked_config.option.rp_project, None)), + mock.call("Disabling reporting to RP."), + ] + ) + + +@mock.patch("pytest_reportportal.plugin.LOGGER", wraps=LOGGER) +def test_pytest_configure_misssing_rp_project(mocked_log, mocked_config): + """Test plugin configuration in case of missing rp_project. + + The value of the _reportportal_configured attribute of the pytest Config + object should be changed to False, stopping plugin configuration, if + rp_project is not set. + + :param mocked_config: Pytest fixture + """ + mocked_config.option.rp_enabled = True + mocked_config.option.rp_project = None + mocked_config.getini.return_value = 0 + pytest_configure(mocked_config) + assert mocked_config._rp_enabled is False + mocked_log.debug.assert_has_calls( + [ + mock.call(MANDATORY_PARAMETER_MISSED_PATTERN.format(None, mocked_config.option.rp_endpoint)), + mock.call("Disabling reporting to RP."), + ] + ) + + +@mock.patch("pytest_reportportal.plugin.LAUNCH_WAIT_TIMEOUT", 1) +@mock.patch("pytest_reportportal.plugin.time") +def test_wait_launch(time_mock): + """Test wait_launch() function for the correct behavior.""" + time_mock.time.side_effect = [0, 1, 2] + rp_client = mock.Mock() + rp_client.launch_uuid = None + assert not wait_launch(rp_client) + + +def test_pytest_collection_finish(mocked_session): + """Test collection_finish with the configured RP plugin. + + :param mocked_session: pytest fixture + """ + mocked_session.config.py_test_service = mock.Mock() + pytest_collection_finish(mocked_session) + mocked_session.config.py_test_service.collect_tests.assert_called_with(mocked_session) + + +@mock.patch("pytest_reportportal.plugin.wait_launch", mock.Mock(return_value=True)) +@mock.patch("pytest_reportportal.plugin.is_control", mock.Mock()) +def test_pytest_sessionstart(mocked_session): + """Test session configuration if RP plugin is correctly configured. + + :param mocked_session: pytest fixture + """ + mocked_session.config.pluginmanager.hasplugin.return_value = True + mocked_session.config._reporter_config = mock.Mock(spec=AgentConfig(mocked_session.config)) + mocked_session.config._reporter_config.rp_launch_attributes = [] + mocked_session.config.py_test_service = mock.Mock() + pytest_sessionstart(mocked_session) + expect(lambda: mocked_session.config.py_test_service.init_service.called) + expect(lambda: mocked_session.config.py_test_service.rp is not None) + expect(lambda: mocked_session.config._rp_enabled is True) + expect(lambda: mocked_session.config.py_test_service.start_launch.called) + + assert_expectations() + + +@mock.patch("pytest_reportportal.plugin.LOGGER", wraps=LOGGER) +@mock.patch("pytest_reportportal.plugin.is_control", mock.Mock()) +@mock.patch("pytest_reportportal.plugin.wait_launch", mock.Mock(return_value=False)) +def test_pytest_sessionstart_launch_wait_fail(mocked_log, mocked_session): + """Test session configuration if RP plugin is correctly configured. + + :param mocked_session: pytest fixture + """ + mocked_session.config.pluginmanager.hasplugin.return_value = True + mocked_session.config._reporter_config = mock.Mock(spec=AgentConfig(mocked_session.config)) + mocked_session.config._reporter_config.rp_launch_attributes = [] + mocked_session.config._reporter_config.rp_launch_id = None + mocked_session.config.py_test_service = mock.Mock() + pytest_sessionstart(mocked_session) + expect(lambda: mocked_session.config.py_test_service.rp is None) + expect(lambda: mocked_session.config._rp_enabled is False) + assert_expectations() + mocked_log.error.assert_has_calls([mock.call(FAILED_LAUNCH_WAIT)]) + + +@mock.patch("pytest_reportportal.plugin.wait_launch", mock.Mock()) +def test_pytest_sessionstart_xdist(mocked_session): + """Test session configuration if it's worker xdist node. + + :param mocked_session: pytest fixture + """ + mocked_session.config.pluginmanager.hasplugin.return_value = True + mocked_session.config._reporter_config = mock.Mock(spec=AgentConfig(mocked_session.config)) + mocked_session.config._reporter_config.rp_launch_attributes = [] + mocked_session.config._reporter_config.rp_launch_id = 1 + mocked_session.config.workerinput = 1 + mocked_session.config.py_test_service = mock.Mock() + pytest_sessionstart(mocked_session) + expect(lambda: mocked_session.config.py_test_service.start_launch.assert_not_called()) + assert_expectations() + + +@mock.patch("pytest_reportportal.plugin.is_control", mock.Mock()) +def test_pytest_sessionfinish(mocked_session): + """Test sessionfinish with the configured RP plugin. + + :param mocked_session: pytest fixture + """ + mocked_session.config.py_test_service = mock.Mock() + mocked_session.config._reporter_config.rp_launch_uuid = None + pytest_sessionfinish(mocked_session) + assert mocked_session.config.py_test_service.finish_launch.called + + +def test_pytest_addoption_adds_correct_ini_file_arguments(): + """Test the correct list of options are available in the .ini file.""" + mock_parser = mock.MagicMock(spec=Parser) + expected_argument_names = ( + "rp_launch", + "rp_launch_id", + "rp_launch_uuid", + "rp_launch_description", + "rp_project", + "rp_log_level", + "rp_log_format", + "rp_rerun", + "rp_rerun_of", + "rp_parent_item_id", + "rp_api_key", + "rp_endpoint", + "rp_mode", + "rp_thread_logging", + "rp_launch_uuid_print", + "rp_launch_uuid_print_output", + "rp_oauth_uri", + "rp_oauth_username", + "rp_oauth_password", + "rp_oauth_client_id", + "rp_oauth_client_secret", + "rp_oauth_scope", + "rp_launch_attributes", + "rp_tests_attributes", + "rp_log_batch_size", + "rp_log_batch_payload_limit", + "rp_log_batch_payload_size", + "rp_log_custom_levels", + "rp_ignore_attributes", + "rp_is_skipped_an_issue", + "rp_hierarchy_code", + "rp_hierarchy_dirs_level", + "rp_hierarchy_dirs", + "rp_hierarchy_dir_path_separator", + "rp_hierarchy_test_file", + "rp_issue_system_url", + "rp_bts_issue_url", + "rp_bts_project", + "rp_bts_url", + "rp_verify_ssl", + "rp_issue_id_marks", + "rp_api_retries", + "rp_launch_timeout", + "rp_client_type", + "rp_connect_timeout", + "rp_read_timeout", + "rp_report_fixtures", + ) + + pytest_addoption(mock_parser) + + added_argument_names = [] + for args, kwargs in mock_parser.addini.call_args_list: + added_argument_names.append(args[0] if args else kwargs.get("name")) + assert tuple(added_argument_names) == expected_argument_names + + +def test_pytest_addoption_adds_correct_command_line_arguments(): + """Test the correct list of options are available in the command line.""" + expected_argument_names = ( + "--reportportal", + "--rp-launch", + "--rp-launch-id", + "--rp-launch-uuid", + "--rp-launch-description", + "--rp-project", + "--rp-log-level", + "--rp-log-format", + "--rp-rerun", + "--rp-rerun-of", + "--rp-parent-item-id", + "--rp-api-key", + "--rp-endpoint", + "--rp-mode", + "--rp-thread-logging", + "--rp-launch-uuid-print", + "--rp-launch-uuid-print-output", + "--rp-launch-attributes", + "--rp-tests-attributes", + ) + mock_parser = mock.MagicMock(spec=Parser) + mock_reporting_group = mock_parser.getgroup.return_value + + pytest_addoption(mock_parser) + + mock_parser.getgroup.assert_called_once_with("reporting") + added_argument_names = [] + for args, kwargs in mock_reporting_group.addoption.call_args_list: + added_argument_names.append(args[0] if args else kwargs.get("name")) + assert tuple(added_argument_names) == expected_argument_names diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py new file mode 100644 index 00000000..f28e18a7 --- /dev/null +++ b/tests/unit/test_service.py @@ -0,0 +1,87 @@ +# Copyright (c) 2023 https://reportportal.io . +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +"""This module includes unit tests for the service.py module.""" + +import os + +from delayed_assert import assert_expectations, expect + +from pytest_reportportal.service import _is_pytest_bdd_scenario + + +def test_is_pytest_bdd_scenario_path(): + """pytest-bdd scenario items use forward slashes in location on POSIX and backslashes on Windows.""" + path = os.path.join("project", "pytest_bdd", "scenario.py") + assert _is_pytest_bdd_scenario(path) is True + + +def test_is_pytest_bdd_scenario_regular_test_module(): + """Regular tests must not be treated as pytest-bdd scenario glue.""" + assert _is_pytest_bdd_scenario("/project/tests/test_foo.py") is False + + +def test_get_item_parameters(mocked_item, rp_service): + """Test that parameters are returned in a way supported by the client.""" + mocked_item.callspec.params = {"param": "param_value"} + + expect(rp_service._get_parameters(mocked_item) == {"param": "param_value"}) + + delattr(mocked_item, "callspec") + expect(rp_service._get_parameters(mocked_item) is None) + + assert_expectations() + + +def test_get_method_name_regular(mocked_item, rp_service): + """Test that regular test names are returned as-is.""" + mocked_item.name = "test_simple_function" + mocked_item.originalname = None + + result = rp_service._get_method_name(mocked_item) + + expect(result == "test_simple_function") + assert_expectations() + + +def test_get_method_name_uses_originalname(mocked_item, rp_service): + """Test that originalname is preferred when available.""" + mocked_item.name = "test_verify_data[Daily]@sync_group" + mocked_item.originalname = "test_verify_data" + + result = rp_service._get_method_name(mocked_item) + + expect(result == "test_verify_data") + assert_expectations() + + +def test_get_method_name_strips_suffix(mocked_item, rp_service): + """Test that trailing @suffix is stripped when originalname is None.""" + mocked_item.name = "test_export_data@data_export" + mocked_item.originalname = None + + result = rp_service._get_method_name(mocked_item) + + expect(result == "test_export_data") + assert_expectations() + + +def test_get_method_name_preserves_at_inside_params(mocked_item, rp_service): + """Test that @ inside parameter brackets is preserved.""" + mocked_item.name = "test_email[user@example.com]" + mocked_item.originalname = None + + result = rp_service._get_method_name(mocked_item) + + expect(result == "test_email[user@example.com]") + assert_expectations() diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..92c18257 --- /dev/null +++ b/tox.ini @@ -0,0 +1,46 @@ +[tox] +isolated_build = True +envlist = + pep + nobdd + py39 + py310 + py311 + py312 + py313 + py314 + +[testenv] +deps = + -rrequirements.txt + -rrequirements-dev.txt + -rrequirements-dev-bdd.txt + +setenv = + AGENT_NO_ANALYTICS = 1 + +commands = pytest --cov={envsitepackagesdir}/pytest_reportportal --cov-report=xml tests/ -s -vv + +[testenv:nobdd] +deps = + -rrequirements.txt + -rrequirements-dev.txt + +setenv = + AGENT_NO_ANALYTICS = 1 + +commands = pytest tests/ -s -vv --ignore tests/integration/test_bdd.py + +[testenv:pep] +skip_install = True +deps = pre-commit>=1.19.0 +commands = pre-commit run --all-files --show-diff-on-failure + +[gh-actions] +python = + 3.9: py39 + 3.10: pep, nobdd, py310 + 3.11: py311 + 3.12: py312 + 3.13: py313 + 3.14: py314