diff --git a/.vscode/launch.json b/.vscode/launch.json index 1ce725c9..c193749e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -21,7 +21,8 @@ "module": "robotcode.cli", "justMyCode": false, //"cwd": "${workspaceFolder}/tests/robotcode/language_server/robotframework/parts/data/tests", - "cwd": "${workspaceFolder}/..", + //"cwd": "${workspaceFolder}/..", + "cwd": "E:/source/uvtestprj", //"cwd": "C:\\develop\\robot\\robotframework", // "env": { // "ROBOTCODE_COLOR": "1", @@ -56,12 +57,13 @@ // ".." // "config", // "show", - "repl", + "discover", + "tests" // "-d", "output", // "-i", // "-v", "CMD_LINE_VAR:cmd_line_var", // "E:\\source\\uvtestprj\\tests\\first.robotrepl" - "./robotcode/language-configuration.json" + ] }, { diff --git a/CHANGELOG.md b/CHANGELOG.md index f57ab4e2..22a79f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,45 @@ All notable changes to this project will be documented in this file. See [conventional commits](https://www.conventionalcommits.org/) for commit guidelines. +## [0.95.0](https://github.com/robotcodedev/robotcode/compare/v0.94.0..v0.95.0) - 2024-10-25 + +### Bug Fixes + +- **analyzer:** Fix some spellings ([b622c42](https://github.com/robotcodedev/robotcode/commit/b622c42d33bc0a41fab8ffa00cdc74cbb7fe98c8)) +- **analyzer:** Handle bdd prefixes correctly if keyword is cached ([41ff53f](https://github.com/robotcodedev/robotcode/commit/41ff53f9b3c8cc34f06f34ebeaa7c63691544cb2)) +- **analyzer:** Corrected analyzing of `[Return]`, `[Setup]`, `[Teardown]` statement ([4e17c8f](https://github.com/robotcodedev/robotcode/commit/4e17c8fc10814f62799548467efac25c07fbe429)) +- **analyzer:** Corrected exception in parsing ForHeaders with invalid variable ([0851d4f](https://github.com/robotcodedev/robotcode/commit/0851d4f6eb7bd90958996cc9c25506712885dca9)) +- **analyzer:** Fixed find variables as modules for RF > 5 ([ce787b2](https://github.com/robotcodedev/robotcode/commit/ce787b26d6fab981e432166972429aa2d0d84240)) +- **langserver:** Corrected inlay hints for bdd style keyword calls ([77ce8f1](https://github.com/robotcodedev/robotcode/commit/77ce8f1147a4355ea71986408971fca4f441bee5)) +- **langserver:** Only update direct references to a file, not imports if something changes ([ea24b06](https://github.com/robotcodedev/robotcode/commit/ea24b061c77de38e1f755a74979d2632970ff032)) + + +### Features + +- **analyzer:** Implemented better handling of imports of dynamic libraries ([f6b5b87](https://github.com/robotcodedev/robotcode/commit/f6b5b875daf7cd8d59d0cc46ea24a4d07b02c777)) + + - show also errors on in dynamic library API like in `get_keyword_documentation` and `get_keyword_arguments` + +- **discover:** Rework discover commands ([87e1dd9](https://github.com/robotcodedev/robotcode/commit/87e1dd96c525c9639b2727681bac22cc4c3ca8cc)) + + - show statistics on all commands + - better differention of tests and tasks + - new command `tasks` to show only the tasks + - command tests show only the tests + - new arguments for `tags` command `--tests` and `--tags` + - show the type of test or task in test explorer description + + + +### Performance + +- **analyzer:** Restructured code for handling bdd prefixes ([96fbe90](https://github.com/robotcodedev/robotcode/commit/96fbe9064fafe0a64f82b0d95b017726d18381fb)) +- **analyzer:** Optimized analysing keyword calls ([b1f0f28](https://github.com/robotcodedev/robotcode/commit/b1f0f28dfd0c7f38904ae9ecd0c247f5efec2ab1)) +- **analyzer:** Cache embedded arguments and some more little perf tweaks ([3603ff6](https://github.com/robotcodedev/robotcode/commit/3603ff6395d16473fdeb1fca8de910ea3aab8de2)) +- **analyzer:** Introduced some caching for parsing variables ([e39afe9](https://github.com/robotcodedev/robotcode/commit/e39afe92ef9dfd44d33537b4dec9bbec3738d116)) +- **analyzer:** Implemented DataCache, cache files are now saved in pickle format by default instead of json ([f3ecc22](https://github.com/robotcodedev/robotcode/commit/f3ecc221d0018f9264c958074a7458a6e119b1f6)) + + ## [0.94.0](https://github.com/robotcodedev/robotcode/compare/v0.93.1..v0.94.0) - 2024-10-20 ### Bug Fixes diff --git a/eslint.config.mjs b/eslint.config.mjs index 0d253154..52a1c3c7 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -6,6 +6,7 @@ import eslintPluginPrettierRecommended from "eslint-plugin-prettier/recommended" export default [ { ignores: [ + "**/.venv/", "**/node_modules/", "**/dist/", "**/out/", diff --git a/package-lock.json b/package-lock.json index f416a292..54c6ddad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "robotcode", - "version": "0.94.0", + "version": "0.95.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "robotcode", - "version": "0.94.0", + "version": "0.95.0", "funding": [ { "type": "opencollective", diff --git a/package.json b/package.json index 14b68385..8970ab7a 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Robot Framework IntelliSense, linting, test execution and debugging, code formatting, refactoring, and many more", "icon": "images/icon.png", "publisher": "d-biehl", - "version": "0.94.0", + "version": "0.95.0", "author": { "name": "Daniel Biehl", "url": "https://github.com/robotcodedev/" diff --git a/packages/analyze/pyproject.toml b/packages/analyze/pyproject.toml index f5d3b672..9f65f43f 100644 --- a/packages/analyze/pyproject.toml +++ b/packages/analyze/pyproject.toml @@ -27,9 +27,9 @@ classifiers = [ ] dependencies = [ "robotframework>=4.1.0", - "robotcode-plugin==0.94.0", - "robotcode-robot==0.94.0", - "robotcode==0.94.0", + "robotcode-plugin==0.95.0", + "robotcode-robot==0.95.0", + "robotcode==0.95.0", ] dynamic = ["version"] diff --git a/packages/analyze/src/robotcode/analyze/__version__.py b/packages/analyze/src/robotcode/analyze/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/analyze/src/robotcode/analyze/__version__.py +++ b/packages/analyze/src/robotcode/analyze/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/core/src/robotcode/core/__version__.py b/packages/core/src/robotcode/core/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/core/src/robotcode/core/__version__.py +++ b/packages/core/src/robotcode/core/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/debugger/pyproject.toml b/packages/debugger/pyproject.toml index 4d5d48e1..e6fd3b98 100644 --- a/packages/debugger/pyproject.toml +++ b/packages/debugger/pyproject.toml @@ -28,8 +28,8 @@ classifiers = [ dynamic = ["version"] dependencies = [ "robotframework>=4.1.0", - "robotcode-jsonrpc2==0.94.0", - "robotcode-runner==0.94.0", + "robotcode-jsonrpc2==0.95.0", + "robotcode-runner==0.95.0", ] [project.optional-dependencies] diff --git a/packages/debugger/src/robotcode/debugger/__version__.py b/packages/debugger/src/robotcode/debugger/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/debugger/src/robotcode/debugger/__version__.py +++ b/packages/debugger/src/robotcode/debugger/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/jsonrpc2/pyproject.toml b/packages/jsonrpc2/pyproject.toml index a9740cb1..25b7f310 100644 --- a/packages/jsonrpc2/pyproject.toml +++ b/packages/jsonrpc2/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Framework :: Robot Framework", "Framework :: Robot Framework :: Tool", ] -dependencies = ["robotcode-core==0.94.0"] +dependencies = ["robotcode-core==0.95.0"] dynamic = ["version"] [project.urls] diff --git a/packages/jsonrpc2/src/robotcode/jsonrpc2/__version__.py b/packages/jsonrpc2/src/robotcode/jsonrpc2/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/jsonrpc2/src/robotcode/jsonrpc2/__version__.py +++ b/packages/jsonrpc2/src/robotcode/jsonrpc2/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/language_server/pyproject.toml b/packages/language_server/pyproject.toml index a05a5cab..30c48a1e 100644 --- a/packages/language_server/pyproject.toml +++ b/packages/language_server/pyproject.toml @@ -27,10 +27,10 @@ classifiers = [ ] dependencies = [ "robotframework>=4.1.0", - "robotcode-jsonrpc2==0.94.0", - "robotcode-robot==0.94.0", - "robotcode-analyze==0.94.0", - "robotcode==0.94.0", + "robotcode-jsonrpc2==0.95.0", + "robotcode-robot==0.95.0", + "robotcode-analyze==0.95.0", + "robotcode==0.95.0", ] dynamic = ["version"] diff --git a/packages/language_server/src/robotcode/language_server/__version__.py b/packages/language_server/src/robotcode/language_server/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/language_server/src/robotcode/language_server/__version__.py +++ b/packages/language_server/src/robotcode/language_server/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py index 2c5c0b24..0632f78e 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py @@ -2144,7 +2144,7 @@ def _complete_KeywordCall_or_Fixture( # noqa: N802 keyword_token, [t for t in kw_node.get_tokens(Token.ARGUMENT)], self.namespace, - range_from_token(keyword_token).start, + range_from_token(keyword_token).end, analyse_run_keywords=False, ) diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py index 4105bcbd..ffdf2b0d 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py @@ -11,7 +11,6 @@ Range, ) from robotcode.core.text_document import TextDocument -from robotcode.core.uri import Uri from robotcode.core.utils.logging import LoggingDescriptor from robotcode.language_server.robotframework.configuration import AnalysisConfig from robotcode.robot.diagnostics.entities import ( @@ -87,13 +86,6 @@ def _on_get_related_documents(self, sender: Any, document: TextDocument) -> Opti result = [] - resources = namespace.get_resources().values() - for r in resources: - if r.library_doc.source: - doc = self.parent.documents.get(Uri.from_path(r.library_doc.source).normalized()) - if doc is not None: - result.append(doc) - lib_doc = namespace.get_library_doc() for doc in self.parent.documents.documents: if doc.language_id != "robotframework": diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py index 7d99bf9e..eed9d5e3 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py @@ -107,13 +107,13 @@ def _handle_keywordcall_fixture_template( keyword_token, arguments, namespace, - range_from_token(keyword_token).start, + range_from_token(keyword_token).end, ) if kw_result is None: return None - kw_doc, _ = kw_result + kw_doc, keyword_token = kw_result if kw_doc is None: return None diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/semantic_tokens.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/semantic_tokens.py index b08467e2..22843d50 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/semantic_tokens.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/semantic_tokens.py @@ -36,8 +36,7 @@ Variable, VariablesImport, ) -from robot.utils.escaping import split_from_equals, unescape -from robot.variables.search import is_variable +from robot.utils.escaping import unescape from robotcode.core.concurrent import check_current_task_canceled from robotcode.core.language import language_id @@ -70,6 +69,7 @@ iter_over_keyword_names_and_owners, token_in_range, ) +from robotcode.robot.utils.variables import is_variable, split_from_equals from .protocol_part import RobotLanguageServerProtocolPart @@ -1120,13 +1120,13 @@ def get_tokens() -> Iterator[Tuple[Token, ast.AST]]: if kw_token is not None: kw: Optional[str] = None - for _, name in iter_over_keyword_names_and_owners( + for _, n in iter_over_keyword_names_and_owners( ModelHelper.strip_bdd_prefix(namespace, kw_token).value ): - if name is not None: - matcher = KeywordMatcher(name) + if n is not None: + matcher = KeywordMatcher(n) if matcher in ALL_RUN_KEYWORDS_MATCHERS: - kw = name + kw = n if kw: kw_doc = namespace.find_keyword(kw_token.value) if kw_doc is not None and kw_doc.is_any_run_keyword(): diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py index 83e58391..e03b650d 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py @@ -135,7 +135,7 @@ def _signature_help_KeywordCall_or_Fixture( # noqa: N802 keyword_token, [t for t in kw_node.get_tokens(RobotToken.ARGUMENT)], namespace, - range_from_token(keyword_token).start, + range_from_token(keyword_token).end, analyse_run_keywords=False, ) diff --git a/packages/modifiers/src/robotcode/modifiers/__version__.py b/packages/modifiers/src/robotcode/modifiers/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/modifiers/src/robotcode/modifiers/__version__.py +++ b/packages/modifiers/src/robotcode/modifiers/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/plugin/src/robotcode/plugin/__version__.py b/packages/plugin/src/robotcode/plugin/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/plugin/src/robotcode/plugin/__version__.py +++ b/packages/plugin/src/robotcode/plugin/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/repl/pyproject.toml b/packages/repl/pyproject.toml index 0a6a8e9d..b23de43a 100644 --- a/packages/repl/pyproject.toml +++ b/packages/repl/pyproject.toml @@ -27,8 +27,8 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "robotcode-jsonrpc2==0.94.0", - "robotcode-runner==0.94.0" + "robotcode-jsonrpc2==0.95.0", + "robotcode-runner==0.95.0" ] [project.entry-points.robotcode] diff --git a/packages/repl/src/robotcode/repl/__version__.py b/packages/repl/src/robotcode/repl/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/repl/src/robotcode/repl/__version__.py +++ b/packages/repl/src/robotcode/repl/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/robot/pyproject.toml b/packages/robot/pyproject.toml index 375c2b30..01598c5b 100644 --- a/packages/robot/pyproject.toml +++ b/packages/robot/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "robotframework>=4.1.0", "tomli>=1.1.0; python_version < '3.11'", "platformdirs>=3.2.0,<4.2.0", - "robotcode-core==0.94.0", + "robotcode-core==0.95.0", ] dynamic = ["version"] diff --git a/packages/robot/src/robotcode/robot/__version__.py b/packages/robot/src/robotcode/robot/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/robot/src/robotcode/robot/__version__.py +++ b/packages/robot/src/robotcode/robot/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/robot/src/robotcode/robot/diagnostics/data_cache.py b/packages/robot/src/robotcode/robot/diagnostics/data_cache.py new file mode 100644 index 00000000..11d8f90c --- /dev/null +++ b/packages/robot/src/robotcode/robot/diagnostics/data_cache.py @@ -0,0 +1,83 @@ +import pickle +from abc import ABC, abstractmethod +from enum import Enum +from pathlib import Path +from typing import Any, Tuple, Type, TypeVar, Union, cast + +from robotcode.core.utils.dataclasses import as_json, from_json + +_T = TypeVar("_T") + + +class CacheSection(Enum): + LIBRARY = "libdoc" + VARIABLES = "variables" + + +class DataCache(ABC): + @abstractmethod + def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool: ... + + @abstractmethod + def read_cache_data( + self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]] + ) -> _T: ... + + @abstractmethod + def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None: ... + + +class JsonDataCache(DataCache): + def __init__(self, cache_dir: Path) -> None: + self.cache_dir = cache_dir + + def build_cache_data_filename(self, section: CacheSection, entry_name: str) -> Path: + return self.cache_dir / section.value / (entry_name + ".json") + + def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool: + cache_file = self.build_cache_data_filename(section, entry_name) + return cache_file.exists() + + def read_cache_data( + self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]] + ) -> _T: + cache_file = self.build_cache_data_filename(section, entry_name) + return from_json(cache_file.read_text("utf-8"), types) + + def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None: + cached_file = self.build_cache_data_filename(section, entry_name) + + cached_file.parent.mkdir(parents=True, exist_ok=True) + cached_file.write_text(as_json(data), "utf-8") + + +class PickleDataCache(DataCache): + def __init__(self, cache_dir: Path) -> None: + self.cache_dir = cache_dir + + def build_cache_data_filename(self, section: CacheSection, entry_name: str) -> Path: + return self.cache_dir / section.value / (entry_name + ".pkl") + + def cache_data_exists(self, section: CacheSection, entry_name: str) -> bool: + cache_file = self.build_cache_data_filename(section, entry_name) + return cache_file.exists() + + def read_cache_data( + self, section: CacheSection, entry_name: str, types: Union[Type[_T], Tuple[Type[_T], ...]] + ) -> _T: + cache_file = self.build_cache_data_filename(section, entry_name) + + with cache_file.open("rb") as f: + result = pickle.load(f) + + if isinstance(result, types): + return cast(_T, result) + + raise TypeError(f"Expected {types} but got {type(result)}") + + def save_cache_data(self, section: CacheSection, entry_name: str, data: Any) -> None: + cached_file = self.build_cache_data_filename(section, entry_name) + + cached_file.parent.mkdir(parents=True, exist_ok=True) + with cached_file.open("wb") as f: + pickle.dump(data, f) diff --git a/packages/robot/src/robotcode/robot/diagnostics/entities.py b/packages/robot/src/robotcode/robot/diagnostics/entities.py index b8cd95ad..b5905c54 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/entities.py +++ b/packages/robot/src/robotcode/robot/diagnostics/entities.py @@ -12,11 +12,11 @@ ) from robot.parsing.lexer.tokens import Token -from robot.variables.search import search_variable from robotcode.core.lsp.types import Position, Range from robotcode.robot.utils.match import normalize from ..utils.ast import range_from_token +from ..utils.variables import search_variable if TYPE_CHECKING: from robotcode.robot.diagnostics.library_doc import KeywordDoc, LibraryDoc diff --git a/packages/robot/src/robotcode/robot/diagnostics/errors.py b/packages/robot/src/robotcode/robot/diagnostics/errors.py index 5ec317ac..52d64217 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/errors.py +++ b/packages/robot/src/robotcode/robot/diagnostics/errors.py @@ -6,7 +6,7 @@ @final class Error: VARIABLE_NOT_FOUND = "VariableNotFound" - ENVIROMMENT_VARIABLE_NOT_FOUND = "EnvirommentVariableNotFound" + ENVIRONMENT_VARIABLE_NOT_FOUND = "EnvironmentVariableNotFound" KEYWORD_NOT_FOUND = "KeywordNotFound" LIBRARY_CONTAINS_NO_KEYWORDS = "LibraryContainsNoKeywords" POSSIBLE_CIRCULAR_IMPORT = "PossibleCircularImport" diff --git a/packages/robot/src/robotcode/robot/diagnostics/imports_manager.py b/packages/robot/src/robotcode/robot/diagnostics/imports_manager.py index 17abc956..de4da2d3 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/imports_manager.py +++ b/packages/robot/src/robotcode/robot/diagnostics/imports_manager.py @@ -25,6 +25,7 @@ final, ) +from robot.libraries import STDLIBS from robot.utils.text import split_args_from_name_or_path from robotcode.core.concurrent import RLock, run_as_task from robotcode.core.documents_manager import DocumentsManager @@ -35,14 +36,16 @@ from robotcode.core.text_document import TextDocument from robotcode.core.uri import Uri from robotcode.core.utils.caching import SimpleLRUCache -from robotcode.core.utils.dataclasses import as_json, from_json -from robotcode.core.utils.glob_path import Pattern, iter_files +from robotcode.core.utils.glob_path import Pattern from robotcode.core.utils.logging import LoggingDescriptor from robotcode.core.utils.path import normalized_path, path_is_relative_to from ..__version__ import __version__ from ..utils import get_robot_version, get_robot_version_str from ..utils.robot_path import find_file_ex +from ..utils.variables import contains_variable +from .data_cache import CacheSection +from .data_cache import PickleDataCache as DefaultDataCache from .entities import ( CommandLineVariableDefinition, VariableDefinition, @@ -521,18 +524,10 @@ def __init__( self._logger.trace(lambda: f"use {cache_base_path} as base for caching") self.cache_path = cache_base_path / ".robotcode_cache" - - self.lib_doc_cache_path = ( - self.cache_path - / f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - / get_robot_version_str() - / "libdoc" - ) - self.variables_doc_cache_path = ( + self.data_cache = DefaultDataCache( self.cache_path / f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" / get_robot_version_str() - / "variables" ) self.cmd_variables = variables @@ -564,9 +559,9 @@ def __init__( if environment: self._environment.update(environment) - self._library_files_cache = SimpleLRUCache(1024) - self._resource_files_cache = SimpleLRUCache(1024) - self._variables_files_cache = SimpleLRUCache(1024) + self._library_files_cache = SimpleLRUCache(2048) + self._resource_files_cache = SimpleLRUCache(2048) + self._variables_files_cache = SimpleLRUCache(2048) self._executor_lock = RLock(default_timeout=120, name="ImportsManager._executor_lock") self._executor: Optional[ProcessPoolExecutor] = None @@ -582,6 +577,8 @@ def __init__( weakref.WeakKeyDictionary() ) + self._process_pool_executor: Optional[ProcessPoolExecutor] = None + def __del__(self) -> None: try: if self._executor is not None: @@ -899,12 +896,14 @@ def get_library_meta( ) if result is not None: + # TODO: use IgnoreSpec instead of this ignore_arguments = any( (p.matches(result.name) if result.name is not None else False) or (p.matches(result.origin) if result.origin is not None else False) for p in self.ignore_arguments_for_library_patters ) + # TODO: use IgnoreSpec instead of this if any( (p.matches(result.name) if result.name is not None else False) or (p.matches(result.origin) if result.origin is not None else False) @@ -918,16 +917,16 @@ def get_library_meta( return None, import_name, ignore_arguments if result.origin is not None: - result.mtimes = {result.origin: Path(result.origin).stat().st_mtime_ns} + result.mtimes = {result.origin: os.stat(result.origin, follow_symlinks=False).st_mtime_ns} if result.submodule_search_locations: if result.mtimes is None: result.mtimes = {} result.mtimes.update( { - str(f): f.stat().st_mtime_ns + str(f): os.stat(f, follow_symlinks=False).st_mtime_ns for f in itertools.chain( - *(iter_files(loc, "**/*.py") for loc in result.submodule_search_locations) + *(Path(loc).rglob("**/*.py") for loc in result.submodule_search_locations) ) } ) @@ -987,16 +986,16 @@ def get_variables_meta( return None, import_name if result.origin is not None: - result.mtimes = {result.origin: Path(result.origin).stat().st_mtime_ns} + result.mtimes = {result.origin: os.stat(result.origin, follow_symlinks=False).st_mtime_ns} if result.submodule_search_locations: if result.mtimes is None: result.mtimes = {} result.mtimes.update( { - str(f): f.stat().st_mtime_ns + str(f): os.stat(f, follow_symlinks=False).st_mtime_ns for f in itertools.chain( - *(iter_files(loc, "**/*.py") for loc in result.submodule_search_locations) + *(Path(loc).rglob("**/*.py") for loc in result.submodule_search_locations) ) } ) @@ -1015,7 +1014,10 @@ def find_library( base_dir: str, variables: Optional[Dict[str, Any]] = None, ) -> str: - return self._library_files_cache.get(self._find_library, name, base_dir, variables) + if contains_variable(name, "$@&%"): + return self._library_files_cache.get(self._find_library, name, base_dir, variables) + + return self._library_files_cache.get(self._find_library_simple, name, base_dir) def _find_library( self, @@ -1023,17 +1025,19 @@ def _find_library( base_dir: str, variables: Optional[Dict[str, Any]] = None, ) -> str: - from robot.libraries import STDLIBS - from robot.variables.search import contains_variable + return find_library( + name, + str(self.root_folder), + base_dir, + self.get_resolvable_command_line_variables(), + variables, + ) - if contains_variable(name, "$@&%"): - return find_library( - name, - str(self.root_folder), - base_dir, - self.get_resolvable_command_line_variables(), - variables, - ) + def _find_library_simple( + self, + name: str, + base_dir: str, + ) -> str: if name in STDLIBS: result = ROBOT_LIBRARY_PACKAGE + "." + name @@ -1052,7 +1056,10 @@ def find_resource( file_type: str = "Resource", variables: Optional[Dict[str, Any]] = None, ) -> str: - return self._resource_files_cache.get(self.__find_resource, name, base_dir, file_type, variables) + if contains_variable(name, "$@&%"): + return self._resource_files_cache.get(self.__find_resource, name, base_dir, file_type, variables) + + return self._resource_files_cache.get(self.__find_resource_simple, name, base_dir, file_type) @_logger.call def __find_resource( @@ -1062,64 +1069,71 @@ def __find_resource( file_type: str = "Resource", variables: Optional[Dict[str, Any]] = None, ) -> str: - from robot.variables.search import contains_variable + return find_file( + name, + str(self.root_folder), + base_dir, + self.get_resolvable_command_line_variables(), + variables, + file_type, + ) - if contains_variable(name, "$@&%"): - return find_file( + def __find_resource_simple( + self, + name: str, + base_dir: str, + file_type: str = "Resource", + ) -> str: + return find_file_ex(name, base_dir, file_type) + + def find_variables( + self, + name: str, + base_dir: str, + variables: Optional[Dict[str, Any]] = None, + resolve_variables: bool = True, + resolve_command_line_vars: bool = True, + ) -> str: + if resolve_variables and contains_variable(name, "$@&%"): + return self._variables_files_cache.get( + self.__find_variables, name, - str(self.root_folder), base_dir, - self.get_resolvable_command_line_variables(), variables, - file_type, + resolve_command_line_vars, ) + return self._variables_files_cache.get(self.__find_variables_simple, name, base_dir) - return str(find_file_ex(name, base_dir, file_type)) - - def find_variables( + @_logger.call + def __find_variables( self, name: str, base_dir: str, variables: Optional[Dict[str, Any]] = None, - resolve_variables: bool = True, resolve_command_line_vars: bool = True, ) -> str: - return self._variables_files_cache.get( - self.__find_variables, + return find_variables( name, + str(self.root_folder), base_dir, + self.get_resolvable_command_line_variables() if resolve_command_line_vars else None, variables, - resolve_variables, - resolve_command_line_vars, ) @_logger.call - def __find_variables( + def __find_variables_simple( self, name: str, base_dir: str, - variables: Optional[Dict[str, Any]] = None, - resolve_variables: bool = True, - resolve_command_line_vars: bool = True, ) -> str: - from robot.variables.search import contains_variable - - if resolve_variables and contains_variable(name, "$@&%"): - return find_variables( - name, - str(self.root_folder), - base_dir, - self.get_resolvable_command_line_variables() if resolve_command_line_vars else None, - variables, - ) if get_robot_version() >= (5, 0): if is_variables_by_path(name): - return str(find_file_ex(name, base_dir, "Variables")) + return find_file_ex(name, base_dir, "Variables") return name - return str(find_file_ex(name, base_dir, "Variables")) + return find_file_ex(name, base_dir, "Variables") @property def executor(self) -> ProcessPoolExecutor: @@ -1155,29 +1169,26 @@ def _get_library_libdoc( if meta is not None and not meta.has_errors: - meta_file = Path(self.lib_doc_cache_path, meta.filepath_base + ".meta.json") - if meta_file.exists(): + meta_file = meta.filepath_base + ".meta" + if self.data_cache.cache_data_exists(CacheSection.LIBRARY, meta_file): try: spec_path = None try: - saved_meta = from_json(meta_file.read_text("utf-8"), LibraryMetaData) + saved_meta = self.data_cache.read_cache_data(CacheSection.LIBRARY, meta_file, LibraryMetaData) if saved_meta.has_errors: self._logger.debug( - lambda: "Saved library spec for {name}{args!r} is not used " + lambda: f"Saved library spec for {name}{args!r} is not used " "due to errors in meta data", context_name="import", ) if not saved_meta.has_errors and saved_meta == meta: - spec_path = Path( - self.lib_doc_cache_path, - meta.filepath_base + ".spec.json", - ) + spec_path = meta.filepath_base + ".spec" self._logger.debug( lambda: f"Use cached library meta data for {name}", context_name="import" ) - return from_json(spec_path.read_text("utf-8"), LibraryDoc) + return self.data_cache.read_cache_data(CacheSection.LIBRARY, spec_path, LibraryDoc) except (SystemExit, KeyboardInterrupt): raise @@ -1191,6 +1202,9 @@ def _get_library_libdoc( self._logger.exception(e) self._logger.debug(lambda: f"Load library in process {name}{args!r}", context_name="import") + # if self._process_pool_executor is None: + # self._process_pool_executor = ProcessPoolExecutor(max_workers=1, mp_context=mp.get_context("spawn")) + # executor = self._process_pool_executor executor = ProcessPoolExecutor(max_workers=1, mp_context=mp.get_context("spawn")) try: try: @@ -1222,19 +1236,17 @@ def _get_library_libdoc( if meta is not None: meta.has_errors = bool(result.errors) - meta_file = Path(self.lib_doc_cache_path, meta.filepath_base + ".meta.json") - spec_file = Path(self.lib_doc_cache_path, meta.filepath_base + ".spec.json") - - spec_file.parent.mkdir(parents=True, exist_ok=True) + meta_file = meta.filepath_base + ".meta" + spec_file = meta.filepath_base + ".spec" try: - spec_file.write_text(as_json(result), "utf-8") + self.data_cache.save_cache_data(CacheSection.LIBRARY, spec_file, result) except (SystemExit, KeyboardInterrupt): raise except BaseException as e: raise RuntimeError(f"Cannot write spec file for library '{name}' to '{spec_file}'") from e - meta_file.write_text(as_json(meta), "utf-8") + self.data_cache.save_cache_data(CacheSection.LIBRARY, meta_file, meta) else: self._logger.debug(lambda: f"Skip caching library {name}{args!r}", context_name="import") except (SystemExit, KeyboardInterrupt): @@ -1351,21 +1363,17 @@ def _get_variables_libdoc( ) if meta is not None: - meta_file = Path( - self.variables_doc_cache_path, - meta.filepath_base + ".meta.json", - ) - if meta_file.exists(): + meta_file = meta.filepath_base + ".meta" + + if self.data_cache.cache_data_exists(CacheSection.VARIABLES, meta_file): try: spec_path = None try: - saved_meta = from_json(meta_file.read_text("utf-8"), LibraryMetaData) + saved_meta = self.data_cache.read_cache_data(CacheSection.VARIABLES, meta_file, LibraryMetaData) if saved_meta == meta: - spec_path = Path( - self.variables_doc_cache_path, - meta.filepath_base + ".spec.json", - ) - return from_json(spec_path.read_text("utf-8"), VariablesDoc) + spec_path = meta.filepath_base + ".spec" + + return self.data_cache.read_cache_data(CacheSection.VARIABLES, spec_path, VariablesDoc) except (SystemExit, KeyboardInterrupt): raise except BaseException as e: @@ -1406,23 +1414,16 @@ def _get_variables_libdoc( try: if meta is not None: - meta_file = Path( - self.variables_doc_cache_path, - meta.filepath_base + ".meta.json", - ) - spec_file = Path( - self.variables_doc_cache_path, - meta.filepath_base + ".spec.json", - ) - spec_file.parent.mkdir(parents=True, exist_ok=True) + meta_file = meta.filepath_base + ".meta" + spec_file = meta.filepath_base + ".spec" try: - spec_file.write_text(as_json(result), "utf-8") + self.data_cache.save_cache_data(CacheSection.VARIABLES, spec_file, result) except (SystemExit, KeyboardInterrupt): raise except BaseException as e: raise RuntimeError(f"Cannot write spec file for variables '{name}' to '{spec_file}'") from e - meta_file.write_text(as_json(meta), "utf-8") + self.data_cache.save_cache_data(CacheSection.VARIABLES, meta_file, meta) else: self._logger.debug(lambda: f"Skip caching variables {name}{args!r}", context_name="import") except (SystemExit, KeyboardInterrupt): diff --git a/packages/robot/src/robotcode/robot/diagnostics/keyword_finder.py b/packages/robot/src/robotcode/robot/diagnostics/keyword_finder.py index 23ff7cc0..051802a0 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/keyword_finder.py +++ b/packages/robot/src/robotcode/robot/diagnostics/keyword_finder.py @@ -1,3 +1,5 @@ +import functools +import re from itertools import chain from typing import TYPE_CHECKING, Dict, Iterable, Iterator, List, NamedTuple, Optional, Sequence, Tuple @@ -42,6 +44,8 @@ def __init__(self, namespace: "Namespace", library_doc: LibraryDoc) -> None: self.self_library_doc = library_doc self.diagnostics: List[DiagnosticsEntry] = [] + self.result_bdd_prefix: Optional[str] = None + self.multiple_keywords_result: Optional[List[KeywordDoc]] = None self._cache: Dict[ Tuple[Optional[str], bool], @@ -49,9 +53,10 @@ def __init__(self, namespace: "Namespace", library_doc: LibraryDoc) -> None: Optional[KeywordDoc], List[DiagnosticsEntry], Optional[List[KeywordDoc]], + Optional[str], ], ] = {} - self.handle_bdd_style = True + self._all_keywords: Optional[List[LibraryEntry]] = None self._resource_keywords: Optional[List[ResourceEntry]] = None self._library_keywords: Optional[List[LibraryEntry]] = None @@ -59,7 +64,9 @@ def __init__(self, namespace: "Namespace", library_doc: LibraryDoc) -> None: def reset_diagnostics(self) -> None: self.diagnostics = [] self.multiple_keywords_result = None + self.result_bdd_prefix = None + # TODO: make this threadsafe def find_keyword( self, name: Optional[str], @@ -70,17 +77,16 @@ def find_keyword( try: self.reset_diagnostics() - self.handle_bdd_style = handle_bdd_style - - cached = self._cache.get((name, self.handle_bdd_style), None) + cached = self._cache.get((name, handle_bdd_style), None) if cached is not None: self.diagnostics = cached[1] self.multiple_keywords_result = cached[2] + self.result_bdd_prefix = cached[3] return cached[0] try: - result = self._find_keyword(name) + result = self._find_keyword(name, handle_bdd_style) if result is None: self.diagnostics.append( DiagnosticsEntry( @@ -99,17 +105,22 @@ def find_keyword( result = None self.diagnostics.append(DiagnosticsEntry(str(e), DiagnosticSeverity.ERROR, Error.KEYWORD_ERROR)) - self._cache[(name, self.handle_bdd_style)] = ( + self._cache[(name, handle_bdd_style)] = ( result, self.diagnostics, self.multiple_keywords_result, + self.result_bdd_prefix, ) return result except CancelSearchError: return None - def _find_keyword(self, name: Optional[str]) -> Optional[KeywordDoc]: + def _find_keyword( + self, + name: Optional[str], + handle_bdd_style: bool = True, + ) -> Optional[KeywordDoc]: if not name: self.diagnostics.append( DiagnosticsEntry( @@ -129,14 +140,21 @@ def _find_keyword(self, name: Optional[str]) -> Optional[KeywordDoc]: ) raise CancelSearchError - result = self._get_keyword_from_self(name) + result: Optional[KeywordDoc] = None + + if get_robot_version() >= (7, 0) and handle_bdd_style: + result = self._get_bdd_style_keyword(name) + + if not result: + result = self._get_keyword_from_self(name) + if not result and "." in name: result = self._get_explicit_keyword(name) if not result: result = self._get_implicit_keyword(name) - if not result and self.handle_bdd_style: + if get_robot_version() < (7, 0) and not result and handle_bdd_style: return self._get_bdd_style_keyword(name) return result @@ -264,6 +282,9 @@ def _prioritize_same_file_or_public( def _select_best_matches( self, entries: List[Tuple[Optional[LibraryEntry], KeywordDoc]] ) -> List[Tuple[Optional[LibraryEntry], KeywordDoc]]: + if len(entries) < 2: + return entries + normal = [hand for hand in entries if not hand[1].is_embedded] if normal: return normal @@ -438,21 +459,27 @@ def _create_custom_and_standard_keyword_conflict_warning_message( f"or '{'' if standard[0] is None else standard[0].alias or standard[0].name}.{standard[1].name}'." ) + @functools.cached_property + def bdd_prefix_regexp(self) -> "re.Pattern[str]": + prefixes = ( + "|".join( + self.namespace.languages.bdd_prefixes + if self.namespace.languages is not None + else ["given", "when", "then", "and", "but"] + ) + .replace(" ", r"\s") + .lower() + ) + return re.compile(rf"({prefixes})\s", re.IGNORECASE) + def _get_bdd_style_keyword(self, name: str) -> Optional[KeywordDoc]: - if get_robot_version() < (6, 0): - lower = name.lower() - for prefix in ["given ", "when ", "then ", "and ", "but "]: - if lower.startswith(prefix): - return self._find_keyword(name[len(prefix) :]) - return None + match = self.bdd_prefix_regexp.match(name) + if match: + result = self._find_keyword( + name[match.end() :], handle_bdd_style=False if get_robot_version() >= (7, 0) else True + ) + if result: + self.result_bdd_prefix = str(match.group(0)) - parts = name.split() - if len(parts) < 2: - return None - for index in range(1, len(parts)): - prefix = " ".join(parts[:index]).title() - if prefix.title() in ( - self.namespace.languages.bdd_prefixes if self.namespace.languages is not None else DEFAULT_BDD_PREFIXES - ): - return self._find_keyword(" ".join(parts[index:])) + return result return None diff --git a/packages/robot/src/robotcode/robot/diagnostics/library_doc.py b/packages/robot/src/robotcode/robot/diagnostics/library_doc.py index 3a79948c..c8169b76 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/library_doc.py +++ b/packages/robot/src/robotcode/robot/diagnostics/library_doc.py @@ -1,6 +1,7 @@ from __future__ import annotations import ast +import functools import hashlib import importlib import importlib.util @@ -61,7 +62,6 @@ from robot.variables.filesetter import PythonImporter, YamlImporter from robot.variables.finders import VariableFinder from robot.variables.replacer import VariableReplacer -from robot.variables.search import contains_variable from robotcode.core.lsp.types import Position, Range from robotcode.core.utils.path import normalized_path from robotcode.robot.diagnostics.entities import ( @@ -83,6 +83,8 @@ from robotcode.robot.utils.match import normalize, normalize_namespace from robotcode.robot.utils.stubs import HasError, HasErrors +from ..utils.variables import contains_variable + if get_robot_version() < (7, 0): from robot.running.handlers import _PythonHandler, _PythonInitHandler # pyright: ignore[reportMissingImports] from robot.running.model import ResourceFile # pyright: ignore[reportMissingImports] @@ -197,14 +199,29 @@ def convert_from_rest(text: str) -> str: return text +if get_robot_version() >= (6, 0): + + @functools.lru_cache(maxsize=None) + def _get_embedded_arguments(name: str) -> Any: + try: + return EmbeddedArguments.from_name(name) + except (VariableError, DataError): + return () + +else: + + @functools.lru_cache(maxsize=None) + def _get_embedded_arguments(name: str) -> Any: + try: + return EmbeddedArguments(name) + except (VariableError, DataError): + return () + + def is_embedded_keyword(name: str) -> bool: try: - if get_robot_version() >= (6, 0): - if EmbeddedArguments.from_name(name): - return True - else: - if EmbeddedArguments(name): - return True + if _get_embedded_arguments(name): + return True except (VariableError, DataError): return True @@ -235,18 +252,22 @@ def normalized_name(self) -> str: def embedded_arguments(self) -> Any: if self._embedded_arguments is None: if self._can_have_embedded: - try: - if get_robot_version() >= (6, 0): - self._embedded_arguments = EmbeddedArguments.from_name(self.name) - else: - self._embedded_arguments = EmbeddedArguments(self.name) - except (VariableError, DataError): - self._embedded_arguments = () + self._embedded_arguments = _get_embedded_arguments(self.name) else: self._embedded_arguments = () return self._embedded_arguments + if get_robot_version() >= (6, 0): + + def __match_embedded(self, name: str) -> bool: + return self.embedded_arguments.match(name) is not None + + else: + + def __match_embedded(self, name: str) -> bool: + return self.embedded_arguments.name.match(name) is not None + def __eq__(self, o: object) -> bool: if cached_isinstance(o, KeywordMatcher): if self._is_namespace != o._is_namespace: @@ -261,10 +282,7 @@ def __eq__(self, o: object) -> bool: return False if self.embedded_arguments: - if get_robot_version() >= (6, 0): - return self.embedded_arguments.match(o) is not None - - return self.embedded_arguments.name.match(o) is not None + return self.__match_embedded(o) return self.normalized_name == str(normalize_namespace(o) if self._is_namespace else normalize(o)) @@ -908,8 +926,7 @@ def _matchers(self) -> Dict[KeywordMatcher, KeywordDoc]: return self.__matchers def __getitem__(self, key: str) -> KeywordDoc: - key_matcher = KeywordMatcher(key) - items = [(k, v) for k, v in self._matchers.items() if k == key_matcher] + items = [(k, v) for k, v in self._matchers.items() if k == key] if not items: raise KeyError @@ -936,8 +953,6 @@ def __getitem__(self, key: str) -> KeywordDoc: ) def __contains__(self, _x: object) -> bool: - if not isinstance(_x, KeywordMatcher): - _x = KeywordMatcher(str(_x)) return any(k == _x for k in self._matchers.keys()) def __len__(self) -> int: @@ -968,8 +983,7 @@ def get_all(self, key: str) -> List[KeywordDoc]: return list(self.iter_all(key)) def iter_all(self, key: str) -> Iterable[KeywordDoc]: - key_matcher = KeywordMatcher(key) - yield from (v for k, v in self._matchers.items() if k == key_matcher) + yield from (v for k, v in self._matchers.items() if k == key) @dataclass @@ -1282,10 +1296,12 @@ def to_markdown( return result +@functools.lru_cache(maxsize=256) def is_library_by_path(path: str) -> bool: return path.lower().endswith((".py", "/", os.sep)) +@functools.lru_cache(maxsize=256) def is_variables_by_path(path: str) -> bool: if get_robot_version() >= (6, 1): return path.lower().endswith((".py", ".yml", ".yaml", ".json", "/", os.sep)) @@ -1656,9 +1672,8 @@ def _find_library_internal( robot_variables = None - robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) - if contains_variable(name, "$@&%"): + robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) try: name = robot_variables.replace_string(name, ignore_errors=False) except DataError as error: @@ -1711,6 +1726,52 @@ def get_robot_library_html_doc_str( return output.getvalue() +class _Logger(AbstractLogger): + def __init__(self) -> None: + super().__init__() + self.messages: List[Tuple[str, str, bool]] = [] + + def write(self, message: str, level: str, html: bool = False) -> None: + self.messages.append((message, level, html)) + + +def _import_test_library(name: str) -> Union[Any, Tuple[Any, str]]: + with OutputCapturer(library_import=True): + importer = Importer("test library", LOGGER) + return importer.import_class_or_module(name, return_source=True) + + +def _get_test_library( + libcode: Any, + source: str, + name: str, + args: Optional[Tuple[Any, ...]] = None, + variables: Optional[Dict[str, Optional[Any]]] = None, + create_handlers: bool = True, + logger: Any = LOGGER, +) -> Any: + if get_robot_version() < (7, 0): + libclass = robot.running.testlibraries._get_lib_class(libcode) + lib = libclass(libcode, name, args or [], source, logger, variables) + if create_handlers: + lib.create_handlers() + else: + lib = robot.running.testlibraries.TestLibrary.from_code( + libcode, + name, + source=Path(source), + args=args or [], + variables=variables, + create_keywords=create_handlers, + logger=logger, + ) + + return lib + + +_T = TypeVar("_T") + + def get_library_doc( name: str, args: Optional[Tuple[Any, ...]] = None, @@ -1719,45 +1780,6 @@ def get_library_doc( command_line_variables: Optional[Dict[str, Optional[Any]]] = None, variables: Optional[Dict[str, Optional[Any]]] = None, ) -> LibraryDoc: - class Logger(AbstractLogger): - def __init__(self) -> None: - super().__init__() - self.messages: List[Tuple[str, str, bool]] = [] - - def write(self, message: str, level: str, html: bool = False) -> None: - self.messages.append((message, level, html)) - - def import_test_library(name: str) -> Union[Any, Tuple[Any, str]]: - with OutputCapturer(library_import=True): - importer = Importer("test library", LOGGER) - return importer.import_class_or_module(name, return_source=True) - - def get_test_library( - libcode: Any, - source: str, - name: str, - args: Optional[Tuple[Any, ...]] = None, - variables: Optional[Dict[str, Optional[Any]]] = None, - create_handlers: bool = True, - logger: Any = LOGGER, - ) -> Any: - if get_robot_version() < (7, 0): - libclass = robot.running.testlibraries._get_lib_class(libcode) - lib = libclass(libcode, name, args or [], source, logger, variables) - if create_handlers: - lib.create_handlers() - else: - lib = robot.running.testlibraries.TestLibrary.from_code( - libcode, - name, - source=Path(source), - args=args or [], - variables=variables, - create_keywords=create_handlers, - logger=logger, - ) - - return lib with _std_capture() as std_capturer: import_name, robot_variables = _find_library_internal( @@ -1781,7 +1803,7 @@ def get_test_library( source = None try: - libcode, source = import_test_library(import_name) + libcode, source = _import_test_library(import_name) except (SystemExit, KeyboardInterrupt): raise except BaseException as e: @@ -1821,13 +1843,17 @@ def get_test_library( lib = None try: - lib = get_test_library( + lib = _get_test_library( libcode, source, library_name, args, create_handlers=False, - variables=robot_variables, + variables=( + robot_variables + if robot_variables is not None + else resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) + ), ) if get_robot_version() < (7, 0): _ = lib.get_instance() @@ -1847,7 +1873,7 @@ def get_test_library( if args: try: - lib = get_test_library(libcode, source, library_name, (), create_handlers=False) + lib = _get_test_library(libcode, source, library_name, (), create_handlers=False) if get_robot_version() < (7, 0): _ = lib.get_instance() else: @@ -1862,6 +1888,10 @@ def get_test_library( libdoc = LibraryDoc( name=library_name, source=real_source, + line_no=lib.lineno if lib is not None else -1, + version=str(lib.version) if lib is not None else "", + scope=str(lib.scope) if lib is not None else ROBOT_DEFAULT_SCOPE, + doc_format=(str(lib.doc_format) or ROBOT_DOC_FORMAT) if lib is not None else ROBOT_DOC_FORMAT, module_spec=( module_spec if module_spec is not None @@ -1870,16 +1900,33 @@ def get_test_library( else None ), python_path=sys.path, - line_no=lib.lineno if lib is not None else -1, - doc=str(lib.doc) if lib is not None else "", - version=str(lib.version) if lib is not None else "", - scope=str(lib.scope) if lib is not None else ROBOT_DEFAULT_SCOPE, - doc_format=(str(lib.doc_format) or ROBOT_DOC_FORMAT) if lib is not None else ROBOT_DOC_FORMAT, member_name=module_spec.member_name if module_spec is not None else None, ) if lib is not None: try: + + def _get(handler: Callable[[], _T]) -> Optional[_T]: + try: + return handler() + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as e: + errors.append( + error_from_exception( + e, + source or module_spec.origin if module_spec is not None else None, + ( + 1 + if source is not None or module_spec is not None and module_spec.origin is not None + else None + ), + ) + ) + return None + + libdoc.doc = _get(lambda: str(lib.doc) if lib is not None else "") or "" + if get_robot_version() < (7, 0): libdoc.has_listener = lib.has_listener @@ -1909,10 +1956,10 @@ def get_test_library( keywords=[ KeywordDoc( name=libdoc.name, - arguments=[ArgumentInfo.from_robot(a) for a in kw[0].args], - doc=kw[0].doc, - tags=list(kw[0].tags), - source=kw[0].source, + arguments=_get(lambda: [ArgumentInfo.from_robot(a) for a in kw[0].args]) or [], + doc=_get(lambda: kw[0].doc) or "", + tags=_get(lambda: list(kw[0].tags)) or [], + source=_get(lambda: kw[0].source) or "", line_no=kw[0].lineno if kw[0].lineno is not None else -1, col_offset=-1, end_col_offset=-1, @@ -1923,20 +1970,23 @@ def get_test_library( longname=f"{libdoc.name}.{kw[0].name}", doc_format=str(lib.doc_format) or ROBOT_DOC_FORMAT, is_initializer=True, - arguments_spec=ArgumentSpec.from_robot_argument_spec( - kw[1].arguments if get_robot_version() < (7, 0) else kw[1].args + arguments_spec=_get( + lambda: ArgumentSpec.from_robot_argument_spec( + kw[1].arguments if get_robot_version() < (7, 0) else kw[1].args + ) ), ) for kw in init_keywords ] ) - logger = Logger() - lib.logger = logger + logger = _Logger() if get_robot_version() < (7, 0): + lib.logger = logger lib.create_handlers() else: + lib._logger = logger lib.create_keywords() for m in logger.messages: @@ -1969,10 +2019,10 @@ def get_args_to_process(libdoc_name: str, kw_name: str) -> Any: keywords=[ KeywordDoc( name=kw[0].name, - arguments=[ArgumentInfo.from_robot(a) for a in kw[0].args], - doc=kw[0].doc, - tags=list(kw[0].tags), - source=kw[0].source, + arguments=_get(lambda: [ArgumentInfo.from_robot(a) for a in kw[0].args]) or [], + doc=_get(lambda: kw[0].doc) or "", + tags=_get(lambda: list(kw[0].tags)) or [], + source=_get(lambda: kw[0].source) or "", line_no=kw[0].lineno if kw[0].lineno is not None else -1, col_offset=-1, end_col_offset=-1, @@ -1987,21 +2037,26 @@ def get_args_to_process(libdoc_name: str, kw_name: str) -> Any: is_registered_run_keyword=RUN_KW_REGISTER.is_run_keyword(libdoc.name, kw[0].name), args_to_process=get_args_to_process(libdoc.name, kw[0].name), deprecated=kw[0].deprecated, - arguments_spec=( - ArgumentSpec.from_robot_argument_spec( - kw[1].arguments if get_robot_version() < (7, 0) else kw[1].args + arguments_spec=_get( + lambda: ( + ArgumentSpec.from_robot_argument_spec( + kw[1].arguments if get_robot_version() < (7, 0) else kw[1].args + ) + if not kw[1].is_error_handler + else None ) - if not kw[1].is_error_handler - else None ), - return_type=( - ( - str(kw[1].args.return_type) - if kw[1].args.return_type is not None and kw[1].args.return_type is not type(None) + return_type=_get( + lambda: ( + ( + str(kw[1].args.return_type) + if kw[1].args.return_type is not None + and kw[1].args.return_type is not type(None) + else None + ) + if get_robot_version() >= (7, 0) else None ) - if get_robot_version() >= (7, 0) - else None ), ) for kw in keyword_docs @@ -2086,9 +2141,8 @@ def _find_variables_internal( _update_env(working_dir) - robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) - if contains_variable(name, "$@&%"): + robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) try: name = robot_variables.replace_string(name, ignore_errors=False) except DataError as error: @@ -2109,12 +2163,18 @@ def resolve_args( command_line_variables: Optional[Dict[str, Optional[Any]]] = None, variables: Optional[Dict[str, Optional[Any]]] = None, ) -> Tuple[Any, ...]: - robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) + # robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) + robot_variables: Any = None result = [] for arg in args: if isinstance(arg, str): - result.append(robot_variables.replace_string(arg, ignore_errors=True)) + if contains_variable(arg, "$@&%"): + if robot_variables is None: + robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) + result.append(robot_variables.replace_string(arg, ignore_errors=True)) + else: + result.append(arg) else: result.append(arg) @@ -2389,8 +2449,8 @@ def find_file( ) -> str: _update_env(working_dir) - robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) if contains_variable(name, "$@&%"): + robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) try: name = robot_variables.replace_string(name, ignore_errors=False) except DataError as error: @@ -2492,7 +2552,7 @@ def complete_library_import( if e not in DEFAULT_LIBRARIES ] - if name is not None: + if name is not None and contains_variable(name, "$@&%"): robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) name = robot_variables.replace_string(name, ignore_errors=True) @@ -2568,7 +2628,7 @@ def complete_resource_import( result: List[CompleteResult] = [] - if name is not None: + if name is not None and contains_variable(name, "$@&%"): robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) name = robot_variables.replace_string(name, ignore_errors=True) @@ -2608,7 +2668,7 @@ def complete_variables_import( result: List[CompleteResult] = [] - if name is not None: + if name is not None and contains_variable(name, "$@&%"): robot_variables = resolve_robot_variables(working_dir, base_dir, command_line_variables, variables) name = robot_variables.replace_string(name, ignore_errors=True) diff --git a/packages/robot/src/robotcode/robot/diagnostics/model_helper.py b/packages/robot/src/robotcode/robot/diagnostics/model_helper.py index 39a4bfd8..a1a09fd9 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/model_helper.py +++ b/packages/robot/src/robotcode/robot/diagnostics/model_helper.py @@ -21,9 +21,8 @@ from robot.errors import VariableError from robot.parsing.lexer.tokens import Token -from robot.utils.escaping import split_from_equals, unescape +from robot.utils.escaping import unescape from robot.variables.finders import NOT_FOUND, NumberFinder -from robot.variables.search import contains_variable, search_variable from robotcode.core.lsp.types import Position from ..utils import get_robot_version @@ -35,6 +34,7 @@ whitespace_at_begin_of_token, whitespace_from_begin_of_token, ) +from ..utils.variables import contains_variable, search_variable, split_from_equals from .entities import ( LibraryEntry, VariableDefinition, @@ -210,10 +210,14 @@ def get_keyworddoc_and_token_from_position( position: Position, analyse_run_keywords: bool = True, ) -> Optional[Tuple[Optional[KeywordDoc], Token]]: - keyword_doc = namespace.find_keyword(keyword_name, raise_keyword_error=False) + finder = namespace.get_finder() + keyword_doc = finder.find_keyword(keyword_name, raise_keyword_error=False) if keyword_doc is None: return None + if finder.result_bdd_prefix: + keyword_token = ModelHelper.strip_bdd_prefix(namespace, keyword_token) + if position.is_in_range(range_from_token(keyword_token)): return keyword_doc, keyword_token diff --git a/packages/robot/src/robotcode/robot/diagnostics/namespace.py b/packages/robot/src/robotcode/robot/diagnostics/namespace.py index b9a925f4..ec4542a1 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/namespace.py +++ b/packages/robot/src/robotcode/robot/diagnostics/namespace.py @@ -1,7 +1,6 @@ import ast import enum import itertools -import time import weakref from collections import OrderedDict, defaultdict from concurrent.futures import CancelledError @@ -29,11 +28,6 @@ from robot.parsing.model.statements import ( VariablesImport as RobotVariablesImport, ) -from robot.variables.search import ( - is_scalar_assign, - is_variable, - search_variable, -) from robotcode.core.concurrent import RLock from robotcode.core.event import event from robotcode.core.lsp.types import ( @@ -58,7 +52,12 @@ tokenize_variables, ) from ..utils.stubs import Languages -from ..utils.variables import BUILTIN_VARIABLES +from ..utils.variables import ( + BUILTIN_VARIABLES, + is_scalar_assign, + is_variable, + search_variable, +) from ..utils.visitor import Visitor from .entities import ( ArgumentDefinition, @@ -894,6 +893,7 @@ def get_namespaces(self) -> Dict[KeywordMatcher, List[LibraryEntry]]: self._namespaces[KeywordMatcher(v.alias or v.name or v.import_name, is_namespace=True)].append(v) for v in (self.get_resources()).values(): self._namespaces[KeywordMatcher(v.alias or v.name or v.import_name, is_namespace=True)].append(v) + return self._namespaces def get_resources(self) -> Dict[str, ResourceEntry]: @@ -1794,38 +1794,27 @@ def iter_all_keywords(self) -> Iterator[KeywordDoc]: libdoc = self.get_library_doc() - for doc in itertools.chain( + yield from itertools.chain( self.get_imported_keywords(), libdoc.keywords if libdoc is not None else [], - ): - yield doc + ) @_logger.call def get_keywords(self) -> List[KeywordDoc]: with self._keywords_lock: if self._keywords is None: - current_time = time.monotonic() - self._logger.debug("start collecting keywords") - try: - i = 0 - self.ensure_initialized() + i = 0 - result: Dict[KeywordMatcher, KeywordDoc] = {} + self.ensure_initialized() - for doc in self.iter_all_keywords(): - i += 1 - result[doc.matcher] = doc + result: Dict[KeywordMatcher, KeywordDoc] = {} - self._keywords = list(result.values()) - except BaseException: - self._logger.debug("Canceled collecting keywords ") - raise - else: - self._logger.debug( - lambda: f"end collecting {len(self._keywords) if self._keywords else 0}" - f" keywords in {time.monotonic() - current_time}s analyze {i} keywords" - ) + for doc in self.iter_all_keywords(): + i += 1 + result[doc.matcher] = doc + + self._keywords = list(result.values()) return self._keywords diff --git a/packages/robot/src/robotcode/robot/diagnostics/namespace_analyzer.py b/packages/robot/src/robotcode/robot/diagnostics/namespace_analyzer.py index 56759df8..adf4045c 100644 --- a/packages/robot/src/robotcode/robot/diagnostics/namespace_analyzer.py +++ b/packages/robot/src/robotcode/robot/diagnostics/namespace_analyzer.py @@ -8,7 +8,7 @@ from io import StringIO from pathlib import Path from tokenize import TokenError, generate_tokens -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Union, cast from robot.errors import VariableError from robot.parsing.lexer.tokens import Token @@ -28,9 +28,8 @@ Variable, VariablesImport, ) -from robot.utils.escaping import split_from_equals, unescape +from robot.utils.escaping import unescape from robot.variables.finders import NOT_FOUND, NumberFinder -from robot.variables.search import contains_variable, is_scalar_assign, is_variable, search_variable from robotcode.core.concurrent import check_current_task_canceled from robotcode.core.lsp.types import ( CodeDescription, @@ -43,6 +42,7 @@ Range, ) from robotcode.core.uri import Uri +from robotcode.core.utils.logging import LoggingDescriptor from ..utils import get_robot_version from ..utils.ast import ( @@ -53,11 +53,13 @@ strip_variable_token, tokenize_variables, ) +from ..utils.variables import contains_variable, is_scalar_assign, is_variable, search_variable, split_from_equals from ..utils.visitor import Visitor from .entities import ( ArgumentDefinition, EnvironmentVariableDefinition, GlobalVariableDefinition, + InvalidVariableError, LibraryEntry, LocalVariableDefinition, TestVariableDefinition, @@ -94,6 +96,9 @@ class AnalyzerResult: class NamespaceAnalyzer(Visitor): + + _logger = LoggingDescriptor() + def __init__( self, model: ast.AST, @@ -126,8 +131,11 @@ def __init__( self._overridden_variables: Dict[VariableDefinition, VariableDefinition] = {} self._in_setting = False + self._in_block_setting = False self._suite_variables = self._variables.copy() + self._block_variables: Optional[Dict[VariableMatcher, VariableDefinition]] = None + self._end_block_handlers: Optional[List[Callable[[], None]]] = None def run(self) -> AnalyzerResult: self._diagnostics = [] @@ -146,10 +154,11 @@ def run(self) -> AnalyzerResult: except BaseException as e: self._append_diagnostics( range_from_node(self._model), - message=f"Fatal: can't analyze namespace '{e}')", + message=f"Fatal: can't analyze namespace '{e}'.", severity=DiagnosticSeverity.ERROR, code=type(e).__qualname__, ) + self._logger.exception(e) return AnalyzerResult( self._diagnostics, @@ -379,6 +388,15 @@ def _visit_settings_statement( finally: self._in_setting = False + def _visit_block_settings_statement( + self, node: Statement, severity: DiagnosticSeverity = DiagnosticSeverity.ERROR + ) -> None: + self._in_block_setting = True + try: + self._visit_settings_statement(node, severity) + finally: + self._in_block_setting = False + def _analyze_token_expression_variables( self, token: Token, severity: DiagnosticSeverity = DiagnosticSeverity.ERROR ) -> None: @@ -476,7 +494,7 @@ def _handle_find_variable_result( range=range_from_token(var_token), message=f"Environment variable '{var.name}' not found.", severity=severity, - code=Error.ENVIROMMENT_VARIABLE_NOT_FOUND, + code=Error.ENVIRONMENT_VARIABLE_NOT_FOUND, ) if var.type == VariableDefinitionType.ENVIRONMENT_VARIABLE: @@ -531,6 +549,20 @@ def _append_diagnostics( ) ) + KEYWORDS_WITH_EXPRESSIONS = [ + "BuiltIn.Evaluate", + "BuiltIn.Should Be True", + "BuiltIn.Should Not Be True", + "BuiltIn.Skip If", + "BuiltIn.Continue For Loop If", + "BuiltIn.Exit For Loop If", + "BuiltIn.Return From Keyword If", + "BuiltIn.Run Keyword And Return If", + "BuiltIn.Pass Execution If", + "BuiltIn.Run Keyword If", + "BuiltIn.Run Keyword Unless", + ] + def _analyze_keyword_call( self, node: ast.AST, @@ -552,13 +584,11 @@ def _analyze_keyword_call( if not allow_variables and not is_not_variable_token(keyword_token): return None - result = self._finder.find_keyword(keyword, raise_keyword_error=False, handle_bdd_style=False) + result = self._finder.find_keyword(keyword, raise_keyword_error=False) - if result is None: + if result is not None and self._finder.result_bdd_prefix: keyword_token = ModelHelper.strip_bdd_prefix(self._namespace, keyword_token) - result = self._finder.find_keyword(keyword, raise_keyword_error=False) - kw_range = range_from_token(keyword_token) if keyword: @@ -710,19 +740,7 @@ def _analyze_keyword_call( ) if result is not None: - if result.longname in [ - "BuiltIn.Evaluate", - "BuiltIn.Should Be True", - "BuiltIn.Should Not Be True", - "BuiltIn.Skip If", - "BuiltIn.Continue For Loop If", - "BuiltIn.Exit For Loop If", - "BuiltIn.Return From Keyword If", - "BuiltIn.Run Keyword And Return If", - "BuiltIn.Pass Execution If", - "BuiltIn.Run Keyword If", - "BuiltIn.Run Keyword Unless", - ]: + if result.longname in self.KEYWORDS_WITH_EXPRESSIONS: tokens = argument_tokens if tokens and (token := tokens[0]): self._analyze_token_expression_variables(token) @@ -896,7 +914,29 @@ def visit_Fixture(self, node: Fixture) -> None: # noqa: N802 if keyword_token is not None and keyword_token.value and keyword_token.value.upper() not in ("", "NONE"): self._analyze_token_variables(keyword_token) - self._analyze_statement_variables(node) + self._visit_block_settings_statement(node) + + self._analyze_keyword_call( + node, + keyword_token, + [e for e in node.get_tokens(Token.ARGUMENT)], + allow_variables=True, + ignore_errors_if_contains_variables=True, + ) + + def visit_Teardown(self, node: Fixture) -> None: # noqa: N802 + keyword_token = node.get_token(Token.NAME) + + # TODO: calculate possible variables in NAME + + if keyword_token is not None and keyword_token.value and keyword_token.value.upper() not in ("", "NONE"): + + def _handler() -> None: + self._analyze_token_variables(keyword_token) + self._analyze_statement_variables(node) + + if self._end_block_handlers is not None: + self._end_block_handlers.append(_handler) self._analyze_keyword_call( node, @@ -984,9 +1024,15 @@ def visit_TestCase(self, node: TestCase) -> None: # noqa: N802 self._current_testcase_or_keyword_name = node.name old_variables = self._variables self._variables = self._variables.copy() + self._end_block_handlers = [] try: self.generic_visit(node) + + for handler in self._end_block_handlers: + handler() + finally: + self._end_block_handlers = None self._variables = old_variables self._current_testcase_or_keyword_name = None self._template = None @@ -1029,13 +1075,21 @@ def visit_Keyword(self, node: Keyword) -> None: # noqa: N802 self._current_testcase_or_keyword_name = node.name old_variables = self._variables self._variables = self._variables.copy() + self._end_block_handlers = [] try: arguments = next((v for v in node.body if isinstance(v, Arguments)), None) if arguments is not None: self._visit_Arguments(arguments) + self._block_variables = self._variables.copy() self.generic_visit(node) + + for handler in self._end_block_handlers: + handler() + finally: + self._end_block_handlers = None + self._block_variables = None self._variables = old_variables self._current_testcase_or_keyword_name = None self._current_keyword_doc = None @@ -1132,7 +1186,7 @@ def _visit_Arguments(self, node: Statement) -> None: # noqa: N802 ) ) - except VariableError: + except (VariableError, InvalidVariableError): pass def _analyze_assign_statement(self, node: Statement) -> None: @@ -1172,7 +1226,7 @@ def _analyze_assign_statement(self, node: Statement) -> None: ) ) - except VariableError: + except (VariableError, InvalidVariableError): pass def visit_InlineIfHeader(self, node: Statement) -> None: # noqa: N802 @@ -1186,7 +1240,7 @@ def visit_ForHeader(self, node: Statement) -> None: # noqa: N802 variables = node.get_tokens(Token.VARIABLE) for variable in variables: variable_token = self._get_variable_token(variable) - if variable_token is not None: + if variable_token is not None and is_variable(variable_token.value): existing_var = self._find_variable(variable_token.value) if existing_var is None or existing_var.type not in [ @@ -1248,7 +1302,7 @@ def visit_ExceptHeader(self, node: Statement) -> None: # noqa: N802 source=self._namespace.source, ) - except VariableError: + except (VariableError, InvalidVariableError): pass def _format_template(self, template: str, arguments: Tuple[str, ...]) -> Tuple[str, Tuple[str, ...]]: @@ -1351,7 +1405,7 @@ def visit_DocumentationOrMetadata(self, node: Statement) -> None: # noqa: N802 self._visit_settings_statement(node, DiagnosticSeverity.HINT) def visit_Timeout(self, node: Statement) -> None: # noqa: N802 - self._analyze_statement_variables(node, DiagnosticSeverity.HINT) + self._visit_block_settings_statement(node) def visit_SingleValue(self, node: Statement) -> None: # noqa: N802 self._visit_settings_statement(node, DiagnosticSeverity.HINT) @@ -1399,19 +1453,35 @@ def visit_SectionHeader(self, node: Statement) -> None: # noqa: N802 code=Error.DEPRECATED_HEADER, ) - def visit_ReturnSetting(self, node: Statement) -> None: # noqa: N802 - self._analyze_statement_variables(node) + if get_robot_version() >= (7, 0): - if get_robot_version() >= (7, 0): - token = node.get_token(Token.RETURN_SETTING) - if token is not None and token.error: - self._append_diagnostics( - range=range_from_node_or_token(node, token), - message=token.error, - severity=DiagnosticSeverity.WARNING, - tags=[DiagnosticTag.DEPRECATED], - code=Error.DEPRECATED_RETURN_SETTING, - ) + def visit_ReturnSetting(self, node: Statement) -> None: # noqa: N802 + + def _handler() -> None: + self._analyze_statement_variables(node) + + if self._end_block_handlers is not None: + self._end_block_handlers.append(_handler) + + if get_robot_version() >= (7, 0): + token = node.get_token(Token.RETURN_SETTING) + if token is not None and token.error: + self._append_diagnostics( + range=range_from_node_or_token(node, token), + message=token.error, + severity=DiagnosticSeverity.WARNING, + tags=[DiagnosticTag.DEPRECATED], + code=Error.DEPRECATED_RETURN_SETTING, + ) + + else: + + def visit_Return(self, node: Statement) -> None: # noqa: N802 + def _handler() -> None: + self._analyze_statement_variables(node) + + if self._end_block_handlers is not None: + self._end_block_handlers.append(_handler) def _check_import_name(self, value: Optional[str], node: ast.AST, type: str) -> None: if not value: @@ -1540,11 +1610,18 @@ def _find_variable(self, name: str) -> Optional[VariableDefinition]: default_value=default_value or None, ) - vars = self._suite_variables if self._in_setting else self._variables + vars = ( + self._block_variables + if self._block_variables and self._in_block_setting + else self._suite_variables if self._in_setting else self._variables + ) - matcher = VariableMatcher(name) + try: + matcher = VariableMatcher(name) - return vars.get(matcher, None) + return vars.get(matcher, None) + except (VariableError, InvalidVariableError): + return None def _is_number(self, name: str) -> bool: if name.startswith("$"): diff --git a/packages/robot/src/robotcode/robot/utils/match.py b/packages/robot/src/robotcode/robot/utils/match.py index 06169ec6..9ede42fa 100644 --- a/packages/robot/src/robotcode/robot/utils/match.py +++ b/packages/robot/src/robotcode/robot/utils/match.py @@ -3,13 +3,13 @@ _transform_table = str.maketrans("", "", "_ ") -@lru_cache(maxsize=5000) +@lru_cache(maxsize=None) def normalize(text: str) -> str: # return text.lower().replace("_", "").replace(" ", "") return text.casefold().translate(_transform_table) -@lru_cache(maxsize=5000) +@lru_cache(maxsize=None) def normalize_namespace(text: str) -> str: return text.lower().replace(" ", "") diff --git a/packages/robot/src/robotcode/robot/utils/robot_path.py b/packages/robot/src/robotcode/robot/utils/robot_path.py index 921fd865..97ff0b7b 100644 --- a/packages/robot/src/robotcode/robot/utils/robot_path.py +++ b/packages/robot/src/robotcode/robot/utils/robot_path.py @@ -1,21 +1,11 @@ -from __future__ import annotations - import sys from os import PathLike from pathlib import Path from typing import Optional, Union -def find_file( - path: Union[Path, PathLike[str], str], - basedir: Union[Path, PathLike[str], str] = ".", - file_type: Optional[str] = None, -) -> str: - return find_file_ex(path, basedir, file_type) - - def find_file_ex( - path: Union[Path, PathLike[str], str], + path: Union[Path, "PathLike[str]", str], basedir: Union[Path, PathLike[str], str] = ".", file_type: Optional[str] = None, ) -> str: @@ -25,6 +15,7 @@ def find_file_ex( ret = _find_absolute_path(path) if path.is_absolute() else _find_relative_path(path, basedir) if ret: return str(ret) + default = file_type or "File" file_type = ( @@ -40,15 +31,23 @@ def find_file_ex( raise DataError("%s '%s' does not exist." % (file_type, path)) -def _find_absolute_path(path: Union[Path, PathLike[str], str]) -> Optional[str]: +def find_file( + path: Union[Path, "PathLike[str]", str], + basedir: Union[Path, PathLike[str], str] = ".", + file_type: Optional[str] = None, +) -> str: + return find_file_ex(path, basedir, file_type) + + +def _find_absolute_path(path: Union[Path, "PathLike[str]", str]) -> Optional[str]: if _is_valid_file(path): return str(path) return None def _find_relative_path( - path: Union[Path, PathLike[str], str], - basedir: Union[Path, PathLike[str], str], + path: Union[Path, "PathLike[str]", str], + basedir: Union[Path, "PathLike[str]", str], ) -> Optional[str]: for base in [basedir, *sys.path]: if not base: @@ -65,6 +64,6 @@ def _find_relative_path( return None -def _is_valid_file(path: Union[Path, PathLike[str], str]) -> bool: +def _is_valid_file(path: Union[Path, "PathLike[str]", str]) -> bool: path = Path(path) return path.is_file() or (path.is_dir() and Path(path, "__init__.py").is_fifo()) diff --git a/packages/robot/src/robotcode/robot/utils/variables.py b/packages/robot/src/robotcode/robot/utils/variables.py index b93aa2a0..c698c0ec 100644 --- a/packages/robot/src/robotcode/robot/utils/variables.py +++ b/packages/robot/src/robotcode/robot/utils/variables.py @@ -1,3 +1,13 @@ +import functools +from typing import Optional, Tuple, cast + +from robot.utils.escaping import split_from_equals as robot_split_from_equals +from robot.variables.search import VariableMatch as RobotVariableMatch +from robot.variables.search import contains_variable as robot_contains_variable +from robot.variables.search import is_scalar_assign as robot_is_scalar_assign +from robot.variables.search import is_variable as robot_is_variable +from robot.variables.search import search_variable as robot_search_variable + BUILTIN_VARIABLES = [ "${CURDIR}", "${EMPTY}", @@ -35,3 +45,28 @@ "${DEBUG_FILE}", "${OUTPUT_DIR}", ] + + +@functools.lru_cache(maxsize=512) +def contains_variable(string: str, identifiers: str = "$@&") -> bool: + return cast(bool, robot_contains_variable(string, identifiers)) + + +@functools.lru_cache(maxsize=512) +def is_scalar_assign(string: str, allow_assign_mark: bool = False) -> bool: + return cast(bool, robot_is_scalar_assign(string, allow_assign_mark)) + + +@functools.lru_cache(maxsize=512) +def is_variable(string: str, identifiers: str = "$@&") -> bool: + return cast(bool, robot_is_variable(string, identifiers)) + + +@functools.lru_cache(maxsize=512) +def search_variable(string: str, identifiers: str = "$@&%*", ignore_errors: bool = False) -> RobotVariableMatch: + return robot_search_variable(string, identifiers, ignore_errors) + + +@functools.lru_cache(maxsize=512) +def split_from_equals(string: str) -> Tuple[str, Optional[str]]: + return cast(Tuple[str, Optional[str]], robot_split_from_equals(string)) diff --git a/packages/runner/pyproject.toml b/packages/runner/pyproject.toml index 818b2c08..410633cc 100644 --- a/packages/runner/pyproject.toml +++ b/packages/runner/pyproject.toml @@ -28,10 +28,10 @@ classifiers = [ dynamic = ["version"] dependencies = [ "robotframework>=4.1.0", - "robotcode-robot==0.94.0", - "robotcode-modifiers==0.94.0", - "robotcode-plugin==0.94.0", - "robotcode==0.94.0", + "robotcode-robot==0.95.0", + "robotcode-modifiers==0.95.0", + "robotcode-plugin==0.95.0", + "robotcode==0.95.0", ] [project.entry-points.robotcode] diff --git a/packages/runner/src/robotcode/runner/__version__.py b/packages/runner/src/robotcode/runner/__version__.py index 4bcb7a21..914574ff 100644 --- a/packages/runner/src/robotcode/runner/__version__.py +++ b/packages/runner/src/robotcode/runner/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/packages/runner/src/robotcode/runner/cli/discover/discover.py b/packages/runner/src/robotcode/runner/cli/discover/discover.py index 473919c7..b63adad6 100644 --- a/packages/runner/src/robotcode/runner/cli/discover/discover.py +++ b/packages/runner/src/robotcode/runner/cli/discover/discover.py @@ -216,6 +216,7 @@ class TestItem: range: Optional[Range] = None tags: Optional[List[str]] = None error: Optional[str] = None + rpa: Optional[bool] = None @dataclass @@ -228,7 +229,9 @@ class ResultItem: class Statistics: suites: int = 0 suites_with_tests: int = 0 + suites_with_tasks: int = 0 tests: int = 0 + tasks: int = 0 def get_rel_source(source: Optional[str]) -> Optional[str]: @@ -254,7 +257,7 @@ def __init__(self) -> None: ) self._current = self.all self.suites: List[TestItem] = [] - self.tests: List[TestItem] = [] + self.test_and_tasks: List[TestItem] = [] self.tags: Dict[str, List[TestItem]] = defaultdict(list) self.normalized_tags: Dict[str, List[TestItem]] = defaultdict(list) self.statistics = Statistics() @@ -294,6 +297,7 @@ def visit_suite(self, suite: TestSuite) -> None: ), children=[], error=suite.error_message if isinstance(suite, ErroneousTestSuite) else None, + rpa=suite.rpa, ) except ValueError as e: raise ValueError(f"Error while parsing suite {suite.source}: {e}") from e @@ -313,7 +317,10 @@ def visit_suite(self, suite: TestSuite) -> None: self.statistics.suites += 1 if suite.tests: - self.statistics.suites_with_tests += 1 + if suite.rpa: + self.statistics.suites_with_tasks += 1 + else: + self.statistics.suites_with_tests += 1 def end_suite(self, _suite: TestSuite) -> None: self._collected.pop() @@ -332,7 +339,7 @@ def visit_test(self, test: TestCase) -> None: try: absolute_path = normalized_path(Path(test.source)) if test.source is not None else None item = TestItem( - type="test", + type="task" if self._current.rpa else "test", id=f"{absolute_path or ''};{test.longname};{test.lineno}", name=test.name, longname=test.longname, @@ -344,6 +351,7 @@ def visit_test(self, test: TestCase) -> None: end=Position(line=test.lineno - 1, character=0), ), tags=list(set(normalize(str(t), ignore="_") for t in test.tags)) if test.tags else None, + rpa=self._current.rpa, ) except ValueError as e: raise ValueError(f"Error while parsing suite {test.source}: {e}") from e @@ -352,10 +360,12 @@ def visit_test(self, test: TestCase) -> None: self.tags[str(tag)].append(item) self.normalized_tags[normalize(str(tag), ignore="_")].append(item) - self.tests.append(item) + self.test_and_tasks.append(item) self._current.children.append(item) - - self.statistics.tests += 1 + if self._current.rpa: + self.statistics.tasks += 1 + else: + self.statistics.tests += 1 @click.group(invoke_without_command=False) @@ -543,6 +553,28 @@ def handle_options( raise UnknownError("Unexpected error happened.") +def print_statistics(app: Application, suite: TestSuite, collector: Collector) -> None: + def print() -> Iterable[str]: + yield click.style("Statistics:", underline=True, fg="blue") + yield os.linesep + yield click.style(" - Suites: ", underline=True, bold=True, fg="blue") + yield f"{collector.statistics.suites}{os.linesep}" + if collector.statistics.suites_with_tests: + yield click.style(" - Suites with tests: ", underline=True, bold=True, fg="blue") + yield f"{collector.statistics.suites_with_tests}{os.linesep}" + if collector.statistics.suites_with_tasks: + yield click.style(" - Suites with tasks: ", underline=True, bold=True, fg="blue") + yield f"{collector.statistics.suites_with_tasks}{os.linesep}" + if collector.statistics.tests: + yield click.style(" - Tests: ", underline=True, bold=True, fg="blue") + yield f"{collector.statistics.tests}{os.linesep}" + if collector.statistics.tasks: + yield click.style(" - Tasks: ", underline=True, bold=True, fg="blue") + yield f"{collector.statistics.tasks}{os.linesep}" + + app.echo_via_pager(print()) + + @discover.command( context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, add_help_option=True, @@ -551,7 +583,7 @@ def handle_options( @click.option( "--tags / --no-tags", "show_tags", - default=False, + default=True, show_default=True, help="Show the tags that are present.", ) @@ -591,48 +623,70 @@ def all( if collector.all.children: if app.config.output_format is None or app.config.output_format == OutputFormat.TEXT: - tests_or_tasks = "Task" if suite.rpa else "Test" def print(item: TestItem, indent: int = 0) -> Iterable[str]: - type = click.style( - item.type.capitalize() if item.type == "suite" else tests_or_tasks.capitalize(), - fg="green", - ) - - if item.type == "test": + if item.type in ["test", "task"]: yield " " - yield type - yield click.style(f": {item.longname}", bold=True) + yield click.style(f"{item.type.capitalize()}: ", fg="blue") + yield click.style(item.longname, bold=True) yield click.style( f" ({item.source if full_paths else item.rel_source}" f":{item.range.start.line + 1 if item.range is not None else 1}){os.linesep}" ) if show_tags and item.tags: - yield click.style(" Tags:", bold=True, fg="green") + yield click.style(" Tags:", bold=True, fg="yellow") yield f" {', '. join(normalize(str(tag), ignore='_') for tag in sorted(item.tags))}{os.linesep}" else: - yield type - yield f": {item.longname}" + yield click.style(f"{item.type.capitalize()}: ", fg="green") + yield click.style(item.longname, bold=True) yield click.style(f" ({item.source if full_paths else item.rel_source}){os.linesep}") for child in item.children or []: yield from print(child, indent + 2) - if indent == 0: - yield os.linesep - - yield click.style("Suites: ", underline=True, bold=True, fg="blue") - yield f"{collector.statistics.suites}{os.linesep}" - yield click.style(f"Suites with {tests_or_tasks}: ", underline=True, bold=True, fg="blue") - yield f"{collector.statistics.suites_with_tests}{os.linesep}" - yield click.style(f"{tests_or_tasks}: ", underline=True, bold=True, fg="blue") - yield f"{collector.statistics.tests}{os.linesep}" - app.echo_via_pager(print(collector.all.children[0])) + print_statistics(app, suite, collector) else: app.print_data(ResultItem([collector.all], diagnostics), remove_defaults=True) +def _test_or_tasks( + selected_type: str, + app: Application, + full_paths: bool, + show_tags: bool, + by_longname: Tuple[str, ...], + exclude_by_longname: Tuple[str, ...], + robot_options_and_args: Tuple[str, ...], +) -> None: + suite, collector, diagnostics = handle_options(app, by_longname, exclude_by_longname, robot_options_and_args) + + if collector.all.children: + if app.config.output_format is None or app.config.output_format == OutputFormat.TEXT: + + def print(items: List[TestItem]) -> Iterable[str]: + for item in items: + if item.type != selected_type: + continue + + yield click.style(f"{item.type.capitalize()}: ", fg="blue") + yield click.style(item.longname, bold=True) + yield click.style( + f" ({item.source if full_paths else item.rel_source}" + f":{item.range.start.line + 1 if item.range is not None else 1}){os.linesep}" + ) + if show_tags and item.tags: + yield click.style(" Tags:", bold=True, fg="yellow") + yield f" {', '. join(normalize(str(tag), ignore='_') for tag in sorted(item.tags))}{os.linesep}" + + if collector.test_and_tasks: + app.echo_via_pager(print(collector.test_and_tasks)) + print_statistics(app, suite, collector) + + else: + app.print_data(ResultItem(collector.test_and_tasks, diagnostics), remove_defaults=True) + + @discover.command( context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, add_help_option=True, @@ -677,34 +731,53 @@ def tests( ``` """ - suite, collector, diagnostics = handle_options(app, by_longname, exclude_by_longname, robot_options_and_args) - - if collector.all.children: - if app.config.output_format is None or app.config.output_format == OutputFormat.TEXT: + _test_or_tasks("test", app, full_paths, show_tags, by_longname, exclude_by_longname, robot_options_and_args) - tests_or_tasks = "Task" if suite.rpa else "Test" - def print(items: List[TestItem]) -> Iterable[str]: - for item in items: - type = click.style( - item.type.capitalize() if item.type == "suite" else tests_or_tasks.capitalize(), - fg="blue", - ) - yield type - yield click.style(f": {item.longname}", bold=True) - yield click.style( - f" ({item.source if full_paths else item.rel_source}" - f":{item.range.start.line + 1 if item.range is not None else 1}){os.linesep}" - ) - if show_tags and item.tags: - yield click.style(" Tags:", bold=True, fg="green") - yield f" {', '. join(normalize(str(tag), ignore='_') for tag in sorted(item.tags))}{os.linesep}" +@discover.command( + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + add_help_option=True, + epilog="Use `-- --help` to see `robot` help.", +) +@click.option( + "--tags / --no-tags", + "show_tags", + default=False, + show_default=True, + help="Show the tags that are present.", +) +@click.option( + "--full-paths / --no-full-paths", + "full_paths", + default=False, + show_default=True, + help="Show full paths instead of releative.", +) +@add_options(*ROBOT_OPTIONS) +@pass_application +def tasks( + app: Application, + full_paths: bool, + show_tags: bool, + by_longname: Tuple[str, ...], + exclude_by_longname: Tuple[str, ...], + robot_options_and_args: Tuple[str, ...], +) -> None: + """\ + Discover tasks with the selected configuration, profiles, options and + arguments. - if collector.tests: - app.echo_via_pager(print(collector.tests)) + You can use all known `robot` arguments to filter for example by tags or to use pre-run-modifier. - else: - app.print_data(ResultItem(collector.tests, diagnostics), remove_defaults=True) + \b + Examples: + ``` + robotcode discover tasks + robotcode --profile regression discover tasks + robotcode --profile regression discover tasks --include regression --exclude wipANDnotready + ``` + """ + _test_or_tasks("task", app, full_paths, show_tags, by_longname, exclude_by_longname, robot_options_and_args) @discover.command( @@ -743,7 +816,7 @@ def suites( ``` """ - _suite, collector, diagnostics = handle_options(app, by_longname, exclude_by_longname, robot_options_and_args) + suite, collector, diagnostics = handle_options(app, by_longname, exclude_by_longname, robot_options_and_args) if collector.all.children: if app.config.output_format is None or app.config.output_format == OutputFormat.TEXT: @@ -760,6 +833,8 @@ def print(items: List[TestItem]) -> Iterable[str]: if collector.suites: app.echo_via_pager(print(collector.suites)) + print_statistics(app, suite, collector) + else: app.print_data(ResultItem(collector.suites, diagnostics), remove_defaults=True) @@ -788,6 +863,13 @@ class TagsResult: show_default=True, help="Show tests where the tag is present.", ) +@click.option( + "--tasks / --no-tasks", + "show_tasks", + default=False, + show_default=True, + help="Show tasks where the tag is present.", +) @click.option( "--full-paths / --no-full-paths", "full_paths", @@ -801,6 +883,7 @@ def tags( app: Application, normalized: bool, show_tests: bool, + show_tasks: bool, full_paths: bool, by_longname: Tuple[str, ...], exclude_by_longname: Tuple[str, ...], @@ -822,7 +905,7 @@ def tags( ``` """ - _suite, collector, _diagnostics = handle_options(app, by_longname, exclude_by_longname, robot_options_and_args) + suite, collector, _diagnostics = handle_options(app, by_longname, exclude_by_longname, robot_options_and_args) if collector.all.children: if app.config.output_format is None or app.config.output_format == OutputFormat.TEXT: @@ -832,11 +915,17 @@ def print(tags: Dict[str, List[TestItem]]) -> Iterable[str]: yield click.style( f"{tag}{os.linesep}", bold=show_tests, - fg="green" if show_tests else None, + fg="yellow" if show_tests else None, ) - if show_tests: + if show_tests or show_tasks: for t in items: - yield click.style(f" {t.longname}", bold=True) + click.style( + if show_tests != show_tasks: + if show_tests and t.type != "test": + continue + if show_tasks and t.type != "task": + continue + yield click.style(f" {t.type.capitalize()}: ", fg="blue") + yield click.style(t.longname, bold=True) + click.style( f" ({t.source if full_paths else t.rel_source}" f":{t.range.start.line + 1 if t.range is not None else 1}){os.linesep}" ) @@ -844,6 +933,8 @@ def print(tags: Dict[str, List[TestItem]]) -> Iterable[str]: if collector.normalized_tags: app.echo_via_pager(print(collector.normalized_tags if normalized else collector.tags)) + print_statistics(app, suite, collector) + else: app.print_data(TagsResult(collector.normalized_tags), remove_defaults=True) diff --git a/pyproject.toml b/pyproject.toml index 1681be54..6dca1ce1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,9 +51,9 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "robotcode-core==0.94.0", - "robotcode-plugin==0.94.0", - "robotcode-robot==0.94.0", + "robotcode-core==0.95.0", + "robotcode-plugin==0.95.0", + "robotcode-robot==0.95.0", ] dynamic = ["version"] @@ -71,22 +71,22 @@ robotcode = "robotcode.cli.__main__:main" [project.optional-dependencies] -debugger = ["robotcode-debugger==0.94.0"] -languageserver = ["robotcode-language-server==0.94.0"] -runner = ["robotcode-runner==0.94.0"] -analyze = ["robotcode-analyze==0.94.0"] +debugger = ["robotcode-debugger==0.95.0"] +languageserver = ["robotcode-language-server==0.95.0"] +runner = ["robotcode-runner==0.95.0"] +analyze = ["robotcode-analyze==0.95.0"] yaml = ["PyYAML>=5.4"] lint = ["robotframework-robocop>=2.0.0"] tidy = ["robotframework-tidy>=2.0.0"] rest = ["docutils"] -repl = ["robotcode-repl==0.94.0"] +repl = ["robotcode-repl==0.95.0"] colored = ["rich"] all = [ - "robotcode-debugger==0.94.0", - "robotcode-language-server==0.94.0", - "robotcode-runner==0.94.0", - "robotcode-analyze==0.94.0", - "robotcode-repl==0.94.0", + "robotcode-debugger==0.95.0", + "robotcode-language-server==0.95.0", + "robotcode-runner==0.95.0", + "robotcode-analyze==0.95.0", + "robotcode-repl==0.95.0", "PyYAML>=5.4", "robotframework-robocop>=2.0.0", "robotframework-tidy>=2.0.0", diff --git a/src/robotcode/cli/__version__.py b/src/robotcode/cli/__version__.py index 4bcb7a21..914574ff 100644 --- a/src/robotcode/cli/__version__.py +++ b/src/robotcode/cli/__version__.py @@ -1 +1 @@ -__version__ = "0.94.0" +__version__ = "0.95.0" diff --git a/tests/robotcode/language_server/robotframework/parts/data/.vscode/settings.json b/tests/robotcode/language_server/robotframework/parts/data/.vscode/settings.json index 1d87ebbc..84bc9667 100644 --- a/tests/robotcode/language_server/robotframework/parts/data/.vscode/settings.json +++ b/tests/robotcode/language_server/robotframework/parts/data/.vscode/settings.json @@ -13,7 +13,8 @@ } }, "robotcode.languageServer.extraArgs": [ - //"--debugpy", + "--verbose", + "--debugpy", // "--debugpy-wait-for-client", // "--log", // "--log-level", "INFO", @@ -30,14 +31,6 @@ "**/lib/alibrary.py", "LibraryWithErrors" ], - // "robotcode.extraArgs": [ - // "--debugpy", - // "--debugpy-wait-for-client", - // "--log", - // "--log-level", - // "DEBUG", - // // "--log-calls" - // ], "robotcode.robot.variableFiles": [ "${EXECDIR}/resources/testvars.yml" ], diff --git a/tests/robotcode/language_server/robotframework/parts/data/tests/tasks/some_tasks.robot b/tests/robotcode/language_server/robotframework/parts/data/tests/tasks/some_tasks.robot new file mode 100644 index 00000000..a32231bc --- /dev/null +++ b/tests/robotcode/language_server/robotframework/parts/data/tests/tasks/some_tasks.robot @@ -0,0 +1,6 @@ +*** Tasks *** +a simple task + print Hello, world! + +another simple task + print Hello, world! diff --git a/vscode-client/testcontrollermanager.ts b/vscode-client/testcontrollermanager.ts index bec59bfb..0e33c63c 100644 --- a/vscode-client/testcontrollermanager.ts +++ b/vscode-client/testcontrollermanager.ts @@ -971,8 +971,14 @@ export class TestControllerManager { if (robotTestItem.range !== undefined) { testItem.range = toVsCodeRange(robotTestItem.range); } + testItem.label = robotTestItem.name; - testItem.description = robotTestItem.description; + if (robotTestItem.type == "test" || robotTestItem.type == "task") { + testItem.description = robotTestItem.type + (robotTestItem.description ? ` - ${robotTestItem.description}` : ""); + } else { + testItem.description = robotTestItem.description; + } + if (robotTestItem.error !== undefined) { testItem.error = robotTestItem.error; }