From b61d6eed6a26ad7751f97fcf9224282c9c2f0d50 Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 02:20:12 +0900 Subject: [PATCH 1/9] Support --stdin-alt-name and the mix style such as `vint a.vim - b.vim` --- dev_tool/show_encoding.py | 2 +- test/asserting/policy.py | 8 +- test/fixture/lint_target.vim | 0 .../vint/ast/plugin/test_scope_plugin.py | 3 +- test/integration/vint/linting/test_linter.py | 5 +- .../scope_plugin/test_call_node_parser.py | 3 +- .../test_identifier_classifier.py | 3 +- .../scope_plugin/test_identifier_collector.py | 3 +- .../test_redir_assignment_parser.py | 3 +- .../test_reference_reachability_tester.py | 3 +- .../plugin/scope_plugin/test_scope_linker.py | 5 +- test/unit/vint/ast/test_parsing.py | 11 +- test/unit/vint/ast/test_traversing.py | 3 +- .../linting/formatter/test_json_formatter.py | 7 +- .../linting/policy/test_abstract_policy.py | 14 +- test/unit/vint/linting/test_cli.py | 17 +- test/unit/vint/linting/test_env.py | 4 +- test/unit/vint/linting/test_lint_target.py | 62 +++++ vint/asset/default_config.yaml | 1 + vint/ast/parsing.py | 26 +- vint/bootstrap.py | 5 +- vint/encodings/decoder.py | 26 +- vint/linting/cli.py | 243 ++++++++++-------- vint/linting/config/config_cmdargs_source.py | 6 + vint/linting/env.py | 5 +- vint/linting/file_filter.py | 7 +- vint/linting/formatter/abstract_formatter.py | 6 + vint/linting/formatter/formatter.py | 73 +++--- vint/linting/formatter/json_formatter.py | 43 ++-- vint/linting/formatter/statistic_formatter.py | 3 +- vint/linting/lint_target.py | 52 ++++ vint/linting/linter.py | 33 +-- vint/linting/policy/abstract_policy.py | 2 +- .../policy/prohibit_missing_scriptencoding.py | 33 +-- .../policy/prohibit_no_abort_function.py | 2 +- 35 files changed, 439 insertions(+), 283 deletions(-) create mode 100644 test/fixture/lint_target.vim create mode 100644 test/unit/vint/linting/test_lint_target.py create mode 100644 vint/linting/formatter/abstract_formatter.py create mode 100644 vint/linting/lint_target.py diff --git a/dev_tool/show_encoding.py b/dev_tool/show_encoding.py index 38810dd..e9f3e3b 100644 --- a/dev_tool/show_encoding.py +++ b/dev_tool/show_encoding.py @@ -19,5 +19,5 @@ file_path = Path(namespace['file'][0]) decoder = Decoder(default_decoding_strategy) - decoder.read(file_path) + decoder.decode(file_path) pprint(decoder.debug_hint) diff --git a/test/asserting/policy.py b/test/asserting/policy.py index 91c7933..172efbe 100644 --- a/test/asserting/policy.py +++ b/test/asserting/policy.py @@ -4,6 +4,8 @@ from vint.compat.itertools import zip_longest from vint.linting.policy_set import PolicySet from vint.linting.linter import Linter +from vint.linting.config.config_default_source import ConfigDefaultSource +from vint.linting.lint_target import LintTargetFile from vint.linting.level import Level @@ -31,7 +33,7 @@ def assertFoundViolationsEqual(self, path, Policy, expected_violations, policy_o config_dict['policies'][policy_name] = policy_options linter = Linter(policy_set, config_dict) - violations = linter.lint_file(path) + violations = linter.lint(LintTargetFile(path)) pprint(violations) self.assertEqual(len(violations), len(expected_violations)) @@ -43,9 +45,13 @@ def assertFoundViolationsEqual(self, path, Policy, expected_violations, policy_o def assertViolation(self, actual_violation, expected_violation): self.assertIsNot(actual_violation, None) self.assertIsNot(expected_violation, None) + + pprint(actual_violation) + self.assertEqual(actual_violation['name'], expected_violation['name']) self.assertEqual(actual_violation['position'], expected_violation['position']) self.assertEqual(actual_violation['level'], expected_violation['level']) + self.assertIsInstance(actual_violation['description'], str) diff --git a/test/fixture/lint_target.vim b/test/fixture/lint_target.vim new file mode 100644 index 0000000..e69de29 diff --git a/test/integration/vint/ast/plugin/test_scope_plugin.py b/test/integration/vint/ast/plugin/test_scope_plugin.py index ef91bdf..b78d946 100644 --- a/test/integration/vint/ast/plugin/test_scope_plugin.py +++ b/test/integration/vint/ast/plugin/test_scope_plugin.py @@ -10,6 +10,7 @@ is_declarative_identifier, ) from vint.ast.plugin.scope_plugin import ScopePlugin +from vint.linting.lint_target import LintTargetFile FIXTURE_BASE_PATH = Path('test', 'fixture', 'ast', 'scope_plugin') @@ -41,7 +42,7 @@ class Fixtures(enum.Enum): class TestScopePlugin(unittest.TestCase): def create_ast(self, file_path): parser = Parser() - ast = parser.parse_file(file_path.value) + ast = parser.parse(LintTargetFile(file_path.value)) return ast diff --git a/test/integration/vint/linting/test_linter.py b/test/integration/vint/linting/test_linter.py index 8a1531f..42be4fd 100644 --- a/test/integration/vint/linting/test_linter.py +++ b/test/integration/vint/linting/test_linter.py @@ -4,6 +4,7 @@ from vint.linting.level import Level from vint.linting.policy.abstract_policy import AbstractPolicy from vint.linting.linter import Linter +from vint.linting.lint_target import LintTargetFile INVALID_VIM_SCRIPT = Path('test', 'fixture', 'linter', 'invalid.vim') BROKEN_VIM_SCRIPT = Path('test', 'fixture', 'linter', 'broken.vim') @@ -86,7 +87,7 @@ def test_lint(self): } linter = Linter(policy_set, config_dict_global) - got_violations = linter.lint_file(INVALID_VIM_SCRIPT) + got_violations = linter.lint(LintTargetFile(INVALID_VIM_SCRIPT)) expected_violations = [ { @@ -147,7 +148,7 @@ def test_lint_with_broken_file(self): } linter = Linter(policy_set, config_dict_global) - got_violations = linter.lint_file(BROKEN_VIM_SCRIPT) + got_violations = linter.lint(LintTargetFile(BROKEN_VIM_SCRIPT)) expected_violations = [ { diff --git a/test/unit/vint/ast/plugin/scope_plugin/test_call_node_parser.py b/test/unit/vint/ast/plugin/scope_plugin/test_call_node_parser.py index a2025fa..6ddace9 100644 --- a/test/unit/vint/ast/plugin/scope_plugin/test_call_node_parser.py +++ b/test/unit/vint/ast/plugin/scope_plugin/test_call_node_parser.py @@ -10,6 +10,7 @@ get_lambda_string_expr_content, FUNCTION_REFERENCE_STRING_EXPR_CONTENT, ) +from vint.linting.lint_target import LintTargetFile FIXTURE_BASE_PATH = Path('test', 'fixture', 'ast', 'scope_plugin') @@ -27,7 +28,7 @@ class Fixtures(enum.Enum): class TestCallNodeParser(unittest.TestCase): def create_ast(self, file_path): parser = Parser() - ast = parser.parse_file(file_path.value) + ast = parser.parse(LintTargetFile(file_path.value)) return ast diff --git a/test/unit/vint/ast/plugin/scope_plugin/test_identifier_classifier.py b/test/unit/vint/ast/plugin/scope_plugin/test_identifier_classifier.py index f4e9843..3e78e67 100644 --- a/test/unit/vint/ast/plugin/scope_plugin/test_identifier_classifier.py +++ b/test/unit/vint/ast/plugin/scope_plugin/test_identifier_classifier.py @@ -21,6 +21,7 @@ IDENTIFIER_ATTRIBUTE_LAMBDA_ARGUMENT_FLAG, IDENTIFIER_ATTRIBUTE_LAMBDA_BODY_CONTEXT, ) +from vint.linting.lint_target import LintTargetFile FIXTURE_BASE_PATH = Path('test', 'fixture', 'ast', 'scope_plugin') @@ -64,7 +65,7 @@ class TestIdentifierClassifier(unittest.TestCase): def create_ast(self, file_path): parser = Parser() - ast = parser.parse_file(file_path) + ast = parser.parse(LintTargetFile(file_path)) return ast diff --git a/test/unit/vint/ast/plugin/scope_plugin/test_identifier_collector.py b/test/unit/vint/ast/plugin/scope_plugin/test_identifier_collector.py index e047b16..9992054 100644 --- a/test/unit/vint/ast/plugin/scope_plugin/test_identifier_collector.py +++ b/test/unit/vint/ast/plugin/scope_plugin/test_identifier_collector.py @@ -3,6 +3,7 @@ from vint.ast.parsing import Parser from vint.ast.plugin.scope_plugin.identifier_classifier import IdentifierClassifier +from vint.linting.lint_target import LintTargetFile FIXTURE_BASE_PATH = Path('test', 'fixture', 'ast', 'scope_plugin') @@ -15,7 +16,7 @@ class TestIdentifierCollector(unittest.TestCase): def create_ast(self, file_path): parser = Parser() - ast = parser.parse_file(file_path) + ast = parser.parse(LintTargetFile(file_path)) id_classifier = IdentifierClassifier() attached_ast = id_classifier.attach_identifier_attributes(ast) diff --git a/test/unit/vint/ast/plugin/scope_plugin/test_redir_assignment_parser.py b/test/unit/vint/ast/plugin/scope_plugin/test_redir_assignment_parser.py index ed7c828..7e1bb19 100644 --- a/test/unit/vint/ast/plugin/scope_plugin/test_redir_assignment_parser.py +++ b/test/unit/vint/ast/plugin/scope_plugin/test_redir_assignment_parser.py @@ -9,6 +9,7 @@ RedirAssignmentParser, get_redir_content, ) +from vint.linting.lint_target import LintTargetFile FIXTURE_BASE_PATH = Path('test', 'fixture', 'ast', 'scope_plugin') @@ -22,7 +23,7 @@ class Fixtures(enum.Enum): class TestRedirAssignmentParser(unittest.TestCase): def create_ast(self, file_path): parser = Parser() - ast = parser.parse_file(file_path.value) + ast = parser.parse(LintTargetFile(file_path.value)) return ast diff --git a/test/unit/vint/ast/plugin/scope_plugin/test_reference_reachability_tester.py b/test/unit/vint/ast/plugin/scope_plugin/test_reference_reachability_tester.py index aff5684..6095804 100644 --- a/test/unit/vint/ast/plugin/scope_plugin/test_reference_reachability_tester.py +++ b/test/unit/vint/ast/plugin/scope_plugin/test_reference_reachability_tester.py @@ -8,6 +8,7 @@ is_reachable_reference_identifier, is_referenced_declarative_identifier, ) +from vint.linting.lint_target import LintTargetFile FIXTURE_BASE_PATH = Path('test', 'fixture', 'ast', 'scope_plugin') @@ -31,7 +32,7 @@ class Fixtures(enum.Enum): class TestReferenceReachabilityTester(unittest.TestCase): def create_ast(self, file_path): parser = Parser() - ast = parser.parse_file(file_path.value) + ast = parser.parse(LintTargetFile(file_path.value)) return ast diff --git a/test/unit/vint/ast/plugin/scope_plugin/test_scope_linker.py b/test/unit/vint/ast/plugin/scope_plugin/test_scope_linker.py index 867b4e9..61e82c1 100644 --- a/test/unit/vint/ast/plugin/scope_plugin/test_scope_linker.py +++ b/test/unit/vint/ast/plugin/scope_plugin/test_scope_linker.py @@ -4,6 +4,8 @@ from pathlib import Path from vint.ast.parsing import Parser +from vint.ast.plugin.scope_plugin.scope_linker import ScopeLinker, ScopeVisibility +from vint.linting.lint_target import LintTargetFile from vint.ast.plugin.scope_plugin.scope_linker import ScopeLinker from vint.ast.plugin.scope_plugin.scope import ( Scope, @@ -34,7 +36,8 @@ class Fixtures(enum.Enum): class TestScopeLinker(unittest.TestCase): def create_ast(self, file_path): parser = Parser() - ast = parser.parse_file(file_path.value) + lint_target = LintTargetFile(file_path.value) + ast = parser.parse(lint_target) return ast diff --git a/test/unit/vint/ast/test_parsing.py b/test/unit/vint/ast/test_parsing.py index 92a2e16..f2a4f42 100644 --- a/test/unit/vint/ast/test_parsing.py +++ b/test/unit/vint/ast/test_parsing.py @@ -1,6 +1,7 @@ import unittest from vint.ast.parsing import Parser from vint.ast.node_type import NodeType +from vint.linting.lint_target import LintTargetFile from test.asserting.ast import get_fixture_path FIXTURE_FILE = get_fixture_path('fixture_to_parse.vim') @@ -10,27 +11,27 @@ class TestParser(unittest.TestCase): - def test_parse_file(self): + def test_parse(self): parser = Parser() - ast = parser.parse_file(FIXTURE_FILE) + ast = parser.parse(LintTargetFile(FIXTURE_FILE)) self.assertIs(ast['type'], 1) def test_parse_file_on_ff_dos_and_fenc_cp932(self): parser = Parser() - ast = parser.parse_file(FIXTURE_FILE_FF_DOS_FENC_CP932) + ast = parser.parse(LintTargetFile(FIXTURE_FILE_FF_DOS_FENC_CP932)) self.assertIs(ast['type'], 1) def test_parse_file_when_neovim_enabled(self): parser = Parser(enable_neovim=True) - ast = parser.parse_file(FIXTURE_FILE_NEOVIM) + ast = parser.parse(LintTargetFile(FIXTURE_FILE_NEOVIM)) self.assertIs(ast['type'], 1) def test_parse_empty_file(self): parser = Parser() - ast = parser.parse_file(FIXTURE_FILE_EMPTY) + ast = parser.parse(LintTargetFile(FIXTURE_FILE_EMPTY)) self.assertIs(ast['type'], 1) diff --git a/test/unit/vint/ast/test_traversing.py b/test/unit/vint/ast/test_traversing.py index f5b37d5..31a46e8 100644 --- a/test/unit/vint/ast/test_traversing.py +++ b/test/unit/vint/ast/test_traversing.py @@ -4,6 +4,7 @@ from vint.ast.parsing import Parser from vint.ast.node_type import NodeType from vint.ast.traversing import traverse, SKIP_CHILDREN +from vint.linting.lint_target import LintTargetFile FIXTURE_FILE = get_fixture_path('fixture_to_traverse.vim') @@ -11,7 +12,7 @@ class TestTraverse(unittest.TestCase): def setUp(self): parser = Parser() - self.ast = parser.parse_file(FIXTURE_FILE) + self.ast = parser.parse(LintTargetFile(FIXTURE_FILE)) def test_traverse(self): expected_order_of_events = [ diff --git a/test/unit/vint/linting/formatter/test_json_formatter.py b/test/unit/vint/linting/formatter/test_json_formatter.py index df6896d..8db6116 100644 --- a/test/unit/vint/linting/formatter/test_json_formatter.py +++ b/test/unit/vint/linting/formatter/test_json_formatter.py @@ -9,8 +9,7 @@ class TestJSONFormatter(FormatterAssertion, unittest.TestCase): def test_format_violations(self): - cmdargs = {} - formatter = JSONFormatter(cmdargs) + formatter = JSONFormatter() violations = [ { @@ -21,7 +20,7 @@ def test_format_violations(self): 'position': { 'line': 1, 'column': 2, - 'path': Path('path', 'to', 'file1') + 'path': str(Path('path', 'to', 'file1')) }, }, { @@ -32,7 +31,7 @@ def test_format_violations(self): 'position': { 'line': 11, 'column': 21, - 'path': Path('path', 'to', 'file2') + 'path': str(Path('path', 'to', 'file2')) }, }, ] diff --git a/test/unit/vint/linting/policy/test_abstract_policy.py b/test/unit/vint/linting/policy/test_abstract_policy.py index 8c04125..052ccbf 100644 --- a/test/unit/vint/linting/policy/test_abstract_policy.py +++ b/test/unit/vint/linting/policy/test_abstract_policy.py @@ -1,4 +1,7 @@ import unittest +from io import BytesIO +from pathlib import Path +from vint.linting.lint_target import LintTargetBufferedStream from vint.linting.policy.abstract_policy import AbstractPolicy @@ -23,7 +26,12 @@ def test_create_violation_report(self): } node = {'pos': pos} - env = {'path': 'path/to/file.vim'} + lint_context = { + 'lint_target': LintTargetBufferedStream( + alternate_path=Path('path/to/file.vim'), + buffered_io=BytesIO(), + ) + } expected_violation = { 'name': 'ConcretePolicy', @@ -33,13 +41,13 @@ def test_create_violation_report(self): 'position': { 'column': 3, 'line': 3, - 'path': 'path/to/file.vim', + 'path': Path('path/to/file.vim'), }, } policy = ConcretePolicy() self.assertEqual( - policy.create_violation_report(node, env), + policy.create_violation_report(node, lint_context), expected_violation) def test_get_policy_options(self): diff --git a/test/unit/vint/linting/test_cli.py b/test/unit/vint/linting/test_cli.py index 90da474..b60f6fd 100644 --- a/test/unit/vint/linting/test_cli.py +++ b/test/unit/vint/linting/test_cli.py @@ -1,7 +1,7 @@ import unittest from vint.compat.unittest import mock -from vint.linting.cli import CLI +from vint.linting.cli import start_cli from vint.bootstrap import import_all_policies @@ -9,15 +9,14 @@ class TestCLI(unittest.TestCase): @classmethod def setUpClass(cls): # For test_start_with_invalid_file_path. - # The test case want to load saveral policies. + # The test case want to load several policies. import_all_policies() def assertExitWithSuccess(self, argv): with mock.patch('sys.argv', argv): with self.assertRaises(SystemExit) as e: - cli = CLI() - cli.start() + start_cli() self.assertEqual(e.exception.code, 0) @@ -25,8 +24,7 @@ def assertExitWithSuccess(self, argv): def assertExitWithFailure(self, argv): with mock.patch('sys.argv', argv): with self.assertRaises(SystemExit) as e: - cli = CLI() - cli.start() + start_cli() self.assertNotEqual(e.exception.code, 0) @@ -61,5 +59,12 @@ def test_passing_code_to_stdin_lints_the_code_from_stdin(self): argv = ['bin/vint', '-'] self.assertExitWithSuccess(argv) + + @mock.patch('sys.stdin', open('test/fixture/cli/valid1.vim')) + def test_multiple_stdin_symbol(self): + argv = ['bin/vint', '-', '-'] + self.assertExitWithFailure(argv) + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/vint/linting/test_env.py b/test/unit/vint/linting/test_env.py index 089515a..04ff862 100644 --- a/test/unit/vint/linting/test_env.py +++ b/test/unit/vint/linting/test_env.py @@ -27,12 +27,12 @@ def test_build_environment(self): 'warning': True, 'max_violations': 10, }, - 'file_paths': set([ + 'file_paths': [ Path(FIXTURE_PATH, '1.vim'), Path(FIXTURE_PATH, '2.vim'), Path(FIXTURE_PATH, 'sub', '3.vim'), Path(FIXTURE_PATH, 'sub', '4.vim'), - ]), + ], 'home_path': home, 'xdg_config_home': xdg_config_home, 'cwd': cwd, diff --git a/test/unit/vint/linting/test_lint_target.py b/test/unit/vint/linting/test_lint_target.py new file mode 100644 index 0000000..f37d40a --- /dev/null +++ b/test/unit/vint/linting/test_lint_target.py @@ -0,0 +1,62 @@ +import unittest +from io import BytesIO +from pathlib import Path +import enum + + +from vint.linting.lint_target import ( + AbstractLintTarget, + LintTargetFile, + LintTargetBufferedStream, + CachedLintTarget, +) + + +class Fixture(enum.Enum): + FILE = Path('test', 'fixture', 'lint_target.vim') + + + +class TestLintTarget(unittest.TestCase): + def test_file(self): + lint_target = LintTargetFile(Fixture.FILE.value) + + bytes_seq = lint_target.read() + self.assertIsNotNone(bytes_seq) + self.assertEqual(Fixture.FILE.value, lint_target.path) + + + def test_buffered_stream(self): + alternate_path = Path('dummy'), + lint_target = LintTargetBufferedStream( + alternate_path=alternate_path, + buffered_io=BytesIO() + ) + + bytes_seq = lint_target.read() + self.assertIsNotNone(bytes_seq) + self.assertEqual(alternate_path, lint_target.path) + + + def test_cached(self): + path_stub = Path('stub') + lint_target = CachedLintTarget(LintTargetStub(path_stub, bytes())) + + bytes_seq = lint_target.read() + self.assertIsNotNone(bytes_seq) + self.assertEqual(path_stub, lint_target.path) + + + +class LintTargetStub(AbstractLintTarget): + def __init__(self, path, bytes_seq): # type: (Path, bytes) -> None + super(LintTargetStub, self).__init__(path) + self.bytes_seq = bytes_seq + + + def read(self): # type: () -> bytes + return self.bytes_seq + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/vint/asset/default_config.yaml b/vint/asset/default_config.yaml index 3d664ba..837ba60 100644 --- a/vint/asset/default_config.yaml +++ b/vint/asset/default_config.yaml @@ -7,6 +7,7 @@ cmdargs: json: no env: neovim: no + stdin_alt_path: stdin policies: # Experimental diff --git a/vint/ast/parsing.py b/vint/ast/parsing.py index 4d6727b..1e79a57 100644 --- a/vint/ast/parsing.py +++ b/vint/ast/parsing.py @@ -1,8 +1,10 @@ +from typing import Dict, Any import re from vint._bundles import vimlparser from vint.ast.traversing import traverse from vint.encodings.decoder import Decoder from vint.encodings.decoding_strategy import default_decoding_strategy +from vint.linting.lint_target import AbstractLintTarget class Parser(object): @@ -14,7 +16,16 @@ def __init__(self, plugins=None, enable_neovim=False): self._enable_neovim = enable_neovim - def parse(self, string): + def parse(self, lint_target): # type: (AbstractLintTarget) -> Dict[str, Any] + """ Parse vim script file and return the AST. """ + decoder = Decoder(default_decoding_strategy) + decoded = decoder.decode(lint_target.read()) + decoded_and_lf_normalized = decoded.replace('\r\n', '\n') + + return self.parse_string(decoded_and_lf_normalized) + + + def parse_string(self, string): # type: (str) -> Dict[str, Any] """ Parse vim script string and return the AST. """ lines = string.split('\n') @@ -31,15 +42,6 @@ def parse(self, string): return ast - def parse_file(self, file_path): - """ Parse vim script file and return the AST. """ - decoder = Decoder(default_decoding_strategy) - decoded = decoder.read(file_path) - decoded_and_lf_normalized = decoded.replace('\r\n', '\n') - - return self.parse(decoded_and_lf_normalized) - - def parse_redir(self, redir_cmd): """ Parse a command :redir content. """ redir_cmd_str = redir_cmd['str'] @@ -59,7 +61,7 @@ def parse_redir(self, redir_cmd): } # NOTE: This is a hack to parse variable node. - raw_ast = self.parse('echo ' + redir_cmd_body) + raw_ast = self.parse_string('echo ' + redir_cmd_body) # We need the left node of ECHO node redir_cmd_ast = raw_ast['body'][0]['list'][0] @@ -94,7 +96,7 @@ def parse_string_expr(self, string_expr_node): string_expr_str = string_expr_str.replace('\\"', '"') # NOTE: This is a hack to parse expr1. See :help expr1 - raw_ast = self.parse('echo ' + string_expr_str) + raw_ast = self.parse_string('echo ' + string_expr_str) # We need the left node of ECHO node parsed_string_expr_nodes = raw_ast['body'][0]['list'] diff --git a/vint/bootstrap.py b/vint/bootstrap.py index 066817d..9f39263 100644 --- a/vint/bootstrap.py +++ b/vint/bootstrap.py @@ -2,7 +2,7 @@ import pkgutil from pathlib import Path -from vint.linting.cli import CLI +from vint.linting.cli import start_cli import logging @@ -18,8 +18,7 @@ def init_linter(): def init_cli(): - cli = CLI() - cli.start() + start_cli() def import_all_policies(): diff --git a/vint/encodings/decoder.py b/vint/encodings/decoder.py index 375d583..fb0fa3c 100644 --- a/vint/encodings/decoder.py +++ b/vint/encodings/decoder.py @@ -1,7 +1,6 @@ import sys from typing import Dict, Any from pprint import pformat -from pathlib import Path from vint.encodings.decoding_strategy import DecodingStrategy @@ -16,25 +15,22 @@ def __init__(self, strategy): self.debug_hint = dict(version=sys.version) - def read(self, file_path): - # type: (Path) -> str + def decode(self, bytes_seq): + # type: (bytes) -> str + strings = [] - with file_path.open(mode='rb') as f: - bytes_seq = f.read() - strings = [] + for (loc, hunk) in _split_by_scriptencoding(bytes_seq): + debug_hint_for_the_loc = dict() + self.debug_hint[loc] = debug_hint_for_the_loc - for (loc, hunk) in _split_by_scriptencoding(bytes_seq): - debug_hint_for_the_loc = dict() - self.debug_hint[loc] = debug_hint_for_the_loc + string = self.strategy.decode(hunk, debug_hint=debug_hint_for_the_loc) - string = self.strategy.decode(hunk, debug_hint=debug_hint_for_the_loc) + if string is None: + raise EncodingDetectionError(self.debug_hint) - if string is None: - raise EncodingDetectionError(self.debug_hint) + strings.append(string) - strings.append(string) - - return ''.join(strings) + return ''.join(strings) def _split_by_scriptencoding(bytes_seq): diff --git a/vint/linting/cli.py b/vint/linting/cli.py index 6e2423a..be68acf 100644 --- a/vint/linting/cli.py +++ b/vint/linting/cli.py @@ -1,6 +1,7 @@ +from typing import Dict, Any, List import sys from argparse import ArgumentParser -from pathlib import PosixPath +from pathlib import Path import pkg_resources import logging @@ -11,161 +12,183 @@ from vint.linting.config.config_default_source import ConfigDefaultSource from vint.linting.config.config_global_source import ConfigGlobalSource from vint.linting.config.config_project_source import ConfigProjectSource +from vint.linting.config.config_util import get_config_value +from vint.linting.lint_target import ( + AbstractLintTarget, + LintTargetFile, + LintTargetBufferedStream, + CachedLintTarget, +) from vint.linting.policy_set import PolicySet +from vint.linting.formatter.abstract_formatter import AbstractFormatter from vint.linting.policy_registry import get_policy_classes from vint.linting.formatter.formatter import Formatter from vint.linting.formatter.json_formatter import JSONFormatter from vint.linting.formatter.statistic_formatter import StatisticFormatter -class CLI(object): - def start(self): - env = self._build_env(sys.argv) - self._validate(env) +_stdin_symbol = Path('-') - self._adjust_log_level(env) - config_dict = self._build_config_dict(env) - violations = self._lint_all(env, config_dict) +def start_cli(): + env = _build_env(sys.argv) + _validate(env) - parser = self._build_argparser() + _adjust_log_level(env) - if len(violations) == 0: - parser.exit(status=0) + config_dict = _build_config_dict(env) + violations = _lint_all(env, config_dict) - self._print_violations(violations, config_dict) - parser.exit(status=1) + parser = _build_arg_parser() + + if len(violations) == 0: + parser.exit(status=0) + + _print_violations(violations, config_dict) + parser.exit(status=1) - def _validate(self, env): - parser = self._build_argparser() - paths_to_lint = env['file_paths'] +def _validate(env): # type: (Dict[str, Any]) -> None + parser = _build_arg_parser() + paths_to_lint = env['file_paths'] + + if len(paths_to_lint) == 0: + logging.error('nothing to check') + parser.print_help() + parser.exit(status=1) - if len(paths_to_lint) == 0: - logging.error('nothing to check') - parser.print_help() + if paths_to_lint.count(_stdin_symbol) > 1: + logging.error('number of "-" must be less than 2') + parser.exit(status=1) + + for path_to_lint in filter(lambda path: path != _stdin_symbol, paths_to_lint): + if not path_to_lint.exists() or not path_to_lint.is_file(): + logging.error('no such file or directory: `{path}`'.format( + path=str(path_to_lint))) parser.exit(status=1) - if not self._should_read_from_stdin(env): - for path_to_lint in paths_to_lint: - if not path_to_lint.exists() or not path_to_lint.is_file(): - logging.error('no such file or directory: `{path}`'.format( - path=str(path_to_lint))) - parser.exit(status=1) +def _build_env(argv): + """ Build an environment object. + This method take an argv parameter to make function pure. + """ + cmdargs = _build_cmdargs(argv) + env = build_environment(cmdargs) + return env - def _build_config_dict(self, env): - config = ConfigContainer( - ConfigDefaultSource(env), - ConfigGlobalSource(env), - ConfigProjectSource(env), - ConfigCmdargsSource(env), - ) - return config.get_config_dict() +def _build_cmdargs(argv): + """ Build command line arguments dict to use; + - displaying usages + - vint.linting.env.build_environment + This method take an argv parameter to make function pure. + """ + parser = _build_arg_parser() + namespace = parser.parse_args(argv[1:]) - def _build_argparser(self): - parser = ArgumentParser(prog='vint', description='Lint Vim script') + cmdargs = vars(namespace) + return cmdargs - parser.add_argument('-v', '--version', action='version', version=self._get_version()) - parser.add_argument('-V', '--verbose', action='store_const', const=True, help='output verbose message') - parser.add_argument('-e', '--error', action='store_const', const=True, help='report only errors') - parser.add_argument('-w', '--warning', action='store_const', const=True, help='report errors and warnings') - parser.add_argument('-s', '--style-problem', action='store_const', const=True, help='report errors, warnings and style problems') - parser.add_argument('-m', '--max-violations', type=int, help='limit max violations count') - parser.add_argument('-c', '--color', action='store_const', const=True, help='colorize output when possible') - parser.add_argument('--no-color', action='store_const', const=True, help='do not colorize output') - parser.add_argument('-j', '--json', action='store_const', const=True, help='output json style') - parser.add_argument('-t', '--stat', action='store_const', const=True, help='output statistic info') - parser.add_argument('--enable-neovim', action='store_const', const=True, help='Enable Neovim syntax') - parser.add_argument('-f', '--format', help='set output format') - parser.add_argument('files', nargs='*', help='file or directory path to lint') - return parser +def _build_arg_parser(): + parser = ArgumentParser(prog='vint', description='Lint Vim script') + parser.add_argument('-v', '--version', action='version', version=_get_version()) + parser.add_argument('-V', '--verbose', action='store_const', const=True, help='output verbose message') + parser.add_argument('-e', '--error', action='store_const', const=True, help='report only errors') + parser.add_argument('-w', '--warning', action='store_const', const=True, help='report errors and warnings') + parser.add_argument('-s', '--style-problem', action='store_const', const=True, help='report errors, warnings and style problems') + parser.add_argument('-m', '--max-violations', type=int, help='limit max violations count') + parser.add_argument('-c', '--color', action='store_const', const=True, help='colorize output when possible') + parser.add_argument('--no-color', action='store_const', const=True, help='do not colorize output') + parser.add_argument('-j', '--json', action='store_const', const=True, help='output json style') + parser.add_argument('-t', '--stat', action='store_const', const=True, help='output statistic info') + parser.add_argument('--enable-neovim', action='store_const', const=True, help='enable Neovim syntax') + parser.add_argument('-f', '--format', help='set output format') + parser.add_argument('--stdin-alt-path', type=str, help='specify a file path that is used for reporting when linting standard inputs') + parser.add_argument('files', nargs='*', help='file or directory path to lint') - def _build_cmdargs(self, argv): - """ Build command line arguments dict to use; - - displaying usages - - vint.linting.env.build_environment + return parser - This method take an argv parameter to make function pure. - """ - parser = self._build_argparser() - namespace = parser.parse_args(argv[1:]) - cmdargs = vars(namespace) - return cmdargs +def _build_config_dict(env): # type: (Dict[str, Any]) -> Dict[str, Any] + config = ConfigContainer( + ConfigDefaultSource(env), + ConfigGlobalSource(env), + ConfigProjectSource(env), + ConfigCmdargsSource(env), + ) + return config.get_config_dict() - def _build_env(self, argv): - """ Build an environment object. - This method take an argv parameter to make function pure. - """ - cmdargs = self._build_cmdargs(argv) - env = build_environment(cmdargs) - return env +def _lint_all(env, config_dict): # type: (Dict[str, Any], Dict[str, Any]) -> List[Dict[str, Any]] + paths_to_lint = env['file_paths'] + violations = [] + linter = _build_linter(config_dict) - def _build_linter(self, config_dict): - policy_set = PolicySet(get_policy_classes()) - linter = Linter(policy_set, config_dict) - return linter + for path in paths_to_lint: + lint_target = _build_lint_target(path, config_dict) + violations += linter.lint(lint_target) + return violations - def _lint_all(self, env, config_dict): - paths_to_lint = env['file_paths'] - violations = [] - linter = self._build_linter(config_dict) - if self._should_read_from_stdin(env): - violations += linter.lint_text(sys.stdin.read()) - else: - for file_path in paths_to_lint: - violations += linter.lint_file(file_path) +def _build_linter(config_dict): # type: (Dict[str, Any]) -> Linter + policy_set = PolicySet(get_policy_classes()) + linter = Linter(policy_set, config_dict) + return linter - return violations - def _should_read_from_stdin(self, env): - return len(env['file_paths']) == 1 and PosixPath('-') in env['file_paths'] +def _print_violations(violations, config_dict): # type: (List[Dict[str, Any]], Dict[str, Any]) -> None + formatter = _build_formatter(config_dict) + output = formatter.format_violations(violations) + print(output) - def _get_formatter(self, config_dict): - if 'cmdargs' not in config_dict: - return Formatter(config_dict) - cmdargs = config_dict['cmdargs'] - if 'json' in cmdargs and cmdargs['json']: - return JSONFormatter(config_dict) - elif 'stat' in cmdargs and cmdargs['stat']: - return StatisticFormatter(config_dict) - else: - return Formatter(config_dict) +def _build_formatter(config_dict): # type: (Dict[str, Any]) -> AbstractFormatter + if 'cmdargs' not in config_dict: + return Formatter(config_dict) + cmdargs = config_dict['cmdargs'] + if 'json' in cmdargs and cmdargs['json']: + return JSONFormatter() + elif 'stat' in cmdargs and cmdargs['stat']: + return StatisticFormatter(config_dict) + else: + return Formatter(config_dict) - def _print_violations(self, violations, config_dict): - formatter = self._get_formatter(config_dict) - output = formatter.format_violations(violations) - print(output) +def _get_version(): + # In unit tests, pkg_resources cannot find vim-vint. + # So, I decided to return dummy version + try: + version = pkg_resources.require('vim-vint')[0].version + except pkg_resources.DistributionNotFound: + version = 'test_mode' + return version - def _get_version(self): - # In unit tests, pkg_resources cannot find vim-vint. - # So, I decided to return dummy version - try: - version = pkg_resources.require('vim-vint')[0].version - except pkg_resources.DistributionNotFound: - version = 'test_mode' - return version +def _adjust_log_level(env): + cmdargs = env['cmdargs'] + is_verbose = cmdargs.get('verbose', False) + log_level = logging.DEBUG if is_verbose else logging.WARNING - def _adjust_log_level(self, env): - cmdargs = env['cmdargs'] + logger = logging.getLogger() + logger.setLevel(log_level) - is_verbose = cmdargs.get('verbose', False) - log_level = logging.DEBUG if is_verbose else logging.WARNING - logger = logging.getLogger() - logger.setLevel(log_level) +def _build_lint_target(path, config_dict): # type: (Path, Dict[str, Any]) -> AbstractLintTarget + if path == _stdin_symbol: + stdin_alt_path = get_config_value(config_dict, ['cmdargs', 'stdin_alt_path']) + lint_target = LintTargetBufferedStream( + alternate_path=Path(stdin_alt_path), + buffered_io=sys.stdin.buffer + ) + return CachedLintTarget(lint_target) + else: + lint_target = LintTargetFile(path) + return CachedLintTarget(lint_target) diff --git a/vint/linting/config/config_cmdargs_source.py b/vint/linting/config/config_cmdargs_source.py index 9afa169..0c3e313 100644 --- a/vint/linting/config/config_cmdargs_source.py +++ b/vint/linting/config/config_cmdargs_source.py @@ -34,6 +34,7 @@ def _build_config_dict(self, env): config_dict = self._normalize_max_violations(env, config_dict) config_dict = self._normalize_format(env, config_dict) config_dict = self._normalize_env(env, config_dict) + config_dict = self._normalize_stdin_filename(env, config_dict) return config_dict @@ -114,3 +115,8 @@ def _normalize_env(self, env, config_dict): config_dict_cmdargs['env'] = {'neovim': True} return config_dict + + + def _normalize_stdin_filename(self, env, config_dict): + return self._pass_config_by_key('stdin_alt_path', env, config_dict) + diff --git a/vint/linting/env.py b/vint/linting/env.py index a935719..3cb48c3 100644 --- a/vint/linting/env.py +++ b/vint/linting/env.py @@ -1,3 +1,4 @@ +import sys import os import os.path from pathlib import Path @@ -10,7 +11,7 @@ def build_environment(cmdargs): 'home_path': _get_home_path(), 'xdg_config_home': _get_xdg_config_home(), 'cwd': _get_cwd(), - 'file_paths': _get_file_paths(cmdargs) + 'file_paths': _get_file_paths(cmdargs), } @@ -28,7 +29,7 @@ def _get_file_paths(cmdargs): found_file_paths = find_vim_script(map(Path, cmdargs['files'])) - return set(found_file_paths) + return found_file_paths def _get_xdg_config_home(): diff --git a/vint/linting/file_filter.py b/vint/linting/file_filter.py index d9e9877..a72a226 100644 --- a/vint/linting/file_filter.py +++ b/vint/linting/file_filter.py @@ -9,9 +9,8 @@ def find_vim_script(file_paths): for file_path in file_paths: if file_path.is_dir(): vim_script_file_paths += _find_vim_script_into_dir(file_path) - continue - - vim_script_file_paths.append(file_path) + else: + vim_script_file_paths.append(file_path) return vim_script_file_paths @@ -31,7 +30,7 @@ def _find_vim_script_into_dir(dir_path): vim_script_file_paths += _find_vim_script_into_dir(file_path) continue - return vim_script_file_paths + return sorted(vim_script_file_paths) def _is_vim_script(file_path): diff --git a/vint/linting/formatter/abstract_formatter.py b/vint/linting/formatter/abstract_formatter.py new file mode 100644 index 0000000..0043084 --- /dev/null +++ b/vint/linting/formatter/abstract_formatter.py @@ -0,0 +1,6 @@ +from typing import List, Dict, Any + + +class AbstractFormatter: + def format_violations(self, violations): # type: (List[Dict[str, Any]]) -> str + raise NotImplementedError \ No newline at end of file diff --git a/vint/linting/formatter/formatter.py b/vint/linting/formatter/formatter.py index 63e3d7b..a6483c5 100644 --- a/vint/linting/formatter/formatter.py +++ b/vint/linting/formatter/formatter.py @@ -1,5 +1,7 @@ +from typing import List, Dict, Any from pathlib import Path from ansicolor import Colors, colorize +from vint.linting.formatter.abstract_formatter import AbstractFormatter DEFAULT_FORMAT = '{file_path}:{line_number}:{column_number}: {description} (see {reference})' @@ -16,8 +18,8 @@ } -class Formatter(object): - def __init__(self, config_dict): +class Formatter(AbstractFormatter): + def __init__(self, config_dict): # type: (Dict[str, Any]) -> None if 'cmdargs' in config_dict: cmdargs = config_dict['cmdargs'] else: @@ -34,52 +36,55 @@ def __init__(self, config_dict): self._should_be_colorized = False - def _sort_violations(self, violations): - return sorted(violations, key=lambda violation: (violation['position']['path'], - violation['position']['line'])) - - - def format_violations(self, violations): - sorted_violations = self._sort_violations(violations) + def format_violations(self, violations): # type: (List[Dict[str, Any]]) -> str + sorted_violations = _sort_violations(violations) formatted_lines = map(self.format_violation, sorted_violations) return '\n'.join(formatted_lines) - def format_violation(self, violation): + def format_violation(self, violation): # type: (Dict[str, Any]) -> str if self._should_be_colorized: - formatter_map = self._get_colorize_formatter_map(violation) + formatter_map = _get_colorize_formatter_map(violation) else: - formatter_map = self._get_formatter_map(violation) + formatter_map = _get_formatter_map(violation) formatted_line = self._format.format(**formatter_map) return formatted_line - def _get_colorize_formatter_map(self, violation): - formatter_map = self._get_formatter_map(violation) - colorized_formatter_map = {} - for key, value in formatter_map.items(): - if key in FORMAT_COLOR_MAP: - Color = FORMAT_COLOR_MAP[key] - colorized_formatter_map[key] = colorize(str(value), Color()) - else: - colorized_formatter_map[key] = value +def _get_colorize_formatter_map(violation): + formatter_map = _get_formatter_map(violation) + colorized_formatter_map = {} + + for key, value in formatter_map.items(): + if key in FORMAT_COLOR_MAP: + Color = FORMAT_COLOR_MAP[key] + colorized_formatter_map[key] = colorize(str(value), Color()) + else: + colorized_formatter_map[key] = value + + return colorized_formatter_map + + + +def _get_formatter_map(violation): + file_path = Path(violation['position']['path']) - return colorized_formatter_map + return { + 'file_path': file_path, + 'file_name': file_path.name, + 'line_number': violation['position']['line'], + 'column_number': violation['position']['column'], + 'severity': violation['level'].name.lower(), + 'description': violation['description'], + 'policy_name': violation['name'], + 'reference': violation['reference'], + } - def _get_formatter_map(self, violation): - file_path = Path(violation['position']['path']) - return { - 'file_path': file_path, - 'file_name': file_path.name, - 'line_number': violation['position']['line'], - 'column_number': violation['position']['column'], - 'severity': violation['level'].name.lower(), - 'description': violation['description'], - 'policy_name': violation['name'], - 'reference': violation['reference'], - } +def _sort_violations(violations): + return sorted(violations, key=lambda violation: (violation['position']['path'], + violation['position']['line'])) diff --git a/vint/linting/formatter/json_formatter.py b/vint/linting/formatter/json_formatter.py index 8a181f8..e812297 100644 --- a/vint/linting/formatter/json_formatter.py +++ b/vint/linting/formatter/json_formatter.py @@ -1,33 +1,38 @@ +from typing import List, Dict, Any import json from pathlib import Path +from vint.linting.formatter.abstract_formatter import AbstractFormatter -class JSONFormatter(object): - def __init__(self, env): +class JSONFormatter(AbstractFormatter): + def __init__(self): # type: () -> None + super(JSONFormatter, self).__init__() pass - def format_violations(self, violations): - return json.dumps(self._normalize_violations(violations)) + def format_violations(self, violations): # type: (List[Dict[str, Any]]) -> str + return json.dumps(_normalize_violations(violations)) - def _normalize_violations(self, violations): - line_number = lambda violation: violation['position']['line'] - sorted_violations = sorted(violations, key=line_number) - normalized_violations = map(self._normalize_violation, sorted_violations) +def _normalize_violations(violations): # type: (List[Dict[str, Any]]) -> List[Dict[str, Any]] + line_number = lambda violation: violation['position']['line'] + sorted_violations = sorted(violations, key=line_number) - return list(normalized_violations) + normalized_violations = map(_normalize_violation, sorted_violations) + return list(normalized_violations) - def _normalize_violation(self, violation): - return { - 'file_path': str(Path(violation['position']['path'])), - 'line_number': violation['position']['line'], - 'column_number': violation['position']['column'], - 'severity': violation['level'].name.lower(), - 'description': violation['description'], - 'policy_name': violation['name'], - 'reference': violation['reference'], - } + + +def _normalize_violation(violation): # type: (Dict[str, Any]) -> Dict[str, Any] + return { + 'file_path': str(Path(violation['position']['path'])), + 'line_number': violation['position']['line'], + 'column_number': violation['position']['column'], + 'severity': violation['level'].name.lower(), + 'description': violation['description'], + 'policy_name': violation['name'], + 'reference': violation['reference'], + } diff --git a/vint/linting/formatter/statistic_formatter.py b/vint/linting/formatter/statistic_formatter.py index 392995d..31332a7 100644 --- a/vint/linting/formatter/statistic_formatter.py +++ b/vint/linting/formatter/statistic_formatter.py @@ -1,8 +1,9 @@ +from typing import List, Dict, Any from vint.linting.formatter.formatter import Formatter class StatisticFormatter(Formatter): - def format_violations(self, violations): + def format_violations(self, violations): # type: (List[Dict[str, Any]]) -> str violations_count = len(violations) output = super(StatisticFormatter, self).format_violations(violations) + '\n' diff --git a/vint/linting/lint_target.py b/vint/linting/lint_target.py new file mode 100644 index 0000000..f3eecd5 --- /dev/null +++ b/vint/linting/lint_target.py @@ -0,0 +1,52 @@ +from typing import Optional +from pathlib import Path +from io import BufferedIOBase + + + +class AbstractLintTarget: + def __init__(self, path): # type: (Path) -> None + self.path = path + + + def read(self): # type: () -> bytes + raise NotImplementedError() + + +class LintTargetFile(AbstractLintTarget): + def __init__(self, path): + # type: (Path) -> None + super(LintTargetFile, self).__init__(path) + + + def read(self): # type: () -> bytes + with self.path.open('rb') as f: + return f.read() + + +class LintTargetBufferedStream(AbstractLintTarget): + def __init__(self, alternate_path, buffered_io): + # type: (Path, BufferedIOBase) -> None + super(LintTargetBufferedStream, self).__init__(alternate_path) + self._buffered_io = buffered_io + + + def read(self): # type: () -> bytes + return self._buffered_io.read() + + +class CachedLintTarget(AbstractLintTarget): + def __init__(self, lint_target): + # type: (AbstractLintTarget) -> None + super(CachedLintTarget, self).__init__(lint_target.path) + self._target = lint_target + self._cached_bytes = None # type: Optional[bytes] + + + def read(self): # type: () -> bytes + if self._cached_bytes is not None: + return self._cached_bytes + + result = self._target.read() + self._cached_bytes = result + return result diff --git a/vint/linting/linter.py b/vint/linting/linter.py index 497ba4e..fb4cc08 100644 --- a/vint/linting/linter.py +++ b/vint/linting/linter.py @@ -1,3 +1,4 @@ +from typing import Dict, Any, List import re import logging from typing import Dict, List, Any @@ -8,6 +9,7 @@ from vint.ast.node_type import NodeType from vint.ast.traversing import traverse from vint.ast.plugin.scope_plugin import ScopePlugin +from vint.linting.lint_target import AbstractLintTarget from vint.linting.config.config_container import ConfigContainer from vint.linting.config.config_dict_source import ConfigDictSource from vint.linting.config.config_abstract_dynamic_source import ConfigAbstractDynamicSource @@ -89,41 +91,28 @@ def _create_decoding_error(self, path, err_message): } - def lint_file(self, path): - try: - root_ast = self._parser.parse_file(path) - except vimlparser.VimLParserException as exception: - parse_error = self._create_parse_error(path, str(exception)) - return [parse_error] - except EncodingDetectionError as exception: - decoding_error = self._create_decoding_error(path, str(exception)) - return [decoding_error] - - self._traverse(root_ast, path) - - return self._violations - + def lint(self, lint_target): # type: (AbstractLintTarget) -> List[Dict[str, Any]] + logging.debug('checking: `{file_path}`'.format(file_path=lint_target.path)) - def lint_text(self, text): try: - root_ast = self._parser.parse(text) + root_ast = self._parser.parse(lint_target) except vimlparser.VimLParserException as exception: - parse_error = self._create_parse_error('stdin', str(exception)) + parse_error = self._create_parse_error(lint_target.path, str(exception)) return [parse_error] except EncodingDetectionError as exception: - decoding_error = self._create_decoding_error('stdin', str(exception)) + decoding_error = self._create_decoding_error(lint_target, str(exception)) return [decoding_error] - self._traverse(root_ast, 'stdin') + self._traverse(root_ast, lint_target) return self._violations - def _traverse(self, root_ast, path): + def _traverse(self, root_ast, lint_target): if self._is_debug: logging.debug('{cls}: checking `{file_path}`'.format( cls=self.__class__.__name__, - file_path=path) + file_path=lint_target.path) ) logging.debug('{cls}: using config as {config_dict}'.format( cls=self.__class__.__name__, @@ -133,7 +122,7 @@ def _traverse(self, root_ast, path): self._prepare_for_traversal() lint_context = { - 'path': path, + 'lint_target': lint_target, 'root_node': root_ast, 'stack_trace': [], 'plugins': self._plugins, diff --git a/vint/linting/policy/abstract_policy.py b/vint/linting/policy/abstract_policy.py index 0ccba16..63a1595 100644 --- a/vint/linting/policy/abstract_policy.py +++ b/vint/linting/policy/abstract_policy.py @@ -28,7 +28,7 @@ def create_violation_report(self, node, lint_context): 'position': { 'line': node['pos']['lnum'], 'column': node['pos']['col'], - 'path': lint_context['path'], + 'path': lint_context['lint_target'].path, }, } diff --git a/vint/linting/policy/prohibit_missing_scriptencoding.py b/vint/linting/policy/prohibit_missing_scriptencoding.py index 52a12c2..581dcb3 100644 --- a/vint/linting/policy/prohibit_missing_scriptencoding.py +++ b/vint/linting/policy/prohibit_missing_scriptencoding.py @@ -1,9 +1,9 @@ import chardet -import sys from vint.ast.node_type import NodeType from vint.ast.traversing import traverse, SKIP_CHILDREN from vint.linting.level import Level +from vint.linting.lint_target import AbstractLintTarget from vint.linting.policy.abstract_policy import AbstractPolicy from vint.linting.policy_registry import register_policy @@ -33,7 +33,7 @@ def is_valid(self, node, lint_context): if self.has_scriptencoding: return True - return not self._check_script_has_multibyte_char(lint_context) + return not _has_multibyte_char(lint_context) def _check_scriptencoding(self, node): @@ -49,28 +49,7 @@ def _check_scriptencoding(self, node): self.has_scriptencoding = node['str'].startswith('scripte') - def _check_script_has_multibyte_char(self, lint_context): - if self._is_from_stdin(lint_context): - return self._check_stdin_has_multibyte_char() - else: - return self._check_file_has_multibyte_char(lint_context) - - - def _is_from_stdin(self, lint_context): - return lint_context['path'] == 'stdin' - - - def _check_stdin_has_multibyte_char(self): - sys.stdin.seek(0) - - try: - sys.stdin.read().encode('ascii') - return False - except UnicodeError: - return True - - def _check_file_has_multibyte_char(self, lint_context): - # TODO: Use cache to make performance efficiency - with lint_context['path'].open(mode='br') as f: - byte_seq = f.read() - return len(byte_seq) and chardet.detect(byte_seq)['encoding'] != 'ascii' +def _has_multibyte_char(lint_context): + lint_target = lint_context['lint_target'] # type: AbstractLintTarget + byte_seq = lint_target.read() + return len(byte_seq) > 0 and chardet.detect(byte_seq)['encoding'] != 'ascii' diff --git a/vint/linting/policy/prohibit_no_abort_function.py b/vint/linting/policy/prohibit_no_abort_function.py index aaf54be..51d6a02 100644 --- a/vint/linting/policy/prohibit_no_abort_function.py +++ b/vint/linting/policy/prohibit_no_abort_function.py @@ -24,7 +24,7 @@ def is_valid(self, node, lint_context): This policy prohibits functions in autoload that have no 'abort' or bang """ - if 'autoload' not in lint_context['path'].parts: + if 'autoload' not in lint_context['lint_target'].path.parts: return True has_bang = node['ea']['forceit'] != 0 From 327f7f9d4079203e45984c259dc09cbb67629c8d Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 02:27:53 +0900 Subject: [PATCH 2/9] Care Python 2 --- vint/linting/formatter/abstract_formatter.py | 2 +- vint/linting/lint_target.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vint/linting/formatter/abstract_formatter.py b/vint/linting/formatter/abstract_formatter.py index 0043084..f053455 100644 --- a/vint/linting/formatter/abstract_formatter.py +++ b/vint/linting/formatter/abstract_formatter.py @@ -1,6 +1,6 @@ from typing import List, Dict, Any -class AbstractFormatter: +class AbstractFormatter(object): def format_violations(self, violations): # type: (List[Dict[str, Any]]) -> str raise NotImplementedError \ No newline at end of file diff --git a/vint/linting/lint_target.py b/vint/linting/lint_target.py index f3eecd5..79adaf6 100644 --- a/vint/linting/lint_target.py +++ b/vint/linting/lint_target.py @@ -4,7 +4,7 @@ -class AbstractLintTarget: +class AbstractLintTarget(object): def __init__(self, path): # type: (Path) -> None self.path = path From 264b862bf63d54b129869ff5f96af0e553b2232c Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 02:36:59 +0900 Subject: [PATCH 3/9] Add available options in README --- README.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.rst b/README.rst index 7f40641..25e73c6 100644 --- a/README.rst +++ b/README.rst @@ -122,6 +122,40 @@ You can configure linting severity, max errors, ... as following: $ vint --color --style ~/.vimrc +And you can see all available options by `--help` as following: + +:: + $ vint --help + usage: vint [-h] [-v] [-V] [-e] [-w] [-s] [-m MAX_VIOLATIONS] [-c] + [--no-color] [-j] [-t] [--enable-neovim] [-f FORMAT] + [--stdin-alt-path STDIN_ALT_PATH] + [files [files ...]] + + Lint Vim script + + positional arguments: + files file or directory path to lint + + optional arguments: + -h, --help show this help message and exit + -v, --version show program's version number and exit + -V, --verbose output verbose message + -e, --error report only errors + -w, --warning report errors and warnings + -s, --style-problem report errors, warnings and style problems + -m MAX_VIOLATIONS, --max-violations MAX_VIOLATIONS + limit max violations count + -c, --color colorize output when possible + --no-color do not colorize output + -j, --json output json style + -t, --stat output statistic info + --enable-neovim enable Neovim syntax + -f FORMAT, --format FORMAT + set output format + --stdin-alt-path STDIN_ALT_PATH + specify a file path that is used for reporting when + linting standard inputs + Comment config ~~~~~~~~~~~~~~ From 50052d6c063b0275926311fa203aeec2a737d529 Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 02:37:41 +0900 Subject: [PATCH 4/9] fixup! Care Python 2 --- vint/linting/cli.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/vint/linting/cli.py b/vint/linting/cli.py index be68acf..c1553e2 100644 --- a/vint/linting/cli.py +++ b/vint/linting/cli.py @@ -184,10 +184,21 @@ def _adjust_log_level(env): def _build_lint_target(path, config_dict): # type: (Path, Dict[str, Any]) -> AbstractLintTarget if path == _stdin_symbol: stdin_alt_path = get_config_value(config_dict, ['cmdargs', 'stdin_alt_path']) - lint_target = LintTargetBufferedStream( - alternate_path=Path(stdin_alt_path), - buffered_io=sys.stdin.buffer - ) + + # NOTE: In Python 3, sys.stdin is a string not bytes. Then we can get bytes by sys.stdin.buffer. + # But in Python 2, sys.stdin.buffer is not defined. But we can get bytes by sys.stdin directly. + is_python_3 = hasattr(sys.stdin, 'buffer') + if is_python_3: + lint_target = LintTargetBufferedStream( + alternate_path=Path(stdin_alt_path), + buffered_io=sys.stdin.buffer + ) + else: + lint_target = LintTargetBufferedStream( + alternate_path=Path(stdin_alt_path), + buffered_io=sys.stdin + ) + return CachedLintTarget(lint_target) else: lint_target = LintTargetFile(path) From e135b4891845182796f23f86a183a040ed0ce4f0 Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 02:40:45 +0900 Subject: [PATCH 5/9] fixup! Care Python 2 --- vint/linting/cli.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vint/linting/cli.py b/vint/linting/cli.py index c1553e2..d5bd6f6 100644 --- a/vint/linting/cli.py +++ b/vint/linting/cli.py @@ -194,12 +194,21 @@ def _build_lint_target(path, config_dict): # type: (Path, Dict[str, Any]) -> Abs buffered_io=sys.stdin.buffer ) else: + # NOTE: Python 2 on Windows opens sys.stdin in text mode, and + # binary data that read from it becomes corrupted on \r\n + # SEE: https://stackoverflow.com/questions/2850893/reading-binary-data-from-stdin/38939320#38939320 + if sys.platform == 'win32': + # set sys.stdin to binary mode + import os, msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + lint_target = LintTargetBufferedStream( alternate_path=Path(stdin_alt_path), buffered_io=sys.stdin ) return CachedLintTarget(lint_target) + else: lint_target = LintTargetFile(path) return CachedLintTarget(lint_target) From c4d544ccb9e26e2370ef157cf030187d7dbc4dc6 Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 13:04:28 +0900 Subject: [PATCH 6/9] Update README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 25e73c6..2a71e9e 100644 --- a/README.rst +++ b/README.rst @@ -122,7 +122,7 @@ You can configure linting severity, max errors, ... as following: $ vint --color --style ~/.vimrc -And you can see all available options by `--help` as following: +And you can see all available options by using `--help`: :: $ vint --help From 9023bc1fc4f4e250ff979cdb3e95c9a99e9ebe4e Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 13:08:03 +0900 Subject: [PATCH 7/9] Rename --stdin-alt-path to --stdin-display-name --- README.rst | 2 +- vint/asset/default_config.yaml | 2 +- vint/linting/cli.py | 4 ++-- vint/linting/config/config_cmdargs_source.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 2a71e9e..139e309 100644 --- a/README.rst +++ b/README.rst @@ -152,7 +152,7 @@ And you can see all available options by using `--help`: --enable-neovim enable Neovim syntax -f FORMAT, --format FORMAT set output format - --stdin-alt-path STDIN_ALT_PATH + --stdin-display-name STDIN_DISPLAY_NAME specify a file path that is used for reporting when linting standard inputs diff --git a/vint/asset/default_config.yaml b/vint/asset/default_config.yaml index 837ba60..b9f23aa 100644 --- a/vint/asset/default_config.yaml +++ b/vint/asset/default_config.yaml @@ -7,7 +7,7 @@ cmdargs: json: no env: neovim: no - stdin_alt_path: stdin + stdin_display_name: stdin policies: # Experimental diff --git a/vint/linting/cli.py b/vint/linting/cli.py index d5bd6f6..79433cf 100644 --- a/vint/linting/cli.py +++ b/vint/linting/cli.py @@ -105,7 +105,7 @@ def _build_arg_parser(): parser.add_argument('-t', '--stat', action='store_const', const=True, help='output statistic info') parser.add_argument('--enable-neovim', action='store_const', const=True, help='enable Neovim syntax') parser.add_argument('-f', '--format', help='set output format') - parser.add_argument('--stdin-alt-path', type=str, help='specify a file path that is used for reporting when linting standard inputs') + parser.add_argument('--stdin-display-name', type=str, help='specify a file path that is used for reporting when linting standard inputs') parser.add_argument('files', nargs='*', help='file or directory path to lint') return parser @@ -183,7 +183,7 @@ def _adjust_log_level(env): def _build_lint_target(path, config_dict): # type: (Path, Dict[str, Any]) -> AbstractLintTarget if path == _stdin_symbol: - stdin_alt_path = get_config_value(config_dict, ['cmdargs', 'stdin_alt_path']) + stdin_alt_path = get_config_value(config_dict, ['cmdargs', 'stdin_display_name']) # NOTE: In Python 3, sys.stdin is a string not bytes. Then we can get bytes by sys.stdin.buffer. # But in Python 2, sys.stdin.buffer is not defined. But we can get bytes by sys.stdin directly. diff --git a/vint/linting/config/config_cmdargs_source.py b/vint/linting/config/config_cmdargs_source.py index 0c3e313..d25b06f 100644 --- a/vint/linting/config/config_cmdargs_source.py +++ b/vint/linting/config/config_cmdargs_source.py @@ -118,5 +118,5 @@ def _normalize_env(self, env, config_dict): def _normalize_stdin_filename(self, env, config_dict): - return self._pass_config_by_key('stdin_alt_path', env, config_dict) + return self._pass_config_by_key('stdin_display_name', env, config_dict) From 0663c0b6ed4d48ad73f61f1aec16e26f4a5b023d Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Thu, 21 Jun 2018 13:28:33 +0900 Subject: [PATCH 8/9] Add a test for stdin --- test/acceptance/test_cli.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/acceptance/test_cli.py b/test/acceptance/test_cli.py index b529365..46d639b 100644 --- a/test/acceptance/test_cli.py +++ b/test/acceptance/test_cli.py @@ -144,5 +144,17 @@ def test_exec_vint_with_color_flag(self): self.assertRegex(got_output, expected_output_pattern) + def test_exec_vint_with_pipe(self): + cmd = 'echo "foo" =~ "bar" | bin/vint --stdin-display-name STDIN_TEST -' + + with self.assertRaises(subprocess.CalledProcessError) as context_manager: + subprocess.check_output(cmd, shell=True, universal_newlines=True) + + got_output = context_manager.exception.output + + expected_output_pattern = '^STDIN_TEST:' + self.assertRegex(got_output, expected_output_pattern) + + if __name__ == '__main__': unittest.main() From 037b2704c8f6ac245ef7ab68d5ee496508d76fcd Mon Sep 17 00:00:00 2001 From: Kuniwak Date: Fri, 29 Jun 2018 12:22:51 +0900 Subject: [PATCH 9/9] fixup! Support --stdin-alt-name and the mix style such as `vint a.vim - b.vim` --- .../linting/config/test_config_next_line_comment_source.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/unit/vint/linting/config/test_config_next_line_comment_source.py b/test/unit/vint/linting/config/test_config_next_line_comment_source.py index 42120a1..6ee74f0 100644 --- a/test/unit/vint/linting/config/test_config_next_line_comment_source.py +++ b/test/unit/vint/linting/config/test_config_next_line_comment_source.py @@ -1,6 +1,7 @@ import unittest import enum from vint.linting.linter import Linter +from vint.linting.lint_target import LintTargetFile from vint.linting.policy_set import PolicySet from vint.ast.node_type import NodeType from vint.linting.level import Level @@ -20,9 +21,10 @@ def test_simple_example(self): global_config_dict = {'cmdargs': {'severity': Level.ERROR}} policy_set = PolicySet([TestConfigNextLineCommentSource.ProhibitStringPolicy]) linter = Linter(policy_set, global_config_dict) + lint_target = LintTargetFile(Fixtures.SIMPLE.value) reported_string_node_values = [violation['description'] - for violation in linter.lint_file(Fixtures.SIMPLE.value)] + for violation in linter.lint(lint_target)] self.assertEqual(reported_string_node_values, [ "'report me because I have no line config comments'", @@ -34,9 +36,10 @@ def test_lambda_string_expr(self): global_config_dict = {'cmdargs': {'severity': Level.ERROR}} policy_set = PolicySet([TestConfigNextLineCommentSource.ProhibitStringPolicy]) linter = Linter(policy_set, global_config_dict) + lint_target = LintTargetFile(Fixtures.LAMBDA_STRING_EXPR.value) reported_string_node_values = [violation['description'] - for violation in linter.lint_file(Fixtures.LAMBDA_STRING_EXPR.value)] + for violation in linter.lint(lint_target)] self.assertEqual(reported_string_node_values, [ "'report me because I have no line config comments'",