wbrawner/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support comments in ansible-test flat files.

Browse source 
(cherry picked from commit 5a3000af19)
This commit is contained in:
Matt Clay 2018-09-19 23:20:27 -07:00 committed by Toshio Kuratomi
parent 7842b99106
commit cfe86582f2
13 changed files with 135 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
test/runner/lib/cli.py
View file

@ -12,6 +12,7 @@ from lib.util import (
raw_command,
get_docker_completion,
generate_pip_command,
read_lines_without_comments,
)
from lib.delegation import (
@ -687,8 +688,7 @@ def complete_remote(prefix, parsed_args, **_):
"""
del parsed_args
with open('test/runner/completion/remote.txt', 'r') as completion_fd:
images = completion_fd.read().splitlines()
images = read_lines_without_comments('test/runner/completion/remote.txt', remove_blank_lines=True)
return [i for i in images if i.startswith(prefix)]
@ -701,12 +701,10 @@ def complete_remote_shell(prefix, parsed_args, **_):
"""
del parsed_args
with open('test/runner/completion/remote.txt', 'r') as completion_fd:
images = completion_fd.read().splitlines()
images = read_lines_without_comments('test/runner/completion/remote.txt', remove_blank_lines=True)
# 2008 doesn't support SSH so we do not add to the list of valid images
with open('test/runner/completion/windows.txt', 'r') as completion_fd:
images.extend(["windows/%s" % i for i in completion_fd.read().splitlines() if i != '2008'])
images.extend(["windows/%s" % i for i in read_lines_without_comments('test/runner/completion/windows.txt', remove_blank_lines=True) if i != '2008'])
return [i for i in images if i.startswith(prefix)]
@ -730,8 +728,7 @@ def complete_windows(prefix, parsed_args, **_):
:type parsed_args: any
:rtype: list[str]
"""
with open('test/runner/completion/windows.txt', 'r') as completion_fd:
images = completion_fd.read().splitlines()
images = read_lines_without_comments('test/runner/completion/windows.txt', remove_blank_lines=True)
return [i for i in images if i.startswith(prefix) and (not parsed_args.windows or i not in parsed_args.windows)]
@ -742,8 +739,7 @@ def complete_network_platform(prefix, parsed_args, **_):
:type parsed_args: any
:rtype: list[str]
"""
with open('test/runner/completion/network.txt', 'r') as completion_fd:
images = completion_fd.read().splitlines()
images = read_lines_without_comments('test/runner/completion/network.txt', remove_blank_lines=True)
return [i for i in images if i.startswith(prefix) and (not parsed_args.platform or i not in parsed_args.platform)]

5
test/runner/lib/sanity/__init__.py
View file

@ -18,6 +18,7 @@ from lib.util import (
parse_to_dict,
ABC,
is_binary_file,
read_lines_without_comments,
)
from lib.ansible_util import (
@ -134,8 +135,8 @@ def collect_code_smell_tests():
"""
:rtype: tuple[SanityCodeSmellTest]
"""
with open('test/sanity/code-smell/skip.txt', 'r') as skip_fd:
skip_tests = skip_fd.read().splitlines()
skip_file = 'test/sanity/code-smell/skip.txt'
skip_tests = read_lines_without_comments(skip_file, remove_blank_lines=True)
paths = glob.glob('test/sanity/code-smell/*')
paths = sorted(p for p in paths if os.access(p, os.X_OK) and os.path.isfile(p) and os.path.basename(p) not in skip_tests)

5
test/runner/lib/sanity/ansible_doc.py
View file

@ -17,6 +17,7 @@ from lib.util import (
SubprocessError,
display,
intercept_command,
read_lines_without_comments,
)
from lib.ansible_util import (
@ -37,8 +38,8 @@ class AnsibleDocTest(SanityMultipleVersion):
:type python_version: str
:rtype: TestResult
"""
with open('test/sanity/ansible-doc/skip.txt', 'r') as skip_fd:
skip_modules = set(skip_fd.read().splitlines())
skip_file = 'test/sanity/ansible-doc/skip.txt'
skip_modules = set(read_lines_without_comments(skip_file, remove_blank_lines=True))
plugin_type_blacklist = set([
# not supported by ansible-doc

8
test/runner/lib/sanity/compile.py
View file

@ -17,6 +17,7 @@ from lib.util import (
run_command,
display,
find_python,
read_lines_without_comments,
)
from lib.config import (
@ -37,12 +38,10 @@ class CompileTest(SanityMultipleVersion):
:type python_version: str
:rtype: TestResult
"""
# optional list of regex patterns to exclude from tests
skip_file = 'test/sanity/compile/python%s-skip.txt' % python_version
if os.path.exists(skip_file):
with open(skip_file, 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
skip_paths = read_lines_without_comments(skip_file)
else:
skip_paths = []
@ -87,6 +86,9 @@ class CompileTest(SanityMultipleVersion):
for path in skip_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
results.append(SanityMessage(

8
test/runner/lib/sanity/import.py
View file

@ -19,6 +19,7 @@ from lib.util import (
remove_tree,
display,
find_python,
read_lines_without_comments,
)
from lib.ansible_util import (
@ -43,9 +44,8 @@ class ImportTest(SanityMultipleVersion):
:type python_version: str
:rtype: TestResult
"""
with open('test/sanity/import/skip.txt', 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
skip_file = 'test/sanity/import/skip.txt'
skip_paths = read_lines_without_comments(skip_file, remove_blank_lines=True)
skip_paths_set = set(skip_paths)
paths = sorted(
@ -121,7 +121,7 @@ class ImportTest(SanityMultipleVersion):
column=int(r['column']),
) for r in results]
results = [result for result in results if result.path not in skip_paths]
results = [result for result in results if result.path not in skip_paths_set]
if results:
return SanityFailure(self.name, messages=results, python_version=python_version)

22
test/runner/lib/sanity/pep8.py
View file

@ -15,6 +15,7 @@ from lib.util import (
SubprocessError,
display,
run_command,
read_lines_without_comments,
)
from lib.config import (
@ -37,17 +38,14 @@ class Pep8Test(SanitySingleVersion):
:type targets: SanityTargets
:rtype: TestResult
"""
with open(PEP8_SKIP_PATH, 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
skip_paths = read_lines_without_comments(PEP8_SKIP_PATH)
legacy_paths = read_lines_without_comments(PEP8_LEGACY_PATH)
with open(PEP8_LEGACY_PATH, 'r') as legacy_fd:
legacy_paths = legacy_fd.read().splitlines()
legacy_ignore_file = 'test/sanity/pep8/legacy-ignore.txt'
legacy_ignore = set(read_lines_without_comments(legacy_ignore_file, remove_blank_lines=True))
with open('test/sanity/pep8/legacy-ignore.txt', 'r') as ignore_fd:
legacy_ignore = set(ignore_fd.read().splitlines())
with open('test/sanity/pep8/current-ignore.txt', 'r') as ignore_fd:
current_ignore = sorted(ignore_fd.read().splitlines())
current_ignore_file = 'test/sanity/pep8/current-ignore.txt'
current_ignore = sorted(read_lines_without_comments(current_ignore_file, remove_blank_lines=True))
skip_paths_set = set(skip_paths)
legacy_paths_set = set(legacy_paths)
@ -106,6 +104,9 @@ class Pep8Test(SanitySingleVersion):
for path in legacy_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(SanityMessage(
@ -133,6 +134,9 @@ class Pep8Test(SanitySingleVersion):
for path in skip_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(SanityMessage(

37
test/runner/lib/sanity/pslint.py
View file

@ -18,6 +18,7 @@ from lib.util import (
SubprocessError,
run_command,
find_executable,
read_lines_without_comments,
)
from lib.config import (
@ -41,30 +42,31 @@ class PslintTest(SanitySingleVersion):
:type targets: SanityTargets
:rtype: TestResult
"""
with open(PSLINT_SKIP_PATH, 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
skip_paths = read_lines_without_comments(PSLINT_SKIP_PATH)
invalid_ignores = []
with open(PSLINT_IGNORE_PATH, 'r') as ignore_fd:
ignore_entries = ignore_fd.read().splitlines()
ignore = collections.defaultdict(dict)
line = 0
ignore_entries = read_lines_without_comments(PSLINT_IGNORE_PATH)
ignore = collections.defaultdict(dict)
line = 0
for ignore_entry in ignore_entries:
line += 1
for ignore_entry in ignore_entries:
line += 1
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
if not ignore_entry:
continue
path, code = ignore_entry.split(' ', 1)
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
if not os.path.exists(path):
invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path))
continue
path, code = ignore_entry.split(' ', 1)
ignore[path][code] = line
if not os.path.exists(path):
invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path))
continue
ignore[path][code] = line
paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.ps1', '.psm1', '.psd1') and i.path not in skip_paths)
@ -138,6 +140,9 @@ class PslintTest(SanitySingleVersion):
for path in skip_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(SanityMessage(

57
test/runner/lib/sanity/pylint.py
View file

@ -24,6 +24,7 @@ from lib.util import (
run_command,
display,
find_executable,
read_lines_without_comments,
)
from lib.executor import (
@ -69,43 +70,44 @@ class PylintTest(SanitySingleVersion):
display.warning('Skipping pylint on unsupported Python version %s.' % args.python_version)
return SanitySkipped(self.name)
with open(PYLINT_SKIP_PATH, 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
skip_paths = read_lines_without_comments(PYLINT_SKIP_PATH)
invalid_ignores = []
supported_versions = set(SUPPORTED_PYTHON_VERSIONS) - set(UNSUPPORTED_PYTHON_VERSIONS)
supported_versions = set([v.split('.')[0] for v in supported_versions]) | supported_versions
with open(PYLINT_IGNORE_PATH, 'r') as ignore_fd:
ignore_entries = ignore_fd.read().splitlines()
ignore = collections.defaultdict(dict)
line = 0
ignore_entries = read_lines_without_comments(PYLINT_IGNORE_PATH)
ignore = collections.defaultdict(dict)
line = 0
for ignore_entry in ignore_entries:
line += 1
for ignore_entry in ignore_entries:
line += 1
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
if not ignore_entry:
continue
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
path, code = ignore_entry.split(' ', 1)
if not os.path.exists(path):
invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path))
continue
if ' ' in code:
code, version = code.split(' ', 1)
if version not in supported_versions:
invalid_ignores.append((line, 'Invalid version: %s' % version))
continue
path, code = ignore_entry.split(' ', 1)
if version != args.python_version and version != args.python_version.split('.')[0]:
continue # ignore version specific entries for other versions
if not os.path.exists(path):
invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path))
continue
if ' ' in code:
code, version = code.split(' ', 1)
if version not in supported_versions:
invalid_ignores.append((line, 'Invalid version: %s' % version))
continue
if version != args.python_version and version != args.python_version.split('.')[0]:
continue # ignore version specific entries for other versions
ignore[path][code] = line
ignore[path][code] = line
skip_paths_set = set(skip_paths)
@ -193,6 +195,9 @@ class PylintTest(SanitySingleVersion):
for path in skip_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(SanityMessage(

5
test/runner/lib/sanity/rstcheck.py
View file

@ -17,6 +17,7 @@ from lib.util import (
parse_to_dict,
display,
find_executable,
read_lines_without_comments,
)
from lib.config import (
@ -40,8 +41,8 @@ class RstcheckTest(SanitySingleVersion):
display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version)
return SanitySkipped(self.name)
with open('test/sanity/rstcheck/ignore-substitutions.txt', 'r') as ignore_fd:
ignore_substitutions = sorted(set(ignore_fd.read().splitlines()))
ignore_file = 'test/sanity/rstcheck/ignore-substitutions.txt'
ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True)))
paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.rst',))

9
test/runner/lib/sanity/shellcheck.py
View file

@ -19,6 +19,7 @@ from lib.sanity import (
from lib.util import (
SubprocessError,
run_command,
read_lines_without_comments,
)
from lib.config import (
@ -34,11 +35,11 @@ class ShellcheckTest(SanitySingleVersion):
:type targets: SanityTargets
:rtype: TestResult
"""
with open('test/sanity/shellcheck/skip.txt', 'r') as skip_fd:
skip_paths = set(skip_fd.read().splitlines())
skip_file = 'test/sanity/shellcheck/skip.txt'
skip_paths = set(read_lines_without_comments(skip_file, remove_blank_lines=True))
with open('test/sanity/shellcheck/exclude.txt', 'r') as exclude_fd:
exclude = set(exclude_fd.read().splitlines())
exclude_file = 'test/sanity/shellcheck/exclude.txt'
exclude = set(read_lines_without_comments(exclude_file, remove_blank_lines=True))
paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths)

34
test/runner/lib/sanity/validate_modules.py
View file

@ -17,6 +17,7 @@ from lib.util import (
SubprocessError,
display,
run_command,
read_lines_without_comments,
)
from lib.ansible_util import (
@ -44,9 +45,7 @@ class ValidateModulesTest(SanitySingleVersion):
:type targets: SanityTargets
:rtype: TestResult
"""
with open(VALIDATE_SKIP_PATH, 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
skip_paths = read_lines_without_comments(VALIDATE_SKIP_PATH)
skip_paths_set = set(skip_paths)
env = ansible_environment(args, color=False)
@ -65,21 +64,23 @@ class ValidateModulesTest(SanitySingleVersion):
invalid_ignores = []
with open(VALIDATE_IGNORE_PATH, 'r') as ignore_fd:
ignore_entries = ignore_fd.read().splitlines()
ignore = collections.defaultdict(dict)
line = 0
ignore_entries = read_lines_without_comments(VALIDATE_IGNORE_PATH)
ignore = collections.defaultdict(dict)
line = 0
for ignore_entry in ignore_entries:
line += 1
for ignore_entry in ignore_entries:
line += 1
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
if not ignore_entry:
continue
path, code = ignore_entry.split(' ', 1)
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
ignore[path][code] = line
path, code = ignore_entry.split(' ', 1)
ignore[path][code] = line
if args.base_branch:
cmd.extend([
@ -139,9 +140,14 @@ class ValidateModulesTest(SanitySingleVersion):
confidence=calculate_confidence(VALIDATE_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None,
))
line = 0
for path in skip_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(SanityMessage(

5
test/runner/lib/target.py
View file

@ -12,6 +12,7 @@ import sys
from lib.util import (
ApplicationError,
read_lines_without_comments,
)
MODULE_EXTENSIONS = '.py', '.ps1'
@ -511,8 +512,8 @@ class IntegrationTarget(CompletionTarget):
# static_aliases
try:
with open(os.path.join(path, 'aliases'), 'r') as aliases_file:
static_aliases = tuple(aliases_file.read().splitlines())
aliases_path = os.path.join(path, 'aliases')
static_aliases = tuple(read_lines_without_comments(aliases_path, remove_blank_lines=True))
except IOError as ex:
if ex.errno != errno.ENOENT:
raise

20
test/runner/lib/util.py
View file

@ -42,8 +42,7 @@ def get_docker_completion():
:rtype: dict[str, str]
"""
if not DOCKER_COMPLETION:
with open('test/runner/completion/docker.txt', 'r') as completion_fd:
images = completion_fd.read().splitlines()
images = read_lines_without_comments('test/runner/completion/docker.txt', remove_blank_lines=True)
DOCKER_COMPLETION.update(dict(kvp for kvp in [parse_docker_completion(i) for i in images] if kvp))
@ -81,6 +80,23 @@ def remove_file(path):
os.remove(path)
def read_lines_without_comments(path, remove_blank_lines=False):
"""
:type path: str
:type remove_blank_lines: bool
:rtype: list[str]
"""
with open(path, 'r') as path_fd:
lines = path_fd.read().splitlines()
lines = [re.sub(r' *#.*$', '', line) for line in lines]
if remove_blank_lines:
lines = [line for line in lines if line]
return lines
def find_executable(executable, cwd=None, path=None, required=True):
"""
:type executable: str