diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..dd8b293 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,407 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,missing-docstring,fixme,superfluous-parens + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=145 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 59e942e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:2.7 - -RUN pip install boto3 -RUN pip install botocore - -COPY ./openshift-install / -COPY ./process-ignition-manifests-and-kubeconfig.py / - -ENTRYPOINT ["python", "process-ignition-manifests-and-kubeconfig.py"] \ No newline at end of file diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 0000000..8fa2b83 --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,9 @@ +FROM python:3 + +COPY requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt + +COPY dev-requirements.txt /tmp/dev-requirements.txt +RUN pip install -r /tmp/dev-requirements.txt + +RUN rm /tmp/*requirements.txt diff --git a/Dockerfile.baremetal b/Dockerfile.ignition-manifests-and-kubeconfig-generate similarity index 76% rename from Dockerfile.baremetal rename to Dockerfile.ignition-manifests-and-kubeconfig-generate index b87ce73..cdd2060 100644 --- a/Dockerfile.baremetal +++ b/Dockerfile.ignition-manifests-and-kubeconfig-generate @@ -18,9 +18,9 @@ RUN chmod 777 $WORK_DIR # [TODO] - change this line to use openshift-installer from the release, once we are ready COPY --from=0 /root/installer/openshift-install $WORK_DIR -COPY ./process-ignition-manifests-and-kubeconfig.py $WORK_DIR +COPY ./render_files.py $WORK_DIR ENV WORK_DIR=$WORK_DIR -ENV EXEC_PATH=$WORK_DIR/process-ignition-manifests-and-kubeconfig.py -ENTRYPOINT python $EXEC_PATH +ENV EXEC_PATH=$WORK_DIR/render_files.py +ENTRYPOINT python $EXEC_PATH \ No newline at end of file diff --git a/Dockerfile.installer-image b/Dockerfile.installer-image index 15a247b..2625f40 100644 --- a/Dockerfile.installer-image +++ b/Dockerfile.installer-image @@ -1,3 +1,5 @@ +#This dockerfile require an openshift installer binary in CWD FROM alpine:latest RUN mkdir /root/installer COPY ./openshift-install /root/installer + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8980769 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +all: pep8 pylint build + +build: + skipper build ignition-manifests-and-kubeconfig-generate + +.DEFAULT: + skipper -v $(MAKE) $@ \ No newline at end of file diff --git a/Makefile.skipper b/Makefile.skipper new file mode 100644 index 0000000..02b2722 --- /dev/null +++ b/Makefile.skipper @@ -0,0 +1,14 @@ +all: pep8 pylint + +pep8: + pep8 --max-line-length=145 render_files.py + +pylint: + mkdir -p reports + PYLINTHOME=reports/ pylint render_files.py + +clean: + rm -rf build dist *egg-info ./__pycache__ + find -name *.pyc -delete + +.PHONY: pep8 pylint clean diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..cfa44af --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,2 @@ +pep8 +pylint diff --git a/process-ignition-manifests-and-kubeconfig.py b/render_files.py similarity index 63% rename from process-ignition-manifests-and-kubeconfig.py rename to render_files.py index 26dcb4e..78cdb03 100644 --- a/process-ignition-manifests-and-kubeconfig.py +++ b/render_files.py @@ -1,34 +1,31 @@ #!/usr/bin/env python +import argparse +import json +import logging import subprocess -import random +import sys import os import boto3 from botocore.exceptions import NoCredentialsError -from base64 import b64decode -import logging -import argparse -import sys -import json def get_s3_client(s3_endpoint_url): + aws_access_key_id = os.environ.get("aws_access_key_id", "accessKey1") + aws_secret_access_key = os.environ.get("aws_secret_access_key", "verySecretKey1") - aws_access_key_id = os.environ.get("aws_access_key_id", "accessKey1") - aws_secret_access_key = os.environ.get("aws_secret_access_key", "verySecretKey1") - - s3 = boto3.client( - 's3', - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - endpoint_url=s3_endpoint_url - ) - return s3 + s3_client = boto3.client( + 's3', + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + endpoint_url=s3_endpoint_url + ) + return s3_client -def upload_to_aws(s3, local_file, bucket, s3_file): +def upload_to_aws(s3_client, local_file, bucket, s3_file): try: - s3.upload_file(local_file, bucket, s3_file, ExtraArgs={'ACL': 'public-read'}) + s3_client.upload_file(local_file, bucket, s3_file, ExtraArgs={'ACL': 'public-read'}) print("Upload Successful") return True except NoCredentialsError: @@ -38,43 +35,43 @@ def upload_to_aws(s3, local_file, bucket, s3_file): def remove_bmo_provisioning(ignition_file): found = False - with open(ignition_file, "r") as f: - data = json.load(f) - storageFiles = data['storage']['files'] + with open(ignition_file, "r") as file_obj: + data = json.load(file_obj) + storage_files = data['storage']['files'] # Iterate through a copy of the list - for fileData in storageFiles[:]: - if 'baremetal-provisioning-config' in fileData['path']: - storageFiles.remove(fileData) + for file_data in storage_files[:]: + if 'baremetal-provisioning-config' in file_data['path']: + storage_files.remove(file_data) found = True break if found: - with open(ignition_file,"w") as f: - json.dump(data, f) + with open(ignition_file, "w") as file_obj: + json.dump(data, file_obj) def upload_to_s3(s3_endpoint_url, bucket, install_dir): - s3 = get_s3_client(s3_endpoint_url) + s3_client = get_s3_client(s3_endpoint_url) prefix = os.environ.get("CLUSTER_ID") for root, _, files in os.walk(install_dir): - for f in files: - logging.info("Uplading file: {}".format(f)) - file_path = os.path.join(root, f) - if f == "kubeconfig": - f = "kubeconfig-noingress" - s3_file_name = "{}/{}".format(prefix, f) + for file_name in files: + logging.info("Uploading file: %s", file_name) + file_path = os.path.join(root, file_name) + if file_name == "kubeconfig": + file_name = "kubeconfig-noingress" + s3_file_name = "{}/{}".format(prefix, file_name) print(s3_file_name) - uploaded = upload_to_aws(s3, file_path, bucket, s3_file_name) + upload_to_aws(s3_client, file_path, bucket, s3_file_name) def debug_print_upload_to_s3(install_dir): prefix = "dummy_cluster_id" for root, _, files in os.walk(install_dir): - for f in files: - file_path = os.path.join(root, f) - if f == "kubeconfig": - f = "kubeconfig-noingress" - s3_file_name = "{}/{}".format(prefix, f) + for file_name in files: + file_path = os.path.join(root, file_name) + if file_name == "kubeconfig": + file_name = "kubeconfig-noingress" + s3_file_name = "{}/{}".format(prefix, file_name) print("Uploading file %s as object %s" % (file_path, s3_file_name)) @@ -92,9 +89,8 @@ def main(): config_dir = os.path.join(work_dir, "installer_dir") if install_config: subprocess.check_output(["mkdir", "-p", config_dir]) - with open(os.path.join(config_dir, 'install-config.yaml'), 'w+') as f: - f.write(install_config) - + with open(os.path.join(config_dir, 'install-config.yaml'), 'w+') as file_obj: + file_obj.write(install_config) if not os.path.isdir(config_dir): raise Exception('installer directory is not mounted') @@ -107,7 +103,6 @@ def main(): except Exception as ex: raise Exception('Failed to generate files, exception: {}'.format(ex)) - try: remove_bmo_provisioning("%s/bootstrap.ign" % config_dir) except Exception as ex: diff --git a/reports/process-ignition-manifests-and-kubeconfig1.stats b/reports/process-ignition-manifests-and-kubeconfig1.stats new file mode 100644 index 0000000..4fb324a Binary files /dev/null and b/reports/process-ignition-manifests-and-kubeconfig1.stats differ diff --git a/reports/render_files1.stats b/reports/render_files1.stats new file mode 100644 index 0000000..55142c0 Binary files /dev/null and b/reports/render_files1.stats differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..22ca957 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +boto3 +botocore diff --git a/skipper.yaml b/skipper.yaml new file mode 100644 index 0000000..6d221a5 --- /dev/null +++ b/skipper.yaml @@ -0,0 +1,3 @@ +build-container-image: build +make: + makefile: Makefile.skipper \ No newline at end of file