From 0c117aef1bff896a0b2d12ca0df1098ffedc9d45 Mon Sep 17 00:00:00 2001 From: Lim Hoang Date: Thu, 15 Apr 2021 15:57:03 +0100 Subject: [PATCH] [KED-2518] Fix starters for windows with python 3.7 (#1070) --- RELEASE.md | 1 + kedro/framework/cli/starters.py | 38 ++++++++++++++++++++-------- tests/framework/cli/test_starters.py | 6 +---- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index a90b753970..9089420781 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -23,6 +23,7 @@ print(pipelines["__default__"]) # pipeline loading is only triggered here * When `kedro new` is invoked using a configuration yaml file, `output_dir` is no longer a required key; by default the current working directory will be used. * When `kedro new` is invoked using a configuration yaml file, the appropriate `prompts.yml` file is now used for validating the provided configuration. Previously, validation was always performed against the kedro project template `prompts.yml` file. * When a relative path to a starter template is provided, `kedro new` now generates user prompts to obtain configuration rather than supplying empty configuration. +* Fixed error when using starter on Windows with Python 3.7 (Issue [#722](https://github.com/quantumblacklabs/kedro/issues/722)) ## Minor breaking changes to the API diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index 3159f46ad8..99702acd7c 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -32,11 +32,14 @@ This module implements commands available from the kedro CLI for creating projects. """ +import os import re +import shutil +import stat import tempfile from collections import OrderedDict from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple import click import git @@ -80,6 +83,15 @@ ) +# pylint: disable=unused-argument +def _remove_readonly(func: Callable, path: Path, excinfo: Tuple): # pragma: no cover + """Remove readonly files on Windows + See: https://docs.python.org/3/library/shutil.html?highlight=shutil#rmtree-example + """ + os.chmod(path, stat.S_IWRITE) + func(path) + + # pylint: disable=missing-function-docstring @click.group(context_settings=CONTEXT_SETTINGS, name="Kedro") def create_cli(): # pragma: no cover @@ -121,16 +133,20 @@ def new( template_path = str(TEMPLATE_PATH) # Get prompts.yml to find what information the user needs to supply as config. - with tempfile.TemporaryDirectory() as tmpdir: - cookiecutter_dir = _get_cookiecutter_dir( - template_path, checkout, directory, tmpdir - ) - prompts_required = _get_prompts_required(cookiecutter_dir) - # We only need to make cookiecutter_context if interactive prompts are needed. - if not config_path: - cookiecutter_context = _make_cookiecutter_context_for_prompts( - cookiecutter_dir - ) + + tmpdir = tempfile.mkdtemp() + cookiecutter_dir = _get_cookiecutter_dir(template_path, checkout, directory, tmpdir) + prompts_required = _get_prompts_required(cookiecutter_dir) + # We only need to make cookiecutter_context if interactive prompts are needed. + if not config_path: + cookiecutter_context = _make_cookiecutter_context_for_prompts(cookiecutter_dir) + + # Cleanup the tmpdir after it's no longer required. + # Ideally we would want to be able to use tempfile.TemporaryDirectory() context manager + # but it causes an issue with readonly files on windows + # see: https://bugs.python.org/issue26660. + # So onerror, we will attempt to clear the readonly bits and re-attempt the cleanup + shutil.rmtree(tmpdir, onerror=_remove_readonly) # Obtain config, either from a file or from interactive user prompts. if not prompts_required: diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index c553675d65..2d33021329 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -362,12 +362,8 @@ def test_new_with_invalid_starter_path_should_raise(self, fake_kedro_cli): ], ) def test_new_with_starter_alias( - self, alias, expected_starter_repo, fake_kedro_cli, mocker, tmp_path + self, alias, expected_starter_repo, fake_kedro_cli, mocker ): - mocker.patch( - "kedro.framework.cli.starters.tempfile.TemporaryDirectory", - return_value=tmp_path, - ) mocked_cookie = mocker.patch("cookiecutter.main.cookiecutter") CliRunner().invoke( fake_kedro_cli,