Skip to content

Commit

Permalink
multi-file add
Browse files Browse the repository at this point in the history
  • Loading branch information
casperdcl committed Oct 14, 2019
1 parent 931e7f4 commit 339380f
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 38 deletions.
19 changes: 7 additions & 12 deletions dvc/command/add.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase, append_doc_link
from dvc.exceptions import RecursiveAddingWhileUsingFilename
from dvc.progress import Tqdm


logger = logging.getLogger(__name__)
Expand All @@ -20,17 +19,13 @@ def run(self):
"can't use '--file' with multiple targets"
)

with Tqdm(
total=len(self.args.targets), desc="Adding", unit="file"
) as pbar:
for target in self.args.targets:
self.repo.add(
target,
recursive=self.args.recursive,
no_commit=self.args.no_commit,
fname=self.args.file,
pbar=pbar,
)
self.repo.add(
self.args.targets,
recursive=self.args.recursive,
no_commit=self.args.no_commit,
fname=self.args.file,
progress=True,
)

except DvcException as err:
logger.exception("{}:{}".format(type(err).__name__, err))
Expand Down
71 changes: 45 additions & 26 deletions dvc/repo/add.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,54 +3,73 @@
import os
import logging
import colorama
from six import string_types

from dvc.repo.scm_context import scm_context
from dvc.stage import Stage
from dvc.utils import walk_files, LARGE_DIR_SIZE
from dvc.exceptions import RecursiveAddingWhileUsingFilename

from dvc.progress import Tqdm
from . import locked


logger = logging.getLogger(__name__)


@locked
@scm_context
def add(repo, target, recursive=False, no_commit=False, fname=None, pbar=None):
def add(
repo,
target_list,
recursive=False,
no_commit=False,
fname=None,
progress=False,
):
if recursive and fname:
raise RecursiveAddingWhileUsingFilename()

targets = _find_all_targets(repo, target, recursive)
if pbar is not None:
pbar.total += len(targets) - 1
if isinstance(target_list, string_types):
target_list = [target_list]

if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
logger.warning(
"You are adding a large directory '{target}' recursively,"
" consider tracking it as a whole instead.\n"
"{purple}HINT:{nc} Remove the generated DVC-file and then"
" run {cyan}dvc add {target}{nc}".format(
purple=colorama.Fore.MAGENTA,
cyan=colorama.Fore.CYAN,
nc=colorama.Style.RESET_ALL,
target=target,
)
)
stages_list = []
with Tqdm(
total=len(target_list),
desc="Adding",
unit="file",
disable=not progress,
) as pbar:
for target in target_list:
targets = _find_all_targets(repo, target, recursive)
pbar.total += len(targets) - 1

stages = _create_stages(repo, targets, fname, pbar=pbar)
if os.path.isdir(target) and len(targets) > LARGE_DIR_SIZE:
logger.warning(
"You are adding a large directory '{target}' recursively,"
" consider tracking it as a whole instead.\n"
"{purple}HINT:{nc} Remove the generated DVC-file and then"
" run {cyan}dvc add {target}{nc}".format(
purple=colorama.Fore.MAGENTA,
cyan=colorama.Fore.CYAN,
nc=colorama.Style.RESET_ALL,
target=target,
)
)

repo.check_modified_graph(stages)
stages = _create_stages(repo, targets, fname, pbar=pbar)

for stage in stages:
stage.save()
repo.check_modified_graph(stages)

if not no_commit:
stage.commit()
for stage in stages:
stage.save()

stage.dump()
if not no_commit:
stage.commit()

return stages
stage.dump()

stages_list += stages

return stages_list


def _find_all_targets(repo, target, recursive):
Expand Down

0 comments on commit 339380f

Please sign in to comment.