From f0f2bde789259d6b82fe87daefdd3e2720db28bf Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 2 Mar 2017 21:16:41 -0500 Subject: [PATCH 1/3] detect cycles at end of compilation --- dbt/compilation.py | 5 +++++ dbt/linker.py | 12 ++++++++++++ test/unit/test_linker.py | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/dbt/compilation.py b/dbt/compilation.py index 7feb0e4edd7..e7fc5bce45d 100644 --- a/dbt/compilation.py +++ b/dbt/compilation.py @@ -464,6 +464,11 @@ def compile_nodes(self, linker, nodes, macro_generator): "dependency {} not found in graph!".format( dependency)) + cycle = linker.find_cycles() + + if cycle: + raise RuntimeError("Found a cycle: {}".format(cycle)) + return wrapped_nodes, written_nodes def generate_macros(self, all_macros): diff --git a/dbt/linker.py b/dbt/linker.py index a56dc29777b..d7c3765c6d0 100644 --- a/dbt/linker.py +++ b/dbt/linker.py @@ -28,6 +28,18 @@ def run_type(self): def get_node(self, node): return self.graph.node[node] + def find_cycles(self): + try: + cycles = nx.algorithms.find_cycle(self.graph) + except nx.exception.NetworkXNoCycle: + return None + + if cycles: + return " --> ".join([".".join(node) for node in cycles]) + + return None + + def as_topological_ordering(self, limit_to=None): try: return nx.topological_sort(self.graph, nbunch=limit_to) diff --git a/test/unit/test_linker.py b/test/unit/test_linker.py index 9b9b77cb8d2..0cbf81645a2 100644 --- a/test/unit/test_linker.py +++ b/test/unit/test_linker.py @@ -70,3 +70,19 @@ def test_linker_bad_limit_throws_runtime_error(self): self.linker.dependency(l, r) self.assertRaises(RuntimeError, self.linker.as_dependency_list, ['ZZZ']) + + def test__find_cycles__cycles(self): + actual_deps = [('A', 'B'), ('B', 'C'), ('C', 'A')] + + for (l, r) in actual_deps: + self.linker.dependency(l, r) + + self.assertIsNotNone(self.linker.find_cycles()) + + def test__find_cycles__no_cycles(self): + actual_deps = [('A', 'B'), ('B', 'C'), ('C', 'D')] + + for (l, r) in actual_deps: + self.linker.dependency(l, r) + + self.assertIsNone(self.linker.find_cycles()) From 5648af63e61ed4560fdef0c6c78825ce75b71d77 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 2 Mar 2017 21:19:21 -0500 Subject: [PATCH 2/3] actually stop looking for cycles in loop --- dbt/linker.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dbt/linker.py b/dbt/linker.py index d7c3765c6d0..0fe716ecbb6 100644 --- a/dbt/linker.py +++ b/dbt/linker.py @@ -39,7 +39,6 @@ def find_cycles(self): return None - def as_topological_ordering(self, limit_to=None): try: return nx.topological_sort(self.graph, nbunch=limit_to) @@ -115,11 +114,6 @@ def dependency(self, node1, node2): self.graph.add_node(node2) self.graph.add_edge(node2, node1) - if len(list(nx.simple_cycles(self.graph))) > 0: - raise ValidationException( - "Detected a cycle when adding dependency from {} to {}" - .format(node1, node2)) - def add_node(self, node): self.graph.add_node(node) From 4504d2395b1d10e4106f23bd564ec4e764f24e14 Mon Sep 17 00:00:00 2001 From: Connor McArthur Date: Thu, 2 Mar 2017 23:26:11 -0500 Subject: [PATCH 3/3] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a01d69138d1..a66a3e98b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Changes - Graph refactor: fix common issues with load order ([#292](https://github.com/fishtown-analytics/dbt/pull/292)) +- Speedup: detect cycles at the end of compilation ([#307](https://github.com/fishtown-analytics/dbt/pull/307)) ## dbt 0.7.1 (February 28, 2017)