diff --git a/lib/atdisplanningalertsfeed.rb b/lib/atdisplanningalertsfeed.rb index e40bdab..7f5a1f1 100644 --- a/lib/atdisplanningalertsfeed.rb +++ b/lib/atdisplanningalertsfeed.rb @@ -11,10 +11,18 @@ def self.save(url, options = {}) page = feed.applications(lodgement_date_start: options[:lodgement_date_start], lodgement_date_end: options[:lodgement_date_end]) # Save the first page - save_page(page) + pages_processed = [] + pages_processed << page.pagination.current if save_page(page) while page = page.next_page - save_page(page) + # Some ATDIS feeds incorrectly provide pagination + # and permit looping; so halt processing if we've already processed this page + unless pages_processed.index(page.pagination.current).nil? + puts "Page #{page.pagination.current} already processed; halting" + break + end + + pages_processed << page.pagination.current if save_page(page) end end