From 262c5456f726b0610bedca4b3a45a7550e9e60d4 Mon Sep 17 00:00:00 2001 From: palewire Date: Fri, 21 Jan 2022 13:30:35 -0800 Subject: [PATCH] Added __tags__ dunder to all scrapers --- warn/scrapers/ak.py | 3 +++ warn/scrapers/al.py | 3 +++ warn/scrapers/az.py | 3 +++ warn/scrapers/ca.py | 1 + warn/scrapers/co.py | 1 + warn/scrapers/ct.py | 6 ++---- warn/scrapers/dc.py | 1 + warn/scrapers/de.py | 6 ++---- warn/scrapers/fl.py | 1 + warn/scrapers/ga.py | 5 ++--- warn/scrapers/in.py | 6 ++---- warn/scrapers/ks.py | 1 + warn/scrapers/md.py | 1 + warn/scrapers/me.py | 5 ++--- warn/scrapers/mo.py | 1 + warn/scrapers/mt.py | 1 + warn/scrapers/ne.py | 6 ++---- warn/scrapers/nj.py | 6 ++---- warn/scrapers/ny.py | 1 + warn/scrapers/oh.py | 6 ++---- warn/scrapers/ok.py | 6 ++---- warn/scrapers/or.py | 6 +++--- warn/scrapers/ri.py | 1 + warn/scrapers/sd.py | 1 + warn/scrapers/tx.py | 1 + warn/scrapers/ut.py | 1 + warn/scrapers/va.py | 1 + warn/scrapers/vt.py | 5 ++--- warn/scrapers/wa.py | 6 ++---- warn/scrapers/wi.py | 1 + 30 files changed, 49 insertions(+), 44 deletions(-) diff --git a/warn/scrapers/ak.py b/warn/scrapers/ak.py index a0d00939..87f3ab7b 100644 --- a/warn/scrapers/ak.py +++ b/warn/scrapers/ak.py @@ -6,6 +6,9 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = [ + "html", +] def scrape( diff --git a/warn/scrapers/al.py b/warn/scrapers/al.py index 6227fc3c..05ae626c 100644 --- a/warn/scrapers/al.py +++ b/warn/scrapers/al.py @@ -7,6 +7,9 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = [ + "html", +] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/az.py b/warn/scrapers/az.py index b273bafc..2e67d8cd 100755 --- a/warn/scrapers/az.py +++ b/warn/scrapers/az.py @@ -5,6 +5,9 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = [ + "jobcenter", +] def scrape( diff --git a/warn/scrapers/ca.py b/warn/scrapers/ca.py index f04d7042..05cdf713 100644 --- a/warn/scrapers/ca.py +++ b/warn/scrapers/ca.py @@ -12,6 +12,7 @@ from ..cache import Cache __authors__ = ["zstumgoren", "Dilcia19", "ydoc5212"] +__tags__ = ["html", "pdf", "excel"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/co.py b/warn/scrapers/co.py index 5ee13b61..34c3e710 100644 --- a/warn/scrapers/co.py +++ b/warn/scrapers/co.py @@ -5,6 +5,7 @@ from .. import utils __authors__ = ["ydoc5212"] +__tags__ = ["google-sheets"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/ct.py b/warn/scrapers/ct.py index 28b55e01..a6a882db 100644 --- a/warn/scrapers/ct.py +++ b/warn/scrapers/ct.py @@ -6,10 +6,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/dc.py b/warn/scrapers/dc.py index b0e42e5f..1b34a99d 100755 --- a/warn/scrapers/dc.py +++ b/warn/scrapers/dc.py @@ -8,6 +8,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19", "shallotly"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/de.py b/warn/scrapers/de.py index 96cd36d3..0929820d 100644 --- a/warn/scrapers/de.py +++ b/warn/scrapers/de.py @@ -4,10 +4,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["jobcenter"] def scrape( diff --git a/warn/scrapers/fl.py b/warn/scrapers/fl.py index 87baeeff..56556acb 100755 --- a/warn/scrapers/fl.py +++ b/warn/scrapers/fl.py @@ -14,6 +14,7 @@ from ..cache import Cache __authors__ = ["zstumgoren", "Dilcia19", "shallotly"] +__tags__ = ["html", "pdf"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/ga.py b/warn/scrapers/ga.py index b3170f11..2abb1ac0 100644 --- a/warn/scrapers/ga.py +++ b/warn/scrapers/ga.py @@ -7,9 +7,8 @@ from .. import utils from ..cache import Cache -__authors__ = [ - "chriszs", -] +__authors__ = ["chriszs"] +__tags__ = ["html"] def scrape( diff --git a/warn/scrapers/in.py b/warn/scrapers/in.py index 9b3cbeb5..6818a90e 100644 --- a/warn/scrapers/in.py +++ b/warn/scrapers/in.py @@ -6,10 +6,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/ks.py b/warn/scrapers/ks.py index fa755e98..ea4ab3eb 100644 --- a/warn/scrapers/ks.py +++ b/warn/scrapers/ks.py @@ -5,6 +5,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["jobcenter"] def scrape( diff --git a/warn/scrapers/md.py b/warn/scrapers/md.py index a16fe8b6..a86655fa 100644 --- a/warn/scrapers/md.py +++ b/warn/scrapers/md.py @@ -9,6 +9,7 @@ from ..cache import Cache __authors__ = ["zstumgoren", "Dilcia19", "shallotly"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/me.py b/warn/scrapers/me.py index 09049731..16d4726a 100755 --- a/warn/scrapers/me.py +++ b/warn/scrapers/me.py @@ -4,9 +4,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", -] +__authors__ = ["zstumgoren"] +__tags__ = ["jobcenter"] def scrape( diff --git a/warn/scrapers/mo.py b/warn/scrapers/mo.py index c0b13b16..dd113b5e 100644 --- a/warn/scrapers/mo.py +++ b/warn/scrapers/mo.py @@ -7,6 +7,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19", "shallotly"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/mt.py b/warn/scrapers/mt.py index c20f475d..c1125307 100644 --- a/warn/scrapers/mt.py +++ b/warn/scrapers/mt.py @@ -7,6 +7,7 @@ from ..cache import Cache __authors__ = ["zstumgoren", "ydoc5212"] +__tags__ = ["html", "excel"] def scrape( diff --git a/warn/scrapers/ne.py b/warn/scrapers/ne.py index b11181bc..719fff80 100644 --- a/warn/scrapers/ne.py +++ b/warn/scrapers/ne.py @@ -7,10 +7,8 @@ from .. import utils from ..cache import Cache -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/nj.py b/warn/scrapers/nj.py index 10ac46fb..fdf42b0f 100644 --- a/warn/scrapers/nj.py +++ b/warn/scrapers/nj.py @@ -7,10 +7,8 @@ from .. import utils from ..cache import Cache -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/ny.py b/warn/scrapers/ny.py index f598f923..7b6004bc 100644 --- a/warn/scrapers/ny.py +++ b/warn/scrapers/ny.py @@ -7,6 +7,7 @@ from ..cache import Cache __authors__ = ["zstumgoren", "Dilcia19", "ydoc5212"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/oh.py b/warn/scrapers/oh.py index 87d96e92..0f024d1b 100644 --- a/warn/scrapers/oh.py +++ b/warn/scrapers/oh.py @@ -6,10 +6,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/ok.py b/warn/scrapers/ok.py index bc3b043c..e17f9f48 100644 --- a/warn/scrapers/ok.py +++ b/warn/scrapers/ok.py @@ -4,10 +4,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["jobcenter"] def scrape( diff --git a/warn/scrapers/or.py b/warn/scrapers/or.py index 47922daa..913ddd52 100644 --- a/warn/scrapers/or.py +++ b/warn/scrapers/or.py @@ -7,6 +7,7 @@ from ..cache import Cache __authors__ = ["zstumgoren", "Dilcia19", "ydoc5212"] +__tags__ = ["html", "excel"] logger = logging.getLogger(__name__) @@ -39,9 +40,8 @@ def scrape( # Convert the sheet to a list of lists row_list = [] - for r in list(worksheet.rows)[ - 2: - ]: # Skip the first two rows, which contain a crufty header + # Skip the first two rows, which contain a crufty header + for r in list(worksheet.rows)[2:]: column = [cell.value for cell in r] row_list.append(column) diff --git a/warn/scrapers/ri.py b/warn/scrapers/ri.py index 363247c2..89bdf2af 100644 --- a/warn/scrapers/ri.py +++ b/warn/scrapers/ri.py @@ -7,6 +7,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19", "ydoc5212"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/sd.py b/warn/scrapers/sd.py index e220f14b..123ca1cb 100644 --- a/warn/scrapers/sd.py +++ b/warn/scrapers/sd.py @@ -8,6 +8,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19", "ydoc5212"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/tx.py b/warn/scrapers/tx.py index 1f71e7ce..f706b0be 100644 --- a/warn/scrapers/tx.py +++ b/warn/scrapers/tx.py @@ -9,6 +9,7 @@ from ..cache import Cache __authors__ = ["Dilcia19", "ydoc5212"] +__tags__ = ["html", "excel"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/ut.py b/warn/scrapers/ut.py index e9ba883b..d7f05ccc 100644 --- a/warn/scrapers/ut.py +++ b/warn/scrapers/ut.py @@ -7,6 +7,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/va.py b/warn/scrapers/va.py index 025483c8..eb2548c1 100644 --- a/warn/scrapers/va.py +++ b/warn/scrapers/va.py @@ -6,6 +6,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19", "shallotly"] +__tags__ = ["html", "csv"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/vt.py b/warn/scrapers/vt.py index 150c70ad..135fd0a3 100755 --- a/warn/scrapers/vt.py +++ b/warn/scrapers/vt.py @@ -4,9 +4,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", -] +__authors__ = ["zstumgoren"] +__tags__ = ["jobcenter"] def scrape( diff --git a/warn/scrapers/wa.py b/warn/scrapers/wa.py index 4bc97039..270e6a45 100644 --- a/warn/scrapers/wa.py +++ b/warn/scrapers/wa.py @@ -7,10 +7,8 @@ from .. import utils -__authors__ = [ - "zstumgoren", - "Dilcia19", -] +__authors__ = ["zstumgoren", "Dilcia19"] +__tags__ = ["html"] logger = logging.getLogger(__name__) diff --git a/warn/scrapers/wi.py b/warn/scrapers/wi.py index 2f80a285..a60509d9 100644 --- a/warn/scrapers/wi.py +++ b/warn/scrapers/wi.py @@ -8,6 +8,7 @@ from .. import utils __authors__ = ["zstumgoren", "Dilcia19", "ydoc5212"] +__tags__ = ["html"] logger = logging.getLogger(__name__)