switch to more minimalist version

drkane · Apr 20, 2020 · 77be0fa · 77be0fa
1 parent b1eaae2
commit 77be0fa
Show file tree

Hide file tree

Showing 36 changed files with 42,269 additions and 2,173 deletions.
diff --git a/.github/workflows/scrapeall.yml b/.github/workflows/scrapeall.yml
@@ -0,0 +1,76 @@
+name: Get latest cascs
+
+on:
+  push:
+  schedule:
+    - cron: '5 2 * * 1'
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Crawl CASC
+      run: |
+        scrapy crawl casc
+    - name: Crawl CCNI
+      run: |
+        scrapy crawl ccni
+    - name: Crawl government organisations
+      run: |
+        scrapy crawl gor
+    - name: Crawl GRID
+      run: |
+        scrapy crawl grid
+    - name: Crawl Hesa
+      run: |
+        scrapy crawl hesa
+    - name: Crawl Local Authorities in England
+      run: |
+        scrapy crawl lae
+    - name: Crawl Local Authorities in Northern Ireland
+      run: |
+        scrapy crawl lani
+    - name: Crawl Local Authorities in Scotland
+      run: |
+        scrapy crawl las
+    - name: Crawl NHS organisations
+      run: |
+        scrapy crawl nhsods
+    - name: Crawl Local Authorities in Wales
+      run: |
+        scrapy crawl pla
+    - name: Crawl Local Authorities in Wales
+      run: |
+        scrapy crawl pla
+    - name: Crawl Registered Social Landlords
+      run: |
+        scrapy crawl rsl
+    - name: Crawl Northern Ireland Schools
+      run: |
+        scrapy crawl schools_ni
+    - name: Crawl Scotland Schools
+      run: |
+        scrapy crawl schools_scotland
+    - name: Crawl Wales Schools
+      run: |
+        scrapy crawl schools_wales
+    - name: Commit files
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "GitHub Action"
+        git diff --quiet && git diff --staged --quiet || (git commit -m "Update by GitHub Action" -a)
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/Procfile b/Procfile
diff --git a/alembic.ini b/alembic.ini
diff --git a/alembic/README b/alembic/README
diff --git a/alembic/env.py b/alembic/env.py
diff --git a/alembic/script.py.mako b/alembic/script.py.mako
diff --git a/alembic/versions/14feaf6b8df0_orgids_field_name.py b/alembic/versions/14feaf6b8df0_orgids_field_name.py
diff --git a/alembic/versions/2c338e0e9ed7_add_scraping_table.py b/alembic/versions/2c338e0e9ed7_add_scraping_table.py