Move datadotparl scripts to their own directory.

README.md

@@ -14,8 +14,8 @@ The source of data for the 2010 election onwards is
 This data is scraped and stored locally for parsing, but needs updating manually
 using the following process.
 
-1. Change to the `scripts` directory.
-2. Run `./crawl-datadotparl-members`
+1. Change to the `scripts/datadotparl` directory.
+2. Run `./crawl-members`
 
 ## Update from data.parliament.uk
 
@@ -26,15 +26,5 @@ onwards.
 
 Before updating, you should run the data.parliament.uk scraper.
 
-1. Change to the `scripts` directory.
-2. Run `./update-members-from-datadotparliament` to parse the XML and update things.
-
-When done, you should update `people.xml`
-
-## Update `people.xml`
-
-`members/people.xml joins together members and offices, and should be
-`regenerated if either of these things changes.
-
-1. Change to the `members` directory.
-2. Run `./personsets.py` to update `people.xml`.
+1. Change to the `scripts/datadotparl` directory.
+2. Run `./update-members` to parse the XML and update things.

scripts/crawl-datadotparl-members → scripts/datadotparl/crawl-members

@@ -17,7 +17,7 @@ source_urls = {
 
 member_url_templ = 'http://data.parliament.uk/membersdataplatform/services/mnis/members/query/id=%s/FullBiog/'
 
-OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../rawdata/datadotparl')
+OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../rawdata/datadotparl')
 
 
 def write(url, fn):

scripts/json-add-new-parl-ids → scripts/datadotparl/json-add-new-parl-ids

@@ -5,7 +5,7 @@ from lxml import etree
 from popolo import Popolo
 
 cur_dir = os.path.dirname(__file__)
-source_file = os.path.join(cur_dir, '..', 'rawdata', 'datadotparl', 'all-current-commons.xml')
+source_file = os.path.join(cur_dir, '..', '..', 'rawdata', 'datadotparl', 'all-current-commons.xml')
 
 parser = etree.ETCompatXMLParser()
 etree.set_default_parser(parser)

scripts/update-members-from-datadotparliament → scripts/datadotparl/update-members

@@ -35,7 +35,7 @@ class Popolo(object):
             'organizations': self.organizations,
             'memberships': self.memberships,
         }.items()))
-        json.dump(out, open('../members/ministers-2010.json', 'w'), indent=2)
+        json.dump(out, open('../../members/ministers-2010.json', 'w'), indent=2)
 
     def add_membership(self, mship):
         self.memberships.append(mship)
@@ -165,23 +165,23 @@ def parse(fn, root_tag):
 
 parl_member_ids = set()
 for source_file in source_files:
-  parl_members_slim = parse('../rawdata/datadotparl/' + source_file + '.xml', 'Members')
+  parl_members_slim = parse('../../rawdata/datadotparl/' + source_file + '.xml', 'Members')
   parl_member_ids |= set(member.attrib['Member_Id'] for member in parl_members_slim)
   print 'Found %d members from %s.xml.' % (len(parl_members_slim), source_file)
 print 'Total of %d distinct people.' % len(parl_member_ids)
 
 # Enrich the data!
 parl_members = dict()
 for member_id in parl_member_ids:
-  member_fullbio_xml = parse('../rawdata/datadotparl/' + member_id + '.xml', 'Members')
+  member_fullbio_xml = parse('../../rawdata/datadotparl/' + member_id + '.xml', 'Members')
   parl_members[member_id] = member_fullbio_xml.find('Member')
 
 ##########
 # At this point parl_members contains a complete set of data.
 ##########
 
 # Import people.xml
-people_json = json.load(open('../members/people.json'))
+people_json = json.load(open('../../members/people.json'))
 people = []
 for person in people_json['persons']:
   for i in person.get('identifiers', []):
@@ -206,5 +206,5 @@ for person in people:
 print 'Done for ' + source_file + '.xml.'
 
 # Write out the updated ministers file
-popolo.output('../members/ministers-2010.json')
+popolo.output('../../members/ministers-2010.json')
 print 'ministers-2010.json updated!'

scripts/weeklyupdate

@@ -6,9 +6,9 @@
 source ~/parlparse/scripts/consts
 
 # run data.parliament scrapers
-cd ~/parlparse/scripts
-./crawl-datadotparl-members
-./update-members-from-datadotparliament
+cd ~/parlparse/scripts/datadotparl
+./crawl-members
+./update-members
 
 # Run member scrapers
 # TODO: add more, write to temp file, check errors