Skip to content

Commit

Permalink
Merge branch 'main' of github.com:hotosm/osm-fieldwork
Browse files Browse the repository at this point in the history
Sync with upstream.
  • Loading branch information
robsavoye committed Sep 8, 2023
2 parents bebd27c + d2f9f9b commit c5d9ceb
Show file tree
Hide file tree
Showing 7 changed files with 76 additions and 49 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
## 0.3.6rc1 (2023-09-07)

### Fix

- json2osm via cmdline + programatically

## 0.3.6rc0 (2023-09-01)

### Feat
Expand Down
2 changes: 2 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ RUN set -ex \
"build-essential" \
"gcc" \
"libpcre3-dev" \
"libpq-dev" \
"libspatialindex-dev" \
"libproj-dev" \
"libgeos-dev" \
Expand Down Expand Up @@ -102,6 +103,7 @@ RUN set -ex \
"curl" \
"gosu" \
"libpcre3" \
"postgresql-client" \
"libglib2.0-0" \
"libspatialindex-c6" \
"libproj25" \
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

PACKAGE := org.osm_fieldwork.py
NAME := osm-fieldwork
VERSION := 0.3.6rc0
VERSION := 0.3.6rc1

# All python source files
FILES := $(wildcard ./osm_fieldwork/*.py)
Expand Down
13 changes: 8 additions & 5 deletions osm_fieldwork/OdkCentral.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@
def downloadThread(
project_id: int,
xforms: list,
odk_credentials: dict
odk_credentials: dict,
filters: dict = None
):
"""
Download a list of submissions from ODK Central
Expand All @@ -75,8 +76,8 @@ def downloadThread(
odk_credentials["user"],
odk_credentials["passwd"]
)
submissions = form.getSubmissions(project_id, task, 0, False, True)
subs = form.listSubmissions(project_id, task)
# submissions = form.getSubmissions(project_id, task, 0, False, True)
subs = form.listSubmissions(project_id, task, filters)
if type(subs) == dict:
log.error(f"{subs['message']}, {subs['code']} ")
continue
Expand Down Expand Up @@ -419,6 +420,7 @@ def listForms(self,
def getAllSubmissions(self,
project_id: int,
xforms: list = None,
filters: dict = None
):
"""
Fetch a list of submissions in a project on an ODK Central server.
Expand Down Expand Up @@ -463,7 +465,7 @@ def getAllSubmissions(self,
for current in cycle:
if previous == current:
continue
result = executor.submit(downloadThread, project_id, xforms[previous:current], odk_credentials)
result = executor.submit(downloadThread, project_id, xforms[previous:current], odk_credentials, filters)
previous = current
futures.append(result)
for future in concurrent.futures.as_completed(futures):
Expand Down Expand Up @@ -671,6 +673,7 @@ def listSubmissionBasicInfo(self,
def listSubmissions(self,
projectId: int,
xform: str,
filters: dict = None
):
"""
Fetch a list of submission instances for a given form.
Expand All @@ -683,7 +686,7 @@ def listSubmissions(self,
(list): The list of Submissions
"""
url = f"{self.base}projects/{projectId}/forms/{xform}.svc/Submissions"
result = self.session.get(url, auth=self.auth, verify=self.verify)
result = self.session.get(url, auth=self.auth, params=filters, verify=self.verify)
if result.ok:
self.submissions = result.json()
return self.submissions['value']
Expand Down
2 changes: 1 addition & 1 deletion osm_fieldwork/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.3.6rc0"
__version__ = "0.3.6rc1"
98 changes: 57 additions & 41 deletions osm_fieldwork/json2osm.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def createOSM(self,
Returns:
(OsmFile): An instance of the OSM XML output file
"""
log.debug("Creating OSM XML file: %s" % filespec)
log.debug(f"Creating OSM XML file: {filespec}")
self.osm = OsmFile(filespec)
return self.osm

Expand Down Expand Up @@ -191,6 +191,7 @@ def parse(self,
Returns:
(list): A list of all the features in the input file
"""
log.debug(f"Parsing JSON file {filespec}")
all_tags = list()
if not data:
file = open(filespec, "r")
Expand Down Expand Up @@ -368,54 +369,42 @@ def createEntry(self,

return feature

def main():
"""This main function lets this class be run standalone by a bash script"""
parser = argparse.ArgumentParser(
description="convert JSON from ODK Central to OSM XML"
)
parser.add_argument("-v", "--verbose", action="store_true", help="verbose output")
parser.add_argument("-y", "--yaml", help="Alternate YAML file")
parser.add_argument("-x", "--xlsfile", help="Source XLSFile")
parser.add_argument("-i", "--infile", required=True,
help="The input file downloaded from ODK Central"
)
args = parser.parse_args()

# if verbose, dump to the terminal.
if args.verbose is not None:
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(threadName)10s - %(name)s - %(levelname)s - %(message)s"
)
ch.setFormatter(formatter)
log.addHandler(ch)
def json2osm(input_file, yaml_file=None):
"""
Process the JSON file from ODK Central or the GeoJSON file to OSM XML format.
Args:
input_file (str): The path to the input JSON or GeoJSON file.
yaml_file (str): The path to the YAML config file (optional).
if args.yaml:
jsonin = JsonDump(args.yaml)
Returns:
osmoutfile (str): Path to the converted OSM XML file.
"""
log.info(f"Converting JSON file to OSM: {input_file}")
if yaml_file:
jsonin = JsonDump(yaml_file)
else:
jsonin = JsonDump()

# jsonin.parseXLS(args.xlsfile)

# Modify the input file name for the 2 output files, which will get written
# to the current directory.
infile = Path(args.infile)
base = os.path.splitext(infile.name)[0]

base = Path(input_file).stem
osmoutfile = f"{base}-out.osm"
jsonin.createOSM(osmoutfile)

jsonoutfile = f"{base}-out.geojson"
jsonin.createGeoJson(jsonoutfile)

log.debug("Parsing JSON file %r" % args.infile)
data = jsonin.parse(infile.as_posix())
data = jsonin.parse(input_file)
# This OSM XML file only has OSM appropriate tags and values

for entry in data:
feature = jsonin.createEntry(entry)

# Sometimes bad entries, usually from debugging XForm design, sneak in
if len(feature) == 0:
continue

if len(feature) > 0:
if "lat" not in feature["attrs"]:
if 'geometry' in feature['tags']:
Expand All @@ -426,16 +415,43 @@ def main():
coords = feature['tags']['coordinates']
feature['attrs'] = {'lat': coords[1], 'lon': coords[0]}
else:
log.warning("Bad record! %r" % feature)
continue
log.warning(f"Bad record! {feature}")
continue # Skip bad records

log.debug("Writing final OSM XML file...")
jsonin.writeOSM(feature)
# This GeoJson file has all the data values
jsonin.writeGeoJson(feature)

# jsonin.finishOSM()
jsonin.finishGeoJson()
log.info("Wrote OSM XML file: %r" % osmoutfile)
log.info("Wrote GeoJson file: %r" % jsonoutfile)
jsonin.finishOSM()
log.info(f"Wrote OSM XML file: {osmoutfile}")

return osmoutfile


def main():
"""Run conversion directly from the terminal."""
parser = argparse.ArgumentParser(
description="convert JSON from ODK Central to OSM XML"
)
parser.add_argument("-v", "--verbose", action="store_true", help="verbose output")
parser.add_argument("-y", "--yaml", help="Alternate YAML file")
parser.add_argument("-x", "--xlsfile", help="Source XLSFile")
parser.add_argument("-i", "--infile", required=True,
help="The input file downloaded from ODK Central"
)
args = parser.parse_args()

# if verbose, dump to the terminal.
if args.verbose is not None:
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(threadName)10s - %(name)s - %(levelname)s - %(message)s"
)
ch.setFormatter(formatter)
log.addHandler(ch)

json2osm(args.infile, args.yaml)

if __name__ == "__main__":
"""This is just a hook so this file can be run standlone during development."""
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ pythonpath = "osm_fieldwork"

[tool.commitizen]
name = "cz_conventional_commits"
version = "0.3.6rc0"
version = "0.3.6rc1"
version_files = [
"pyproject.toml:version",
"osm_fieldwork/__version__.py",
Expand Down

0 comments on commit c5d9ceb

Please sign in to comment.