Merge pull request #429 from HSLdevcom/fix-otp-data-tools-image-and-a…

…dd-kela-rule

DT-6576 - Fix otp data tools image and add kela rule

README.md

@@ -132,7 +132,6 @@ Keeps 10 latest versions of the data in storage and removes the rest.
 
 ### otp-data-tools
 
-Contains tools for gtfs manipulation, such as One Bus Away gtfs filter. OBA filter tool version 1.3.9 is embedded into this repository.
-It should be replaced with a newer version if such one appears e.g. into https://mvnrepository.com/.
-
-These tools are packaged inside docker container and are used during the data build process.
+Contains tools, such as the OneBusAway gtfs filter, for gtfs manipulation.
+It uses the [opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli](https://registry.hub.docker.com/r/opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli) as the base image.
+These tools are packaged inside a docker container and are used during the data build process.

kela/gtfs-rules/remove-matching-route.rule

@@ -0,0 +1,2 @@
+# remove routes matching the route_short_name regex
+{"op":"remove", "match":{"file":"routes.txt", "route_short_name":"m/^OB(2|3|36|37|38|4|41|44|6|9|91)$/"}}

otp-data-tools/Dockerfile

@@ -1,9 +1,12 @@
-FROM eclipse-temurin:21-jre
+FROM opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli:4.4.0
 LABEL maintainer="Digitransit (digitransit.fi)" \
-      version="1.0" \
+      version="1.1" \
       repo="https://github.com/HSLdevcom/OpenTripPlanner-data-container"
 
-ADD onebusaway-gtfs-transformer-cli.jar /
-ADD build-otp-data-tools.sh ${WORK}
+RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv
+RUN rm -rf /var/lib/apt/lists/*
+RUN python3 -m venv python-venv && python-venv/bin/pip install future grequests numpy
+RUN git clone -b v3 --single-branch https://github.com/HSLdevcom/OTPQA.git
+
+RUN mkdir /data
 VOLUME /data
-RUN ./build-otp-data-tools.sh && rm build-otp-data-tools.sh

otp-data-tools/README.md

@@ -1,6 +1,9 @@
 # OpenTripPlanner-data-tools
-[![Build](https://github.com/hsldevcom/OpenTripPlanner-data-container/workflows/Process%20master%20push%20or%20pr/badge.svg?branch=master)](https://github.com/HSLdevcom/OpenTripPlanner-data-container/actions)
 
 ## About:
-1. This docker container includes all the needed tools for preparing gtfs data
-   ready for consumption.
+- This docker container includes all the needed tools for preparing gtfs data ready for consumption.
+- It uses opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli as the base image.
+- Useful links:
+  - https://github.com/OneBusAway/onebusaway-gtfs-modules
+      - Documentation can be found in the `docs` folder
+  - https://registry.hub.docker.com/r/opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli

task/OBAFilter.js

@@ -12,7 +12,7 @@ const { postSlackMessage, parseId } = require('../util')
 function OBAFilter (src, dst, rule) {
   process.stdout.write(`filtering ${src} with ${rule}...\n`)
 
-  const cmd = `docker pull ${dataToolImage}; docker run -v ${dataDir}:/data --rm ${dataToolImage} java -Xmx6g -jar onebusaway-gtfs-transformer-cli.jar --transform=/data/${rule} /data/${src} /data/${dst}`
+  const cmd = `docker run -v ${dataDir}:/data --rm ${dataToolImage} --transform=/data/${rule} /data/${src} /data/${dst}`
 
   try {
     execSync(cmd, { stdio: [0, 1, 2] })