diff --git a/.github/workflows/scripts/build_and_push_dev.sh b/.github/workflows/scripts/build_and_push_dev.sh index 6f14411d..d00ccb60 100755 --- a/.github/workflows/scripts/build_and_push_dev.sh +++ b/.github/workflows/scripts/build_and_push_dev.sh @@ -16,7 +16,7 @@ function imagedeploy { # Build image echo "Building $1" - docker build --tag=$DOCKER_IMAGE_TAG_LONG . + docker build --network=host --tag=$DOCKER_IMAGE_TAG_LONG . echo "Pushing $DOCKER_TAG image" docker push $DOCKER_IMAGE_TAG_LONG diff --git a/.gitignore b/.gitignore index fb48bb6a..6b5f1e38 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ router-finland.zip router-hsl.zip router-waltti.zip router-waltti-alt.zip + +.tmp \ No newline at end of file diff --git a/config.js b/config.js index a5b5a345..6d988aba 100644 --- a/config.js +++ b/config.js @@ -1,12 +1,4 @@ -/* - * id = feedid (String) - * url = feed url (String) - * fit = mapfit shapes (true/falsy) - * rules = OBA Filter rules to apply (array of strings or undefined) - * replacements = replace or remove file from gtfs package (format: {'file_to_replace': 'file_to_replace_with' or null}) - * request options = optional special options for request - */ -const mapSrc = (id, url, fit, rules, replacements, request) => ({ id, url, fit, rules, replacements, request }) +const assert = require('assert') // OBA filter erases files which it does not recognize from GTFS packages // this array specifies the file names which should be preserved @@ -22,119 +14,10 @@ if (process.env.MH_BASIC_AUTH) { mhAddress = 'http://digitransit-proxy:8080/out/minfoapi.matkahuolto.fi/gtfs/kokomaa-fi/gtfs.zip' } -const routers = { - hsl: { - id: 'hsl', - src: [ - mapSrc('HSL', 'https://infopalvelut.storage.hsldev.com/gtfs/hsl.zip', false, undefined, { 'trips.txt': 'trips2.txt' }), - // mapSrc('HSLlautta', 'https://koontikartta.navici.com/tiedostot/gtfs_lautat_digitransit.zip') - // src('Sipoo', 'https://koontikartta.navici.com/tiedostot/rae/sipoon_kunta_sibbo_kommun.zip') - ], - osm: ['hsl'], - dem: 'hsl' - }, +assert(process.env.ROUTER_NAME !== undefined, 'ROUTER_NAME must be defined') - finland: { - id: 'finland', - src: [ - mapSrc('HSL', 'https://infopalvelut.storage.hsldev.com/gtfs/hsl.zip', false, ['finland/gtfs-rules/hsl-no-trains.rule'], { 'trips.txt': 'trips2.txt' }), - mapSrc('MATKA', 'https://mobility.mobility-database.fintraffic.fi/static/digitransit_new.zip', true), - mapSrc('tampere', 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_deprecated.zip'), - mapSrc('LINKKI', 'https://tvv.fra1.digitaloceanspaces.com/209.zip', true), - mapSrc('OULU', 'https://tvv.fra1.digitaloceanspaces.com/229.zip'), - mapSrc('digitraffic', 'https://rata.digitraffic.fi/api/v1/trains/gtfs-passenger-stops.zip', false, undefined, undefined, { headers: { 'Accept-Encoding': 'gzip', 'Digitraffic-User': 'Digitransit/OTP-dataloading' } }), - mapSrc('Rauma', 'http://digitransit-proxy:8080/out/raumaadmin.mattersoft.fi/feeds/233.zip'), - mapSrc('Hameenlinna', 'https://tvv.fra1.digitaloceanspaces.com/203.zip', true), - mapSrc('Kotka', 'https://tvv.fra1.digitaloceanspaces.com/217.zip', true), - mapSrc('Kouvola', 'https://tvv.fra1.digitaloceanspaces.com/219.zip', true), - mapSrc('Lappeenranta', 'https://tvv.fra1.digitaloceanspaces.com/225.zip', true), - mapSrc('Mikkeli', 'https://tvv.fra1.digitaloceanspaces.com/227.zip', true), - mapSrc('Vaasa', 'https://tvv.fra1.digitaloceanspaces.com/249.zip', true), - mapSrc('Joensuu', 'https://tvv.fra1.digitaloceanspaces.com/207.zip', true), - mapSrc('FOLI', 'https://data.foli.fi/gtfs/gtfs.zip'), - mapSrc('Lahti', 'https://tvv.fra1.digitaloceanspaces.com/223.zip', true), - mapSrc('Kuopio', 'https://karttapalvelu.kuopio.fi/google_transit/google_transit.zip'), - mapSrc('Rovaniemi', 'https://tvv.fra1.digitaloceanspaces.com/237.zip', true), - mapSrc('Kajaani', 'https://tvv.fra1.digitaloceanspaces.com/211.zip', true), - mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), - mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true), - mapSrc('Viro', 'https://peatus.ee/gtfs/gtfs.zip'), - mapSrc('Raasepori', 'https://tvv.fra1.digitaloceanspaces.com/232.zip', true), - mapSrc('VARELY', 'http://digitransit-proxy:8080/out/varelyadmin.mattersoft.fi/feeds/102.zip', false), - mapSrc('Harma', 'https://harmanliikenne.bussikaista.fi/sites/harma/files/gtfs/export/latest.zip', true), - mapSrc('PohjolanMatka', 'https://minfoapi.matkahuolto.fi/gtfs/458/gtfs.zip', true), - mapSrc('Korsisaari','https://minfoapi.matkahuolto.fi/gtfs/036/gtfs.zip', true) - ], - osm: ['finland', 'estonia'] - }, - - waltti: { - id: 'waltti', - src: [ - mapSrc('Hameenlinna', 'https://tvv.fra1.digitaloceanspaces.com/203.zip', true), - mapSrc('Kotka', 'https://tvv.fra1.digitaloceanspaces.com/217.zip', true), - mapSrc('Kouvola', 'https://tvv.fra1.digitaloceanspaces.com/219.zip', true), - mapSrc('Lappeenranta', 'https://tvv.fra1.digitaloceanspaces.com/225.zip', true), - mapSrc('Mikkeli', 'https://tvv.fra1.digitaloceanspaces.com/227.zip', true), - mapSrc('Vaasa', 'https://tvv.fra1.digitaloceanspaces.com/249.zip', true), - mapSrc('Joensuu', 'https://tvv.fra1.digitaloceanspaces.com/207.zip', true), - mapSrc('FOLI', 'https://data.foli.fi/gtfs/gtfs.zip'), - mapSrc('Lahti', 'https://tvv.fra1.digitaloceanspaces.com/223.zip', true, undefined, {'fare_attributes.txt': 'digitransit_fare_attributes.txt', 'fare_rules.txt': 'digitransit_fare_rules.txt'}), - mapSrc('Kuopio', 'https://karttapalvelu.kuopio.fi/google_transit/google_transit.zip'), - mapSrc('OULU', 'https://tvv.fra1.digitaloceanspaces.com/229.zip'), - mapSrc('LINKKI', 'https://tvv.fra1.digitaloceanspaces.com/209.zip', true, undefined, {'fare_attributes.txt': 'digitransit_fare_attributes.txt', 'fare_rules.txt': 'digitransit_fare_rules.txt'}), - mapSrc('tampere', 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_deprecated.zip'), - mapSrc('Rovaniemi', 'https://tvv.fra1.digitaloceanspaces.com/237.zip', true), - mapSrc('digitraffic', 'https://rata.digitraffic.fi/api/v1/trains/gtfs-passenger-stops.zip', false, undefined, undefined, { headers: { 'Accept-Encoding': 'gzip', 'Digitraffic-User': 'Digitransit/OTP-dataloading' } }), - mapSrc('tampereDRT', 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_kutsuliikenne.zip'), - mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true), - mapSrc('FUNI', 'https://foligtfs.blob.core.windows.net/routeplanner/gtfs-foli-ff.zip', true), - mapSrc('Raasepori', 'https://tvv.fra1.digitaloceanspaces.com/232.zip', true), - mapSrc('KotkaLautat', 'https://koontikartta.navici.com/tiedostot/gtfs_lautat.zip', true, ['waltti/gtfs-rules/only-kotka-ferries.rule']), - mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), - mapSrc('Kajaani', 'https://tvv.fra1.digitaloceanspaces.com/211.zip', true), - ], - osm: ['kajaani', 'oulu', 'pori', 'rovaniemi', 'southeastFinland', 'southwestFinland', 'vaasa'], - dem: 'waltti' - }, - - 'waltti-alt': { - id: 'waltti-alt', - src: [ - mapSrc('WalttiTest', 'http://digitransit-proxy:8080/out/lmjadmin.mattersoft.fi/feeds/229.zip', true), - ], - osm: ['oulu'] - }, - - varely: { - id: 'varely', - src: [ - mapSrc('VARELY', 'http://digitransit-proxy:8080/out/varelyadmin.mattersoft.fi/feeds/102.zip'), - mapSrc('FOLI', 'https://data.foli.fi/gtfs/gtfs.zip'), - mapSrc('Rauma', 'http://digitransit-proxy:8080/out/raumaadmin.mattersoft.fi/feeds/233.zip'), - mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), - mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true) - ], - osm: ['varely'] - }, - - kela: { - id: 'kela', - src: [ - mapSrc('kela', 'https://mobility.mobility-database.fintraffic.fi/static/Kela_suuret.zip'), - mapSrc('kela_varely', 'https://mobility.mobility-database.fintraffic.fi/static/Kela_varely.zip'), - mapSrc('kela_waltti', 'https://mobility.mobility-database.fintraffic.fi/static/kela_waltti.zip'), - mapSrc('matkahuolto', mhAddress, false, ['kela/gtfs-rules/matkahuolto.rule'], { 'transfers.txt': null }), - ], - osm: ['finland'] - } -} - -if (!process.env.ROUTER_NAME || !routers[process.env.ROUTER_NAME]) { - process.stdout.write('Invalid ROUTER_NAME variable \n') - process.exit(1) -} -const router = routers[process.env.ROUTER_NAME] +// Require router config from router directory +const router = require(`./${process.env.ROUTER_NAME}/config`) // EXTRA_SRC format should be {"FOLI": {"url": "https://data.foli.fi/gtfs/gtfs.zip", "fit": false, "rules": ["waltti/gtfs-rules/waltti.rule"]}} // but you can only define, for example, new url and the other key value pairs will remain the same as they are defined in this file. diff --git a/finland/config.js b/finland/config.js new file mode 100644 index 00000000..9d1e71ff --- /dev/null +++ b/finland/config.js @@ -0,0 +1,35 @@ +const { mapSrc } = require('../util') + +module.exports = { + id: 'finland', + src: [ + mapSrc('HSL', 'https://infopalvelut.storage.hsldev.com/gtfs/hsl.zip', false, ['finland/gtfs-rules/hsl-no-trains.rule'], { 'trips.txt': 'trips2.txt' }), + mapSrc('MATKA', 'https://mobility.mobility-database.fintraffic.fi/static/digitransit_new.zip', true), + mapSrc('tampere', 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_deprecated.zip'), + mapSrc('LINKKI', 'https://tvv.fra1.digitaloceanspaces.com/209.zip', true), + mapSrc('OULU', 'https://tvv.fra1.digitaloceanspaces.com/229.zip'), + mapSrc('digitraffic', 'https://rata.digitraffic.fi/api/v1/trains/gtfs-passenger-stops.zip', false, undefined, undefined, { headers: { 'Accept-Encoding': 'gzip', 'Digitraffic-User': 'Digitransit/OTP-dataloading' } }), + mapSrc('Rauma', 'http://digitransit-proxy:8080/out/raumaadmin.mattersoft.fi/feeds/233.zip'), + mapSrc('Hameenlinna', 'https://tvv.fra1.digitaloceanspaces.com/203.zip', true), + mapSrc('Kotka', 'https://tvv.fra1.digitaloceanspaces.com/217.zip', true), + mapSrc('Kouvola', 'https://tvv.fra1.digitaloceanspaces.com/219.zip', true), + mapSrc('Lappeenranta', 'https://tvv.fra1.digitaloceanspaces.com/225.zip', true), + mapSrc('Mikkeli', 'https://tvv.fra1.digitaloceanspaces.com/227.zip', true), + mapSrc('Vaasa', 'https://tvv.fra1.digitaloceanspaces.com/249.zip', true), + mapSrc('Joensuu', 'https://tvv.fra1.digitaloceanspaces.com/207.zip', true), + mapSrc('FOLI', 'https://data.foli.fi/gtfs/gtfs.zip'), + mapSrc('Lahti', 'https://tvv.fra1.digitaloceanspaces.com/223.zip', true), + mapSrc('Kuopio', 'https://karttapalvelu.kuopio.fi/google_transit/google_transit.zip'), + mapSrc('Rovaniemi', 'https://tvv.fra1.digitaloceanspaces.com/237.zip', true), + mapSrc('Kajaani', 'https://tvv.fra1.digitaloceanspaces.com/211.zip', true), + mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), + mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true), + mapSrc('Viro', 'https://peatus.ee/gtfs/gtfs.zip'), + mapSrc('Raasepori', 'https://tvv.fra1.digitaloceanspaces.com/232.zip', true), + mapSrc('VARELY', 'http://digitransit-proxy:8080/out/varelyadmin.mattersoft.fi/feeds/102.zip', false), + mapSrc('Harma', 'https://harmanliikenne.bussikaista.fi/sites/harma/files/gtfs/export/latest.zip', true), + mapSrc('PohjolanMatka', 'https://minfoapi.matkahuolto.fi/gtfs/458/gtfs.zip', true), + mapSrc('Korsisaari', 'https://minfoapi.matkahuolto.fi/gtfs/036/gtfs.zip', true) + ], + osm: ['finland', 'estonia'] +} \ No newline at end of file diff --git a/gulpfile.js b/gulpfile.js index 92d14fcb..6faf04e3 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -17,7 +17,6 @@ const { buildOTPGraphTask } = require('./task/BuildOTPGraph') const { renameGTFSFile } = require('./task/GTFSRename') const { replaceGTFSFilesTask } = require('./task/GTFSReplace') const { extractFromZip, addToZip } = require('./task/ZipTask') -const patchDeploymentFiles = require('./task/PatchDeploymentFiles') const storageCleanup = require('./task/StorageCleanup') const seedSourceDir = `${config.dataDir}/router-${config.router.id}` // e.g. data/router-hsl @@ -182,6 +181,4 @@ gulp.task('router:store', () => gulp.src(`${config.dataDir}/build/${config.router.id}/**/*`, { buffer: false }).pipe(gulp.dest(`${config.storageDir}/${global.storageDirName}/`)) ) -gulp.task('deploy:prepare', () => patchDeploymentFiles()) - gulp.task('storage:cleanup', () => storageCleanup(config.storageDir, config.router.id, process.env.SEED_TAG)) diff --git a/hsl/config.js b/hsl/config.js new file mode 100644 index 00000000..da4bc812 --- /dev/null +++ b/hsl/config.js @@ -0,0 +1,12 @@ +const { mapSrc } = require('../util') + +module.exports = { + id: 'hsl', + src: [ + mapSrc('HSL', 'https://infopalvelut.storage.hsldev.com/gtfs/hsl.zip', false, undefined, { 'trips.txt': 'trips2.txt' }), + // mapSrc('HSLlautta', 'https://koontikartta.navici.com/tiedostot/gtfs_lautat_digitransit.zip') + // src('Sipoo', 'https://koontikartta.navici.com/tiedostot/rae/sipoon_kunta_sibbo_kommun.zip') + ], + osm: ['hsl'], + dem: 'hsl' +} \ No newline at end of file diff --git a/index.js b/index.js index 42c20f16..d90c33c6 100644 --- a/index.js +++ b/index.js @@ -1,13 +1,12 @@ const { postSlackMessage } = require('./util') +const { update } = require('./task/Update') postSlackMessage('Starting data build').then(response => { if (response.ok) { global.messageTimeStamp = response.ts } - const { update } = require('./task/Update') - update() }).catch((err) => { console.log(err) - const { update } = require('./task/Update') +}).finally(() => { update() }) diff --git a/kela/config.js b/kela/config.js new file mode 100644 index 00000000..cf2c2fe7 --- /dev/null +++ b/kela/config.js @@ -0,0 +1,12 @@ +const { mapSrc } = require('../util') + +module.exports = { + id: 'kela', + src: [ + mapSrc('kela', 'https://mobility.mobility-database.fintraffic.fi/static/Kela_suuret.zip'), + mapSrc('kela_varely', 'https://mobility.mobility-database.fintraffic.fi/static/Kela_varely.zip'), + mapSrc('kela_waltti', 'https://mobility.mobility-database.fintraffic.fi/static/kela_waltti.zip'), + mapSrc('matkahuolto', mhAddress, false, ['kela/gtfs-rules/matkahuolto.rule'], { 'transfers.txt': null }), + ], + osm: ['finland'] +} \ No newline at end of file diff --git a/opentripplanner/Dockerfile b/opentripplanner/Dockerfile index 8e257baf..6c9696ac 100644 --- a/opentripplanner/Dockerfile +++ b/opentripplanner/Dockerfile @@ -1,5 +1,11 @@ -FROM hsldevcom/opentripplanner: +ARG OTP_TAG="v2" +FROM hsldevcom/opentripplanner:${OTP_TAG} -WORKDIR /var/otp/ +ARG OTP_GRAPH_DIR +ENV OTP_GRAPH_DIR=$OTP_GRAPH_DIR -ENTRYPOINT java $JAVA_OPTS -cp @/app/jib-classpath-file @/app/jib-main-class-file /var/otp/ --load --serve +ADD entrypoint.sh /var/entrypoint.sh + +WORKDIR /var/otp/${OTP_GRAPH_DIR} + +ENTRYPOINT ["/var/entrypoint.sh"] \ No newline at end of file diff --git a/opentripplanner/deploy-otp.sh b/opentripplanner/deploy-otp.sh index e4ad7181..c762b15b 100755 --- a/opentripplanner/deploy-otp.sh +++ b/opentripplanner/deploy-otp.sh @@ -3,31 +3,45 @@ set -e +cd "$(dirname "$0")" + ROUTER_NAME=${ROUTER_NAME:-hsl} DATE=$1 ORG=${ORG:-hsldevcom} -DOCKER_TAG=${OTP_TAG:-v2}-$ROUTER_NAME +OTP_TAG=${OTP_TAG:-v2} +DOCKER_TAG=$OTP_TAG-$ROUTER_NAME PROJECT=opentripplanner DOCKER_IMAGE=$ORG/$PROJECT DOCKER_DATE_IMAGE=$DOCKER_IMAGE:$DOCKER_TAG-$DATE DOCKER_IMAGE_TAGGED=$DOCKER_IMAGE:$DOCKER_TAG -docker login -u $DOCKER_USER -p $DOCKER_AUTH +if [[ -z "${OTP_GRAPH_DIR}" ]]; then + echo "*** OTP_GRAPH_DIR is not defined." + exit 1 +fi + +if [[ -z "${DOCKER_USER}" ]]; then + echo "*** DOCKER_USER is not defined. Unable to log in to the registry." +else + docker login -u $DOCKER_USER -p $DOCKER_AUTH +fi # remove old version (may be necessary in local use) docker rmi --force $DOCKER_IMAGE_TAGGED &> /dev/null docker rmi --force $DOCKER_DATE_IMAGE &> /dev/null echo "Building router's opentripplanner image..." -# Local file context is not needed -# https://docs.docker.com/reference/cli/docker/image/build/#build-with-- -cd data/build/$ROUTER_NAME -docker build -t $DOCKER_IMAGE_TAGGED - < ../../../opentripplanner/Dockerfile -echo "*** Pushing $DOCKER_IMAGE_TAGGED" -docker push $DOCKER_IMAGE_TAGGED +docker build --progress=plain --network=host --build-arg OTP_TAG=$OTP_TAG --build-arg OTP_GRAPH_DIR=$OTP_GRAPH_DIR -t $DOCKER_IMAGE_TAGGED . docker tag $DOCKER_IMAGE_TAGGED $DOCKER_DATE_IMAGE -echo "*** Pushing $DOCKER_DATE_IMAGE" -docker push $DOCKER_DATE_IMAGE + +if [[ -z "${DOCKER_USER}" ]]; then + echo "*** Not signed into the registry. Image not pushed." +else + echo "*** Pushing $DOCKER_IMAGE_TAGGED" + docker push $DOCKER_IMAGE_TAGGED + echo "*** Pushing $DOCKER_DATE_IMAGE" + docker push $DOCKER_DATE_IMAGE +fi diff --git a/opentripplanner/entrypoint.sh b/opentripplanner/entrypoint.sh new file mode 100755 index 00000000..c01e241a --- /dev/null +++ b/opentripplanner/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +echo "OTP_GRAPH_DIR: ${OTP_GRAPH_DIR}" + +java $JAVA_OPTS -cp @/app/jib-classpath-file @/app/jib-main-class-file /var/otp/${OTP_GRAPH_DIR} --load --serve diff --git a/otp-data-server/Dockerfile b/otp-data-server/Dockerfile index a9c4422b..8613e9c4 100644 --- a/otp-data-server/Dockerfile +++ b/otp-data-server/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:latest -MAINTAINER Digitransit version: 2 +LABEL authors="Digitransit version: 2" RUN apk add --update --no-cache \ lighttpd \ @@ -7,6 +7,8 @@ RUN apk add --update --no-cache \ busybox \ && rm -rf /var/cache/apk/* +ARG OTP_GRAPH_DIR +ENV OTP_GRAPH_DIR=${OTP_GRAPH_DIR} ADD lighttpd.conf /etc/lighttpd/lighttpd.conf RUN printf '#!/bin/sh\ngrep -v "#" /etc/lighttpd/lighttpd.conf |grep port > /dev/null\nif [ "$?" = "1" ]\nthen\n echo "server.port = $PORT" >> /etc/lighttpd/lighttpd.conf\nfi\n/usr/sbin/lighttpd -D -f /etc/lighttpd/lighttpd.conf 2>&1' > /usr/sbin/startlighttpd && chmod 755 /usr/sbin/startlighttpd diff --git a/otp-data-server/deploy.sh b/otp-data-server/deploy.sh index 5c286c65..e67b8693 100755 --- a/otp-data-server/deploy.sh +++ b/otp-data-server/deploy.sh @@ -5,6 +5,8 @@ #DOCKER_AUTH set -e +cd "$(dirname "$0")" + ROUTER_NAME=${ROUTER_NAME:-hsl} DATE=$1 @@ -13,14 +15,29 @@ DOCKER_TAG=${DOCKER_TAG:-v3} CONTAINER=opentripplanner-data-server DOCKER_IMAGE=$ORG/$CONTAINER -docker login -u $DOCKER_USER -p $DOCKER_AUTH +if [[ -z "${OTP_GRAPH_DIR}" ]]; then + echo "*** OTP_GRAPH_DIR is not defined." + exit 1 +fi + +if [[ -z "${DOCKER_USER}" ]]; then + echo "*** DOCKER_USER is not defined. Unable to log in to the registry." +else + docker login -u $DOCKER_USER -p $DOCKER_AUTH +fi DOCKER_DATE_IMAGE=$DOCKER_IMAGE:$DOCKER_TAG-$ROUTER_NAME-$DATE DOCKER_IMAGE_TAGGED=$DOCKER_IMAGE:$DOCKER_TAG-$ROUTER_NAME -cd otp-data-server -docker build -t $DOCKER_DATE_IMAGE . -echo "*** Pushing $DOCKER_DATE_IMAGE" -docker push $DOCKER_DATE_IMAGE + +docker build --progress=plain --network=host --build-arg OTP_GRAPH_DIR=$OTP_GRAPH_DIR -t $DOCKER_DATE_IMAGE . + docker tag $DOCKER_DATE_IMAGE $DOCKER_IMAGE_TAGGED -echo "*** Pushing $DOCKER_IMAGE_TAGGED" -docker push $DOCKER_IMAGE_TAGGED + +if [[ -z "${DOCKER_USER}" ]]; then + echo "*** Not signed into the registry. Image not pushed." +else + echo "*** Pushing $DOCKER_DATE_IMAGE" + docker push $DOCKER_DATE_IMAGE + echo "*** Pushing $DOCKER_IMAGE_TAGGED" + docker push $DOCKER_IMAGE_TAGGED +fi diff --git a/otp-data-server/lighttpd.conf b/otp-data-server/lighttpd.conf index efed0e21..ce5626be 100644 --- a/otp-data-server/lighttpd.conf +++ b/otp-data-server/lighttpd.conf @@ -47,7 +47,7 @@ include "mime-types.conf" #server.username = "lighttpd" #server.groupname = "lighttpd" -server.document-root = var.basedir + "/htdocs/" +server.document-root = var.basedir + "/htdocs/" + env.OTP_GRAPH_DIR server.pid-file = "/run/lighttpd.pid" server.errorlog = "/dev/stderr" diff --git a/task/OTPTest.js b/task/OTPTest.js index b02425a5..36125339 100644 --- a/task/OTPTest.js +++ b/task/OTPTest.js @@ -77,6 +77,10 @@ module.exports = { testOTPFile: () => { return through.obj(function (file, encoding, callback) { const otpFile = file.history[file.history.length - 1] + if (process.env.SKIP_OTP_TESTS) { + process.stdout.write('OTP test skipped because the SKIP_OTP_TESTS environment variable is set\n') + return callback(null, file) + } testWithOTP(otpFile, true).then((success) => { if (success) { callback(null, file) diff --git a/task/PatchDeploymentFiles.js b/task/PatchDeploymentFiles.js deleted file mode 100644 index 4d2b3087..00000000 --- a/task/PatchDeploymentFiles.js +++ /dev/null @@ -1,28 +0,0 @@ -const fs = require('fs') - -const OTP_TAG = process.env.OTP_TAG || 'v2' - -function patchDockerfile (otpTag, graphDir) { - const dockerFile = 'opentripplanner/Dockerfile' - const data = fs.readFileSync(dockerFile, { encoding: 'utf8' }) - const patchedData = data - .replace(//g, otpTag) - .replace(//g, graphDir) - fs.writeFileSync(dockerFile, patchedData, { encoding: 'utf8' }) -} - -function patchLighttpdConf (graphDir) { - const confFile = 'otp-data-server/lighttpd.conf' - const data = fs.readFileSync(confFile, { encoding: 'utf8' }) - const patchedData = data - .replace(//g, graphDir) - fs.writeFileSync(confFile, patchedData, { encoding: 'utf8' }) -} - -module.exports = function () { - return new Promise((resolve) => { - patchDockerfile(OTP_TAG, global.storageDirName) - patchLighttpdConf(global.storageDirName) - resolve() - }) -} diff --git a/task/Update.js b/task/Update.js index d3e4f3e3..fd461e8d 100644 --- a/task/Update.js +++ b/task/Update.js @@ -11,25 +11,31 @@ const fs = require('fs') const { postSlackMessage, updateSlackMessage } = require('../util') require('../gulpfile') const { router } = require('../config') +const assert = require('assert') const MAX_GTFS_FALLBACK = 2 // threshold for aborting data loading const start = promisify((task, cb) => gulp.series(task)(cb)) -async function update () { +async function update() { + // check environmental variables which needs to be defined + assert(process.env.DOCKER_TAG !== undefined, 'DOCKER_TAG must be defined') + if (!process.env.NOSEED) { process.stdout.write('Starting seeding\n') await start('seed') process.stdout.write('Seeded\n') } - // we track data rejections using this global variable - global.hasFailures = false - await start('dem:update') - if (global.hasFailures) { - postSlackMessage('DEM update failed, using previous version :boom:') + if (!process.env.NODEM) { + // we track data rejections using this global variable + global.hasFailures = false + await start('dem:update') + if (global.hasFailures) { + postSlackMessage('DEM update failed, using previous version :boom:') + } } - + // OSM update is more complicated. Download often fails, so there is a retry loop, // which breaks when a big enough file gets loaded global.blobSizeOk = false // ugly hack but gulp does not return any values from tasks @@ -55,7 +61,7 @@ async function update () { process.stdout.write('Build routing graph\n') await start('router:buildGraph') - if (process.env.SKIPPED_SITES === 'all') { + if (process.env.SKIPPED_SITES === 'all' || process.env.SKIP_OTP_TESTS) { process.stdout.write('Skipping all tests') } else { process.stdout.write('Test the newly built graph with OTPQA\n') @@ -93,12 +99,29 @@ async function update () { process.stdout.write('Uploading data to storage\n') await start('router:store') - process.stdout.write(`Patch new storage location ${global.storageDirName} to configs\n`) - await start('deploy:prepare') - process.stdout.write('Deploy docker images\n') - execFileSync('./otp-data-server/deploy.sh', [date], { stdio: [0, 1, 2] }) - execFileSync('./opentripplanner/deploy-otp.sh', [date], { stdio: [0, 1, 2] }) + process.stdout.write('Build and deploy Docker images\n') + execFileSync('./otp-data-server/deploy.sh', [date], { + stdio: [0, 1, 2], env: { + OTP_TAG: process.env.OTP_TAG, + OTP_GRAPH_DIR: global.storageDirName, + ROUTER_NAME: process.env.ROUTER_NAME, + ORG: process.env.ORG, + DOCKER_TAG: process.env.DOCKER_TAG, + DOCKER_USER: process.env.DOCKER_USER, + DOCKER_AUTH: process.env.DOCKER_AUTH + } + }) + execFileSync('./opentripplanner/deploy-otp.sh', [date], { + stdio: [0, 1, 2], env: { + OTP_GRAPH_DIR: global.storageDirName, + ROUTER_NAME: process.env.ROUTER_NAME, + ORG: process.env.ORG, + DOCKER_TAG: process.env.DOCKER_TAG, + DOCKER_USER: process.env.DOCKER_USER, + DOCKER_AUTH: process.env.DOCKER_AUTH + } + }) if (!process.env.NOCLEANUP) { process.stdout.write('Remove oldest data versions from storage\n') diff --git a/util.js b/util.js index 736fd78c..e7adefed 100644 --- a/util.js +++ b/util.js @@ -144,6 +144,16 @@ function dirNameToDate (dirName) { return date instanceof Date && !isNaN(date) ? date : null } +/* + * id = feedid (String) + * url = feed url (String) + * fit = mapfit shapes (true/falsy) + * rules = OBA Filter rules to apply (array of strings or undefined) + * replacements = replace or remove file from gtfs package (format: {'file_to_replace': 'file_to_replace_with' or null}) + * request options = optional special options for request + */ +const mapSrc = (id, url, fit, rules, replacements, request) => ({ id, url, fit, rules, replacements, request }) + module.exports = { zipDir: (zipFile, dir, cb) => { zipWithGlob(zipFile, [`${dir}/*`], undefined, cb) @@ -153,5 +163,6 @@ module.exports = { updateSlackMessage, otpMatching, parseId, - dirNameToDate + dirNameToDate, + mapSrc } diff --git a/varely/config.js b/varely/config.js new file mode 100644 index 00000000..ceb85412 --- /dev/null +++ b/varely/config.js @@ -0,0 +1,13 @@ +const { mapSrc } = require('../util') + +module.exports = { + id: 'varely', + src: [ + mapSrc('VARELY', 'http://digitransit-proxy:8080/out/varelyadmin.mattersoft.fi/feeds/102.zip'), + mapSrc('FOLI', 'https://data.foli.fi/gtfs/gtfs.zip'), + mapSrc('Rauma', 'http://digitransit-proxy:8080/out/raumaadmin.mattersoft.fi/feeds/233.zip'), + mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), + mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true) + ], + osm: ['varely'] +} \ No newline at end of file diff --git a/waltti-alt/config.js b/waltti-alt/config.js new file mode 100644 index 00000000..f7adb943 --- /dev/null +++ b/waltti-alt/config.js @@ -0,0 +1,9 @@ +const { mapSrc } = require('../util') + +module.exports = { + id: 'waltti-alt', + src: [ + mapSrc('WalttiTest', 'http://digitransit-proxy:8080/out/lmjadmin.mattersoft.fi/feeds/229.zip', true), + ], + osm: ['oulu'] +} \ No newline at end of file diff --git a/waltti/config.js b/waltti/config.js new file mode 100644 index 00000000..3e743159 --- /dev/null +++ b/waltti/config.js @@ -0,0 +1,31 @@ +const { mapSrc } = require('../util') + +module.exports = { + id: 'waltti', + src: [ + mapSrc('Hameenlinna', 'https://tvv.fra1.digitaloceanspaces.com/203.zip', true), + mapSrc('Kotka', 'https://tvv.fra1.digitaloceanspaces.com/217.zip', true), + mapSrc('Kouvola', 'https://tvv.fra1.digitaloceanspaces.com/219.zip', true), + mapSrc('Lappeenranta', 'https://tvv.fra1.digitaloceanspaces.com/225.zip', true), + mapSrc('Mikkeli', 'https://tvv.fra1.digitaloceanspaces.com/227.zip', true), + mapSrc('Vaasa', 'https://tvv.fra1.digitaloceanspaces.com/249.zip', true), + mapSrc('Joensuu', 'https://tvv.fra1.digitaloceanspaces.com/207.zip', true), + mapSrc('FOLI', 'https://data.foli.fi/gtfs/gtfs.zip'), + mapSrc('Lahti', 'https://tvv.fra1.digitaloceanspaces.com/223.zip', true, undefined, { 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 'fare_rules.txt': 'digitransit_fare_rules.txt' }), + mapSrc('Kuopio', 'https://karttapalvelu.kuopio.fi/google_transit/google_transit.zip'), + mapSrc('OULU', 'https://tvv.fra1.digitaloceanspaces.com/229.zip'), + mapSrc('LINKKI', 'https://tvv.fra1.digitaloceanspaces.com/209.zip', true, undefined, { 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 'fare_rules.txt': 'digitransit_fare_rules.txt' }), + mapSrc('tampere', 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_deprecated.zip'), + mapSrc('Rovaniemi', 'https://tvv.fra1.digitaloceanspaces.com/237.zip', true), + mapSrc('digitraffic', 'https://rata.digitraffic.fi/api/v1/trains/gtfs-passenger-stops.zip', false, undefined, undefined, { headers: { 'Accept-Encoding': 'gzip', 'Digitraffic-User': 'Digitransit/OTP-dataloading' } }), + mapSrc('tampereDRT', 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_kutsuliikenne.zip'), + mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true), + mapSrc('FUNI', 'https://foligtfs.blob.core.windows.net/routeplanner/gtfs-foli-ff.zip', true), + mapSrc('Raasepori', 'https://tvv.fra1.digitaloceanspaces.com/232.zip', true), + mapSrc('KotkaLautat', 'https://koontikartta.navici.com/tiedostot/gtfs_lautat.zip', true, ['waltti/gtfs-rules/only-kotka-ferries.rule']), + mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), + mapSrc('Kajaani', 'https://tvv.fra1.digitaloceanspaces.com/211.zip', true), + ], + osm: ['kajaani', 'oulu', 'pori', 'rovaniemi', 'southeastFinland', 'southwestFinland', 'vaasa'], + dem: 'waltti' +} \ No newline at end of file