From 563385b577a2675580e5f0c60fe8aab69eb51ebc Mon Sep 17 00:00:00 2001 From: vmarc Date: Fri, 19 Jul 2024 11:39:06 +0200 Subject: [PATCH] #368 migrate edge analysis --- .../analysis/AnalysisStartConfiguration.scala | 1 + .../support/RouteAnalysisCompareTool.scala | 69 +++++++++++++++ .../next/support/RouteAnalysisTool.scala | 2 +- .../next/support/compare/CompareEdges.scala | 85 +++++++++++++++++++ .../route/RouteDetailMainAnalyzer.scala | 3 +- .../route/analyzers/EdgeRouteAnalyzer.scala | 47 ++-------- .../analyzers/RouteAnalysisBuilder.scala | 4 +- .../analyzers/RouteCountryAnalyzerImpl.scala | 1 - .../route/analyzers/RouteMapAnalyzer.scala | 2 +- .../analyzers/RouteStreetsAnalyzer.scala | 2 +- .../route/analyzers/RouteTagAnalyzer.scala | 3 +- .../structure/RouteAnalysisElement.scala | 9 ++ .../analysis/route/structure/Structure.scala | 5 +- .../route/structure/StructurePath.scala | 15 ++++ .../structure/StructurePathElement.scala | 11 +++ 15 files changed, 209 insertions(+), 50 deletions(-) create mode 100644 server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisCompareTool.scala create mode 100644 server/src/main/scala/kpn/core/tools/next/support/compare/CompareEdges.scala diff --git a/server/src/main/scala/kpn/core/tools/analysis/AnalysisStartConfiguration.scala b/server/src/main/scala/kpn/core/tools/analysis/AnalysisStartConfiguration.scala index 4c9d036f4..88cdf4a82 100644 --- a/server/src/main/scala/kpn/core/tools/analysis/AnalysisStartConfiguration.scala +++ b/server/src/main/scala/kpn/core/tools/analysis/AnalysisStartConfiguration.scala @@ -71,6 +71,7 @@ class AnalysisStartConfiguration(options: AnalysisStartToolOptions) { private val database = Mongo.database(Mongo.client, options.databaseName) private val nextDatabase = Mongo.nextDatabase(Mongo.client, options.databaseName) + val oldDatabase = Mongo.oldDatabase(Mongo.client, options.databaseName) val networkRepository: NetworkRepository = new NetworkRepositoryImpl(database) val routeRepository: RouteRepository = new RouteRepositoryImpl(database) diff --git a/server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisCompareTool.scala b/server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisCompareTool.scala new file mode 100644 index 000000000..2eaaf2b85 --- /dev/null +++ b/server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisCompareTool.scala @@ -0,0 +1,69 @@ +package kpn.core.tools.next.support + +import kpn.api.custom.Relation +import kpn.core.doc.OldRouteDoc +import kpn.core.doc.RouteDetailDoc +import kpn.core.tools.analysis.AnalysisStartConfiguration +import kpn.core.tools.analysis.AnalysisStartToolOptions +import kpn.core.tools.next.domain.RouteRelation +import kpn.core.tools.next.support.compare.CompareEdges +import kpn.core.util.Log +import kpn.server.analyzer.engine.analysis.route.RouteDetailDocBuilder + +object RouteAnalysisCompareTool { + def main(args: Array[String]): Unit = { + val configuration = new AnalysisStartConfiguration(AnalysisStartToolOptions("kpn-next")) + new RouteAnalysisCompareTool(configuration).analyze() + } +} + +class RouteAnalysisCompareTool(config: AnalysisStartConfiguration) { + + private val log = Log(classOf[RouteAnalysisTool]) + + def analyze(): Unit = { + log.info("Collecting routeIds") + val routeIds = config.oldDatabase.oldRoutes.ids() + log.info(s"Comparing ${routeIds.size} routes") + routeIds.zipWithIndex.foreach { case (routeId, index) => + if (index % 50 == 0) { + log.info(s"${index + 1}/${routeIds.size}") + } + Log.context(s"${index + 1}/${routeIds.size} route=$routeId") { + try { + analyzeAndCompare(routeId) + } catch { + case e: Throwable => + log.error("Error processing route", e) + } + } + } + log.info(s"Done") + } + + private def analyzeAndCompare(routeId: Long): Unit = { + config.nextRepository.nextRouteRelation(routeId) match { + case None => log.error(s"route not found in route-relations") + case Some(nextRouteRelation) => + analyzeRoute(nextRouteRelation.relation, nextRouteRelation.structure) match { + case None => log.error(s"could not analyze route") + case Some(newRouteDetailDoc) => + config.oldDatabase.oldRoutes.findById(routeId) match { + case None => + case Some(oldRouteDoc) => + compare(oldRouteDoc, newRouteDetailDoc) + } + } + } + } + + private def analyzeRoute(relation: Relation, hierarchy: Option[RouteRelation]): Option[RouteDetailDoc] = { + config.routeDetailMainAnalyzer.analyze(relation, hierarchy).map { context => + new RouteDetailDocBuilder(context).build() + } + } + + private def compare(oldRouteDoc: OldRouteDoc, newRouteDoc: RouteDetailDoc): Unit = { + new CompareEdges(oldRouteDoc, newRouteDoc, log).compare() + } +} diff --git a/server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisTool.scala b/server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisTool.scala index db7172586..dcddd76ee 100644 --- a/server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisTool.scala +++ b/server/src/main/scala/kpn/core/tools/next/support/RouteAnalysisTool.scala @@ -88,7 +88,7 @@ class RouteAnalysisTool(config: AnalysisStartConfiguration) { } DependencySorter.sort(dependencies).foreach { relationId => - config.routeRepository.findRouteDetailById(relationId).match { + config.routeRepository.findRouteDetailById(relationId) match { case None => // TODO redesign - error message? case Some(routeDetailDoc) => config.routeMainAnalyzer.analyze(routeDetailDoc) match { diff --git a/server/src/main/scala/kpn/core/tools/next/support/compare/CompareEdges.scala b/server/src/main/scala/kpn/core/tools/next/support/compare/CompareEdges.scala new file mode 100644 index 000000000..2fbded582 --- /dev/null +++ b/server/src/main/scala/kpn/core/tools/next/support/compare/CompareEdges.scala @@ -0,0 +1,85 @@ +package kpn.core.tools.next.support.compare + +import kpn.api.common.route.RouteEdge +import kpn.core.doc.OldRouteDoc +import kpn.core.doc.RouteDetailDoc +import kpn.core.util.Log + +import scala.math.abs + +case class CompareEdge( + sourceNodeId: Long, + sinkNodeId: Long, + meters: Long +) + +class CompareEdges(oldRouteDoc: OldRouteDoc, newRouteDoc: RouteDetailDoc, log: Log) { + def compare(): Unit = { + if (newRouteDoc.segments.size == 1) { // cannot compare edges + val oldEdges = oldCompareEdges() + val newEdges = newCompareEdges() + if (!edgesEqual(oldEdges, newEdges)) { + val detail = Seq( + oldRouteDoc.edges.map(edge => s"old-edge $edge"), + newRouteDoc.edges.map(edge => s"new-edge $edge"), + oldEdges.map(edge => s"old-edge $edge"), + newEdges.map(edge => s"new-edge $edge"), + ).flatten.mkString("\n") + log.info(s"edge mismatch\n$detail") + } + } + } + + private def newCompareEdges(): Seq[CompareEdge] = { + sort(newRouteDoc.edges.map(toCompareEdge)) + } + + private def oldCompareEdges(): Seq[CompareEdge] = { + sort(distinctOldEdges(Seq.empty, oldRouteDoc.edges)) + } + + private def distinctOldEdges(edges: Seq[CompareEdge], remainingEdges: Seq[RouteEdge]): Seq[CompareEdge] = { + if (remainingEdges.isEmpty) { + edges + } + else { + val edge = toCompareEdge(remainingEdges.head) + if (edges.contains(edge)) { + distinctOldEdges(edges, remainingEdges.tail) + } + else { + distinctOldEdges(edges :+ edge, remainingEdges.tail) + } + } + } + + private def toCompareEdge(edge: RouteEdge): CompareEdge = { + CompareEdge( + edge.sourceNodeId, + edge.sinkNodeId, + edge.meters + ) + } + + private def sort(edges: Seq[CompareEdge]): Seq[CompareEdge] = { + edges.sortBy(edge => edge.sourceNodeId -> edge.sinkNodeId) + } + + private def edgesEqual(oldEdges: Seq[CompareEdge], newEdges: Seq[CompareEdge]): Boolean = { + if (oldEdges.size == newEdges.size) { + oldEdges.zip(newEdges).forall { case (oldEdge, newEdge) => + edgeEqual(oldEdge, newEdge) + } + } + else { + false + } + } + + private def edgeEqual(oldEdge: CompareEdge, newEdge: CompareEdge): Boolean = { + val metersEqual = abs(oldEdge.meters - newEdge.meters) < (newEdge.meters / 20) // 20% + oldEdge.sourceNodeId == newEdge.sourceNodeId && + oldEdge.sinkNodeId == newEdge.sinkNodeId && + metersEqual + } +} diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/RouteDetailMainAnalyzer.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/RouteDetailMainAnalyzer.scala index e31a656ca..588ded550 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/RouteDetailMainAnalyzer.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/RouteDetailMainAnalyzer.scala @@ -11,6 +11,7 @@ import kpn.core.analysis.RouteMember import kpn.core.analysis.RouteMemberWay import kpn.core.tools.next.domain.RouteRelation import kpn.core.util.Log +import kpn.server.analyzer.engine.analysis.route.analyzers.EdgeRouteAnalyzer import kpn.server.analyzer.engine.analysis.route.analyzers.FactCombinationAnalyzer import kpn.server.analyzer.engine.analysis.route.analyzers.FixmeTodoRouteAnalyzer import kpn.server.analyzer.engine.analysis.route.analyzers.GeometryDigestAnalyzer @@ -102,7 +103,7 @@ class RouteDetailMainAnalyzer( RouteElementsAnalyzer, // TODO oldRouteTileAnalyzer, routeTileAnalyzer, - // TODO EdgeRouteAnalyzer, + EdgeRouteAnalyzer, RouteLabelsAnalyzer, // this always should be the last analyzer RouteContextAnalyzer // helper to be used during development only ) diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/EdgeRouteAnalyzer.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/EdgeRouteAnalyzer.scala index caf781cda..26500cb5a 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/EdgeRouteAnalyzer.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/EdgeRouteAnalyzer.scala @@ -1,8 +1,8 @@ package kpn.server.analyzer.engine.analysis.route.analyzers -import kpn.api.common.common.TrackPath import kpn.api.common.route.RouteEdge import kpn.server.analyzer.engine.analysis.route.domain.RouteDetailAnalysisContext +import kpn.server.analyzer.engine.analysis.route.structure.StructurePath object EdgeRouteAnalyzer extends RouteAnalyzer { def analyze(context: RouteDetailAnalysisContext): RouteDetailAnalysisContext = { @@ -13,49 +13,16 @@ object EdgeRouteAnalyzer extends RouteAnalyzer { class EdgeRouteAnalyzer(context: RouteDetailAnalysisContext) { def analyze: RouteDetailAnalysisContext = { - - val routeMap = context.routeMap - val edges = Seq( - toEdges(routeMap.forwardPath), - toEdges(routeMap.backwardPath), - routeMap.freePaths.map(toEdge), - routeMap.startTentaclePaths.map(toEdge), - routeMap.endTentaclePaths.map(toEdge), - ).flatten + val edges = context.structure.nodeNetworkPaths.map(toEdge) context.copy(edges = edges) } - private def toEdges(trackPathOption: Option[TrackPath]): Seq[RouteEdge] = { - trackPathOption match { - case None => Seq.empty - case Some(trackPath) => - if (trackPath.oneWay) { - Seq(toEdge(trackPath)) - } - else { - Seq( - toEdge(trackPath), - toReverseEdge(trackPath) - ) - } - } - } - - private def toEdge(trackPath: TrackPath): RouteEdge = { - RouteEdge( - trackPath.pathId, - trackPath.startNodeId, - trackPath.endNodeId, - trackPath.meters - ) - } - - private def toReverseEdge(trackPath: TrackPath): RouteEdge = { + private def toEdge(path: StructurePath): RouteEdge = { RouteEdge( - 100L + trackPath.pathId, - trackPath.endNodeId, - trackPath.startNodeId, - trackPath.meters + path.id, + path.startNodeId, + path.endNodeId, + path.meters ) } } diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteAnalysisBuilder.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteAnalysisBuilder.scala index 30f3b4b45..1c864eb9b 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteAnalysisBuilder.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteAnalysisBuilder.scala @@ -30,7 +30,7 @@ class RouteAnalysisBuilder(context: RouteDetailAnalysisContext) { val route = buildRouteDetailDoc( title, - context._routeMembers.get, + context.routeMembers, context._ways.get, context.routeMap, context.unexpectedNodeIds, @@ -46,7 +46,7 @@ class RouteAnalysisBuilder(context: RouteDetailAnalysisContext) { routeDetail = route, structure = context.oldStructure, routeNodeAnalysis = context.oldRouteNodeAnalysis, - routeMembers = context._routeMembers.get, + routeMembers = context.routeMembers, ways = context._ways.get, startNodes = context.routeMap.startNodes, endNodes = context.routeMap.endNodes, diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteCountryAnalyzerImpl.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteCountryAnalyzerImpl.scala index b6f307ca9..5f2d1bbeb 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteCountryAnalyzerImpl.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteCountryAnalyzerImpl.scala @@ -15,7 +15,6 @@ class RouteCountryAnalyzerImpl(locationAnalyzer: LocationAnalyzer, routeReposito } context.copy( country = countryOption, - abort = countryOption.isEmpty ) } } diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteMapAnalyzer.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteMapAnalyzer.scala index 7c66882c1..1e1ee9418 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteMapAnalyzer.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteMapAnalyzer.scala @@ -29,7 +29,7 @@ class RouteMapAnalyzer(context: RouteDetailAnalysisContext) { def analyze: RouteDetailAnalysisContext = { - val ways: Seq[Way] = context._routeMembers.get.flatMap { + val ways: Seq[Way] = context.routeMembers.flatMap { case w: RouteMemberWay => Some(w.way) case _ => None } diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteStreetsAnalyzer.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteStreetsAnalyzer.scala index 61716a575..596a07c23 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteStreetsAnalyzer.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteStreetsAnalyzer.scala @@ -13,7 +13,7 @@ object RouteStreetsAnalyzer extends RouteAnalyzer { class RouteStreetsAnalyzer(context: RouteDetailAnalysisContext) { def analyze: RouteDetailAnalysisContext = { - val ways: Seq[Way] = context._routeMembers.get.flatMap { + val ways: Seq[Way] = context.routeMembers.flatMap { case w: RouteMemberWay => Some(w.way) case _ => None } diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteTagAnalyzer.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteTagAnalyzer.scala index 932d2a03c..be550b5f6 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteTagAnalyzer.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/analyzers/RouteTagAnalyzer.scala @@ -22,7 +22,8 @@ class RouteTagAnalyzer(context: RouteDetailAnalysisContext) { } else { context.relation.tagValue("route") match { - case None => context.copy(abort = true).withFacts(RouteTagMissing) + case None => + context.copy(abort = true).withFacts(RouteTagMissing) case Some(routeTagValue) => val superRoute = context.relation.hasTag("type", "superroute") val nodeNetwork = context.relation.hasTag("network:type", "node_network") diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/RouteAnalysisElement.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/RouteAnalysisElement.scala index 11b593a61..2472e3118 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/RouteAnalysisElement.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/RouteAnalysisElement.scala @@ -1,5 +1,7 @@ package kpn.server.analyzer.engine.analysis.route.structure +import kpn.api.common.data.Node + case class RouteAnalysisElement( id: Long, direction: RoutePathDirection, @@ -13,6 +15,13 @@ case class RouteAnalysisElement( fragmentGroups.flatMap(_.fragments) } + def nodes: Seq[Node] = { + fragments.headOption match { + case Some(firstFragment) => firstFragment.nodes ++ fragments.tail.flatMap(_.nodes.tail) + case None => Seq.empty + } + } + def nodeIds: Seq[Long] = { fragments.headOption match { case Some(firstFragment) => firstFragment.nodeIds ++ fragments.tail.flatMap(_.nodeIds.tail) diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/Structure.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/Structure.scala index 67fed353c..4d9c5f1df 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/Structure.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/Structure.scala @@ -7,7 +7,8 @@ case class Structure( endTentaclePaths: Seq[StructurePath], otherPaths: Seq[StructurePath] ) { - def paths: Seq[StructurePath] = { - forwardPath.toSeq ++ backwardPath.toSeq ++ startTentaclePaths ++ endTentaclePaths ++ otherPaths + def nodeNetworkPaths: Seq[StructurePath] = { + // does not include otherPaths + forwardPath.toSeq ++ backwardPath.toSeq ++ startTentaclePaths ++ endTentaclePaths } } diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePath.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePath.scala index 4ad7ddede..862c0e099 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePath.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePath.scala @@ -1,11 +1,22 @@ package kpn.server.analyzer.engine.analysis.route.structure +import kpn.api.common.data.Node +import kpn.core.util.Haversine + case class StructurePath( id: Long, startNodeId: Long, endNodeId: Long, elements: Seq[StructurePathElement] = Seq.empty, ) { + + def nodes: Seq[Node] = { + elements.headOption match { + case Some(firstElement) => firstElement.nodes.head +: elements.flatMap(_.nodes.tail) + case None => Seq.empty + } + } + def nodeIds: Seq[Long] = { elements.headOption match { case Some(firstElement) => firstElement.nodeIds.head +: elements.flatMap(_.nodeIds.tail) @@ -16,4 +27,8 @@ case class StructurePath( def elementIds: Seq[Long] = { elements.map(_.element.id) } + + def meters: Long = { + Haversine.meters(nodes) + } } diff --git a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePathElement.scala b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePathElement.scala index e682dca04..110caf99b 100644 --- a/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePathElement.scala +++ b/server/src/main/scala/kpn/server/analyzer/engine/analysis/route/structure/StructurePathElement.scala @@ -1,10 +1,21 @@ package kpn.server.analyzer.engine.analysis.route.structure +import kpn.api.common.data.Node + case class StructurePathElement( element: RouteAnalysisElement, reversed: Boolean ) { + def nodes: Seq[Node] = { + if (reversed) { + element.nodes.reverse + } + else { + element.nodes + } + } + def nodeIds: Seq[Long] = { if (reversed) { element.nodeIds.reverse