Add option to export GPX files as ZIP, add Osmand line width and colo…

…ur (#246)

frontend/src/lib/components/export-dialog.vue

@@ -44,8 +44,14 @@
 		...Object.values(client.value.types).flatMap((type) => type.fields.map((field) => field.name))
 	]));
 
+	const routeTypeOptions = {
+		"tracks": "Track points",
+		"zip": "Track points, one file per line (ZIP file)",
+		"routes": "Route points"
+	};
+
 	const format = ref<keyof typeof formatOptions>("gpx");
-	const useTracks = ref<"1" | "0">("1");
+	const routeType = ref<keyof typeof routeTypeOptions>("tracks");
 	const filter = ref(true);
 	const hide = ref(new Set<string>());
 	const typeId = ref<ID>();
@@ -69,7 +75,7 @@
 	const resolveTypeId = (typeId: ID | undefined) => typeId != null && client.value.types[typeId] ? typeId : undefined;
 	const resolvedTypeId = computed(() => resolveTypeId(typeId.value));
 
-	const canSelectUseTracks = computed(() => format.value === "gpx");
+	const canSelectRouteType = computed(() => format.value === "gpx");
 	const canSelectType = computed(() => format.value === "csv" || (format.value === "table" && method.value === "copy"));
 	const mustSelectType = computed(() => canSelectType.value);
 	const canSelectHide = computed(() => ["table", "csv"].includes(format.value));
@@ -83,8 +89,8 @@
 
 	const url = computed(() => {
 		const params = new URLSearchParams();
-		if (canSelectUseTracks.value) {
-			params.set("useTracks", useTracks.value);
+		if (canSelectRouteType.value) {
+			params.set("useTracks", routeType.value === "routes" ? "0" : "1");
 		}
 		if (canSelectHide.value && hide.value.size > 0) {
 			params.set("hide", [...hide.value].join(","));
@@ -130,6 +136,16 @@
 				);
 			}
 
+			case "gpx": {
+				return (
+					context.baseUrl
+						+ client.value.padData!.id
+						+ `/${format.value}`
+						+ (routeType.value === "zip" ? `/zip` : "")
+						+ (paramsStr ? `?${paramsStr}` : '')
+				);
+			}
+
 			default: {
 				return (
 					context.baseUrl
@@ -232,24 +248,27 @@
 			</div>
 		</div>
 
-		<div v-if="canSelectUseTracks" class="row mb-3">
+		<div v-if="canSelectRouteType" class="row mb-3">
 			<label class="col-sm-3 col-form-label" :for="`${id}-route-type-select`">
 				Route type
 				<HelpPopover>
 					<p>
 						<strong>Track points</strong> will export your lines exactly as they are on your map.
 					</p>
-
+					<p>
+						<strong>Track points, one file per line (ZIP file)</strong> will create a ZIP file with one GPX file
+						for all markers and one GPX file for each line. This works better with apps such as Osmand that only
+						support one line style per file.
+					</p>
 					<p>
 						<strong>Route points</strong> will export only the from/via/to route points of your lines, and your
 						navigation software/device will have to calculate the route using its own map data and algorithm.
 					</p>
 				</HelpPopover>
 			</label>
 			<div class="col-sm-9">
-				<select class="form-select" v-model="useTracks" :id="`${id}-route-type-select`">
-					<option value="1">Track points</option>
-					<option value="0">Route points</option>
+				<select class="form-select" v-model="routeType" :id="`${id}-route-type-select`">
+					<option v-for="(label, value) in routeTypeOptions" :value="value" :key="value">{{label}}</option>
 				</select>
 			</div>
 		</div>

frontend/src/lib/components/ui/export-dropdown.vue

@@ -7,6 +7,7 @@
 	import { useToasts } from "./toasts/toasts.vue";
 	import { saveAs } from "file-saver";
 	import vTooltip from "../../utils/tooltip";
+	import { getSafeFilename } from "facilmap-utils";
 
 	const context = injectContextRequired();
 	const toasts = useToasts();
@@ -26,7 +27,7 @@
 
 		try {
 			const exported = await props.getExport(format);
-			saveAs(new Blob([exported], { type: "application/gpx+xml" }), `${props.filename}.gpx`);
+			saveAs(new Blob([exported], { type: "application/gpx+xml" }), `${getSafeFilename(props.filename)}.gpx`);
 		} catch(err: any) {
 			toasts.showErrorToast(`fm${context.id}-export-dropdown-error`, "Error exporting route", err);
 		} finally {

frontend/src/lib/components/ui/popover.vue

@@ -149,12 +149,20 @@
 </template>
 
 <style lang="scss">
-	.fm-popover {
+	.fm-popover.fm-popover {
 		display: flex;
 		flex-direction: column;
 
 		> .popover-body {
 			overflow: auto;
+
+			> *:first-child {
+				margin-top: 0;
+			}
+
+			> *:last-child {
+				margin-bottom: 0;
+			}
 		}
 	}
 </style>

server/package.json

@@ -66,6 +66,7 @@
     "string-similarity": "^4.0.4",
     "strip-bom-buf": "^4.0.0",
     "unzipper": "^0.10.14",
+    "zip-stream": "^6.0.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {

server/src/database/helpers.ts

@@ -118,7 +118,7 @@ export default class DatabaseHelpers {
 		this._db = db;
 	}
 
-	async _updateObjectStyles(objects: Marker | Line | AsyncGenerator<Marker | Line, void, void>): Promise<void> {
+	async _updateObjectStyles(objects: Marker | Line | AsyncIterable<Marker | Line>): Promise<void> {
 		const types: Record<ID, Type> = { };
 		for await (const object of Symbol.asyncIterator in objects ? objects : arrayToAsyncIterator([objects])) {
 			const padId = object.padId;
@@ -187,7 +187,7 @@ export default class DatabaseHelpers {
 		return data;
 	}
 
-	async* _getPadObjects<T>(type: string, padId: PadId, condition?: FindOptions): AsyncGenerator<T, void, void> {
+	async* _getPadObjects<T>(type: string, padId: PadId, condition?: FindOptions): AsyncIterable<T> {
 		const includeData = [ "Marker", "Line" ].includes(type);
 
 		if(includeData) {
@@ -390,10 +390,23 @@ export default class DatabaseHelpers {
 		}
 	}
 
-	async _bulkCreateInBatches<T>(model: ModelCtor<Model>, data: Array<Record<string, unknown>>): Promise<Array<T>> {
+	async _bulkCreateInBatches<T>(model: ModelCtor<Model>, data: Iterable<Record<string, unknown>> | AsyncIterable<Record<string, unknown>>): Promise<Array<T>> {
 		const result: Array<any> = [];
-		for(let i=0; i<data.length; i+=ITEMS_PER_BATCH)
-			result.push(...(await model.bulkCreate(data.slice(i, i+ITEMS_PER_BATCH))).map((it) => it.toJSON()));
+		let slice: Array<Record<string, unknown>> = [];
+		const createSlice = async () => {
+			result.push(...(await model.bulkCreate(slice)).map((it) => it.toJSON()));
+			slice = [];
+		};
+
+		for await (const item of data) {
+			slice.push(item);
+			if (slice.length >= ITEMS_PER_BATCH) {
+				createSlice();
+			}
+		}
+		if (slice.length > 0) {
+			createSlice();
+		}
 		return result;
 	}
 

server/src/database/history.ts

@@ -100,7 +100,7 @@ export default class DatabaseHistory {
 	}
 
 
-	getHistory(padId: PadId, types?: HistoryEntryType[]): AsyncGenerator<HistoryEntry, void, never> {
+	getHistory(padId: PadId, types?: HistoryEntryType[]): AsyncIterable<HistoryEntry> {
 		const query: FindOptions = { order: [[ "time", "DESC" ]] };
 		if(types)
 			query.where = {type: types};

server/src/database/line.ts

@@ -179,22 +179,15 @@ export default class DatabaseLines {
 		this.LineModel.hasMany(this.LineDataModel, { foreignKey: "lineId" });
 	}
 
-	getPadLines(padId: PadId, fields?: Array<keyof Line>): AsyncGenerator<Line, void, void> {
+	getPadLines(padId: PadId, fields?: Array<keyof Line>): AsyncIterable<Line> {
 		const cond = fields ? { attributes: fields } : { };
 		return this._db.helpers._getPadObjects<Line>("Line", padId, cond);
 	}
 
-	getPadLinesByType(padId: PadId, typeId: ID): AsyncGenerator<Line, void, void> {
+	getPadLinesByType(padId: PadId, typeId: ID): AsyncIterable<Line> {
 		return this._db.helpers._getPadObjects<Line>("Line", padId, { where: { typeId: typeId } });
 	}
 
-	async* getPadLinesWithPoints(padId: PadId): AsyncGenerator<LineWithTrackPoints, void, void> {
-		for await (const line of this.getPadLines(padId)) {
-			const trackPoints = await this.getAllLinePoints(line.id);
-			yield { ...line, trackPoints };
-		}
-	}
-
 	async getLineTemplate(padId: PadId, data: { typeId: ID }): Promise<Line> {
 		const lineTemplate = {
 			...this.LineModel.build({ ...data, padId: padId } satisfies Partial<CreationAttributes<LineModel>> as any).toJSON(),
@@ -310,7 +303,7 @@ export default class DatabaseLines {
 		return oldLine;
 	}
 
-	async* getLinePointsForPad(padId: PadId, bboxWithZoom: BboxWithZoom & BboxWithExcept): AsyncGenerator<{ id: ID; trackPoints: TrackPoint[] }, void, void> {
+	async* getLinePointsForPad(padId: PadId, bboxWithZoom: BboxWithZoom & BboxWithExcept): AsyncIterable<{ id: ID; trackPoints: TrackPoint[] }> {
 		const lines = await this.LineModel.findAll({ attributes: ["id"], where: { padId } });
 		const chunks = chunk(lines.map((line) => line.id), 50000);
 		for (const lineIds of chunks) {
@@ -336,12 +329,14 @@ export default class DatabaseLines {
 		}
 	}
 
-	async getAllLinePoints(lineId: ID): Promise<TrackPoint[]> {
+	async* getAllLinePoints(lineId: ID): AsyncIterable<TrackPoint> {
 		const points = await this.LineModel.build({ id: lineId } satisfies Partial<CreationAttributes<LineModel>> as any).getLinePoints({
 			attributes: [ "pos", "lat", "lon", "ele", "zoom", "idx" ],
 			order: [["idx", "ASC"]]
 		});
-		return points.map((point) => omit(point.toJSON(), ["pos"]) as TrackPoint);
+		for (const point of points) {
+			yield omit(point.toJSON(), ["pos"]) as TrackPoint;
+		}
 	}
 
 }

server/src/database/marker.ts

@@ -76,11 +76,11 @@ export default class DatabaseMarkers {
 		this.MarkerModel.hasMany(this.MarkerDataModel, { foreignKey: "markerId" });
 	}
 
-	getPadMarkers(padId: PadId, bbox?: BboxWithZoom & BboxWithExcept): AsyncGenerator<Marker, void, void> {
+	getPadMarkers(padId: PadId, bbox?: BboxWithZoom & BboxWithExcept): AsyncIterable<Marker> {
 		return this._db.helpers._getPadObjects<Marker>("Marker", padId, { where: this._db.helpers.makeBboxCondition(bbox) });
 	}
 
-	getPadMarkersByType(padId: PadId, typeId: ID): AsyncGenerator<Marker, void, void> {
+	getPadMarkersByType(padId: PadId, typeId: ID): AsyncIterable<Marker> {
 		return this._db.helpers._getPadObjects<Marker>("Marker", padId, { where: { padId: padId, typeId: typeId } });
 	}
 

server/src/database/route.ts

@@ -6,6 +6,7 @@ import { type BboxWithExcept, createModel, getPosType, getVirtualLatType, getVir
 import { calculateRouteForLine } from "../routing/routing.js";
 import { omit } from "lodash-es";
 import type { Point as GeoJsonPoint } from "geojson";
+import { asyncIteratorToArray } from "../utils/streams.js";
 
 const updateTimes: Record<string, number> = {};
 
@@ -155,24 +156,24 @@ export default class DatabaseRoutes {
 			return;
 
 		const line = await this._db.lines.getLine(padId, lineId);
-		const linePoints = await this._db.lines.getAllLinePoints(lineId);
+		const linePointsIt = this._db.lines.getAllLinePoints(lineId);
+		const linePoints = await asyncIteratorToArray((async function*() {
+			for await (const linePoint of linePointsIt) {
+				yield {
+					routeId,
+					lat: linePoint.lat,
+					lon: linePoint.lon,
+					ele: linePoint.ele,
+					zoom: linePoint.zoom,
+					idx: linePoint.idx
+				};
+			}
+		})());
 
 		if(thisTime != updateTimes[routeId])
 			return;
 
-		const create = [];
-		for(const linePoint of linePoints) {
-			create.push({
-				routeId,
-				lat: linePoint.lat,
-				lon: linePoint.lon,
-				ele: linePoint.ele,
-				zoom: linePoint.zoom,
-				idx: linePoint.idx
-			});
-		}
-
-		await this._db.helpers._bulkCreateInBatches(this.RoutePointModel, create);
+		await this._db.helpers._bulkCreateInBatches(this.RoutePointModel, linePoints);
 
 		if(thisTime != updateTimes[routeId])
 			return;
@@ -214,12 +215,14 @@ export default class DatabaseRoutes {
 		return data.map((d) => omit(d.toJSON(), ["pos"]) as TrackPoint);
 	}
 
-	async getAllRoutePoints(routeId: string): Promise<TrackPoint[]> {
-		const data = await this.RoutePointModel.findAll({
-			where: {routeId},
+	async* getAllRoutePoints(routeId: string): AsyncIterable<TrackPoint> {
+		const points = await this.RoutePointModel.findAll({
+			where: { routeId },
 			attributes: [ "pos", "lat", "lon", "idx", "ele", "zoom"]
 		});
-		return data.map((d) => omit(d.toJSON(), ["pos"]) as TrackPoint);
+		for (const point of points) {
+			yield omit(point.toJSON(), ["pos"]) as TrackPoint;
+		}
 	}
 
 }

server/src/database/type.ts

@@ -85,7 +85,7 @@ export default class DatabaseTypes {
 		PadModel.hasMany(this.TypeModel, { foreignKey: "padId" });
 	}
 
-	getTypes(padId: PadId): AsyncGenerator<Type, void, void> {
+	getTypes(padId: PadId): AsyncIterable<Type> {
 		return this._db.helpers._getPadObjects<Type>("Type", padId);
 	}
 

server/src/database/view.ts

@@ -57,7 +57,7 @@ export default class DatabaseViews {
 		this._db.pads.PadModel.hasMany(this.ViewModel, { foreignKey: "padId" });
 	}
 
-	getViews(padId: PadId): AsyncGenerator<View, void, void> {
+	getViews(padId: PadId): AsyncIterable<View> {
 		return this._db.helpers._getPadObjects<View>("View", padId);
 	}