Skip to content

Commit

Permalink
Improved tile loading parallelism
Browse files Browse the repository at this point in the history
  • Loading branch information
@tumic0
tumic0 committed Dec 1, 2024
1 parent c014526 commit a11ba04
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 60 deletions.
94 changes: 51 additions & 43 deletions src/map/mapsforge/mapdata.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,10 @@ bool MapData::readTags(SubFile &subfile, int count,
return true;
}

bool MapData::readSubFiles()
bool MapData::readSubFiles(QFile &file)
{
/* both _pointFile and _pathFile can be used here */
QDataStream stream(&_pointFile);
QDataStream stream(&file);

for (int i = 0; i < _subFiles.size(); i++) {
const SubFileInfo &f = _subFiles.at(i);
Expand Down Expand Up @@ -422,8 +422,7 @@ bool MapData::readHeader(QFile &file)
return true;
}

MapData::MapData(const QString &fileName)
: _pointFile(fileName), _pathFile(fileName), _valid(false)
MapData::MapData(const QString &fileName) : _fileName(fileName), _valid(false)
{
QFile file(fileName);

Expand Down Expand Up @@ -460,17 +459,13 @@ RectC MapData::bounds() const

void MapData::load()
{
_pointFile.open(QIODevice::ReadOnly | QIODevice::Unbuffered);
_pathFile.open(QIODevice::ReadOnly | QIODevice::Unbuffered);

readSubFiles();
QFile file(_fileName);
if (file.open(QIODevice::ReadOnly | QIODevice::Unbuffered))
readSubFiles(file);
}

void MapData::clear()
{
_pointFile.close();
_pathFile.close();

_pathCache.clear();
_pointCache.clear();

Expand All @@ -494,14 +489,14 @@ void MapData::clearTiles()
bool MapData::pathCb(VectorTile *tile, void *context)
{
PathCTX *ctx = (PathCTX*)context;
ctx->data->paths(tile, ctx->rect, ctx->zoom, ctx->list);
ctx->data->paths(ctx->file, tile, ctx->rect, ctx->zoom, ctx->list);
return true;
}

bool MapData::pointCb(VectorTile *tile, void *context)
{
PointCTX *ctx = (PointCTX*)context;
ctx->data->points(tile, ctx->rect, ctx->zoom, ctx->list);
ctx->data->points(ctx->file, tile, ctx->rect, ctx->zoom, ctx->list);
return true;
}

Expand All @@ -514,13 +509,14 @@ int MapData::level(int zoom) const
return _subFiles.size() - 1;
}

void MapData::points(const RectC &rect, int zoom, QList<Point> *list)
void MapData::points(QFile &file, const RectC &rect, int zoom,
QList<Point> *list)
{
if (!rect.isValid())
return;

int l(level(zoom));
PointCTX ctx(this, rect, zoom, list);
PointCTX ctx(file, this, rect, zoom, list);
double min[2], max[2];

min[0] = rect.left();
Expand All @@ -531,53 +527,58 @@ void MapData::points(const RectC &rect, int zoom, QList<Point> *list)
_tiles.at(l)->Search(min, max, pointCb, &ctx);
}

void MapData::points(const VectorTile *tile, const RectC &rect, int zoom,
QList<Point> *list)
void MapData::points(QFile &file, VectorTile *tile, const RectC &rect,
int zoom, QList<Point> *list)
{
Key key(tile, zoom);

_pointLock.lock();
tile->lock.lock();

_pointCacheLock.lock();
QList<Point> *tilePoints = _pointCache.object(key);

if (!tilePoints) {
_pointCacheLock.unlock();
QList<Point> *p = new QList<Point>();
if (readPoints(tile, zoom, p)) {
if (readPoints(file, tile, zoom, p)) {
copyPoints(rect, p, list);
_pointCacheLock.lock();
_pointCache.insert(key, p);
_pointCacheLock.unlock();
} else
delete p;
} else
} else {
copyPoints(rect, tilePoints, list);
_pointCacheLock.unlock();
}

_pointLock.unlock();


_pathLock.lock();

_pathCacheLock.lock();
QList<Path> *tilePaths = _pathCache.object(key);

if (!tilePaths) {
_pathCacheLock.unlock();
QList<Path> *p = new QList<Path>();
if (readPaths(tile, zoom, p)) {
if (readPaths(file, tile, zoom, p)) {
copyPoints(rect, p, list);
_pathCacheLock.lock();
_pathCache.insert(key, p);
_pathCacheLock.unlock();
} else
delete p;
} else
} else {
copyPoints(rect, tilePaths, list);
_pathCacheLock.unlock();
}

_pathLock.unlock();
tile->lock.unlock();
}

void MapData::paths(const RectC &searchRect, const RectC &boundsRect, int zoom,
QList<Path> *list)
void MapData::paths(QFile &file, const RectC &searchRect,
const RectC &boundsRect, int zoom, QList<Path> *list)
{
if (!searchRect.isValid())
return;

int l(level(zoom));
PathCTX ctx(this, boundsRect, zoom, list);
PathCTX ctx(file, this, boundsRect, zoom, list);
double min[2], max[2];

min[0] = searchRect.left();
Expand All @@ -588,32 +589,38 @@ void MapData::paths(const RectC &searchRect, const RectC &boundsRect, int zoom,
_tiles.at(l)->Search(min, max, pathCb, &ctx);
}

void MapData::paths(const VectorTile *tile, const RectC &rect, int zoom,
void MapData::paths(QFile &file, VectorTile *tile, const RectC &rect, int zoom,
QList<Path> *list)
{
Key key(tile, zoom);

_pathLock.lock();
tile->lock.lock();

_pathCacheLock.lock();
QList<Path> *cached = _pathCache.object(key);

if (!cached) {
_pathCacheLock.unlock();
QList<Path> *p = new QList<Path>();
if (readPaths(tile, zoom, p)) {
if (readPaths(file, tile, zoom, p)) {
copyPaths(rect, p, list);
_pathCacheLock.lock();
_pathCache.insert(key, p);
_pathCacheLock.unlock();
} else
delete p;
} else
} else {
copyPaths(rect, cached, list);
_pathCacheLock.unlock();
}

_pathLock.unlock();
tile->lock.unlock();
}

bool MapData::readPaths(const VectorTile *tile, int zoom, QList<Path> *list)
bool MapData::readPaths(QFile &file, const VectorTile *tile, int zoom,
QList<Path> *list)
{
const SubFileInfo &info = _subFiles.at(level(zoom));
SubFile subfile(_pathFile, info.offset, info.size);
SubFile subfile(file, info.offset, info.size);
int rows = info.max - info.min + 1;
QVector<unsigned> paths(rows);
quint32 blocks, unused, val, cnt = 0;
Expand Down Expand Up @@ -698,10 +705,11 @@ bool MapData::readPaths(const VectorTile *tile, int zoom, QList<Path> *list)
return true;
}

bool MapData::readPoints(const VectorTile *tile, int zoom, QList<Point> *list)
bool MapData::readPoints(QFile &file, const VectorTile *tile, int zoom,
QList<Point> *list)
{
const SubFileInfo &info = _subFiles.at(level(zoom));
SubFile subfile(_pointFile, info.offset, info.size);
SubFile subfile(file, info.offset, info.size);
int rows = info.max - info.min + 1;
QVector<unsigned> points(rows);
quint32 val, unused, cnt = 0;
Expand Down
36 changes: 22 additions & 14 deletions src/map/mapsforge/mapdata.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,15 @@ class MapData
{return point.layer < other.point.layer;}
};

const QString &fileName() const {return _fileName;}
RectC bounds() const;
Range zooms() const
{return Range(_subFiles.first().min, _subFiles.last().max);}
int tileSize() const {return _tileSize;}

void points(const RectC &rect, int zoom, QList<Point> *list);
void paths(const RectC &searchRect, const RectC &boundsRect, int zoom,
QList<Path> *set);
void points(QFile &file, const RectC &rect, int zoom, QList<Point> *list);
void paths(QFile &file, const RectC &searchRect, const RectC &boundsRect,
int zoom, QList<Path> *set);
unsigned tagId(const QByteArray &name) const {return _keys.value(name);}

void load();
Expand All @@ -89,22 +90,27 @@ class MapData

size_t offset;
Coordinates pos;
QMutex lock;
};

struct PathCTX {
PathCTX(MapData *data, const RectC &rect, int zoom, QList<Path> *list)
: data(data), rect(rect), zoom(zoom), list(list) {}
PathCTX(QFile &file, MapData *data, const RectC &rect, int zoom,
QList<Path> *list)
: file(file), data(data), rect(rect), zoom(zoom), list(list) {}

QFile &file;
MapData *data;
const RectC &rect;
int zoom;
QList<Path> *list;
};

struct PointCTX {
PointCTX(MapData *data, const RectC &rect, int zoom, QList<Point> *list)
: data(data), rect(rect), zoom(zoom), list(list) {}
PointCTX(QFile &file, MapData *data, const RectC &rect, int zoom,
QList<Point> *list)
: file(file), data(data), rect(rect), zoom(zoom), list(list) {}

QFile &file;
MapData *data;
const RectC &rect;
int zoom;
Expand Down Expand Up @@ -143,16 +149,18 @@ class MapData
bool readTagInfo(SubFile &hdr, QVector<TagSource> &tags);
bool readMapInfo(SubFile &hdr, QByteArray &projection, bool &debugMap);
bool readHeader(QFile &file);
bool readSubFiles();
bool readSubFiles(QFile &file);
void clearTiles();

int level(int zoom) const;
void paths(const VectorTile *tile, const RectC &rect, int zoom,
void paths(QFile &file, VectorTile *tile, const RectC &rect, int zoom,
QList<Path> *list);
void points(const VectorTile *tile, const RectC &rect, int zoom,
void points(QFile &file, VectorTile *tile, const RectC &rect, int zoom,
QList<Point> *list);
bool readPaths(QFile &file, const VectorTile *tile, int zoom,
QList<Path> *list);
bool readPoints(QFile &file, const VectorTile *tile, int zoom,
QList<Point> *list);
bool readPaths(const VectorTile *tile, int zoom, QList<Path> *list);
bool readPoints(const VectorTile *tile, int zoom, QList<Point> *list);

static bool readTags(SubFile &subfile, int count,
const QVector<TagSource> &tags, QVector<Tag> &list);
Expand All @@ -161,7 +169,7 @@ class MapData

friend HASH_T qHash(const MapData::Key &key);

QFile _pointFile, _pathFile;
QString _fileName;
RectC _bounds;
quint16 _tileSize;
QVector<SubFileInfo> _subFiles;
Expand All @@ -171,7 +179,7 @@ class MapData

QCache<Key, QList<Path> > _pathCache;
QCache<Key, QList<Point> > _pointCache;
QMutex _pathLock, _pointLock;
QMutex _pathCacheLock, _pointCacheLock;

bool _valid;
QString _errorString;
Expand Down
10 changes: 7 additions & 3 deletions src/map/mapsforge/rastertile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,10 @@ void RasterTile::fetchData(QList<MapData::Path> &paths,
QList<MapData::Point> &points) const
{
QPoint ttl(_rect.topLeft());
QFile file(_data->fileName());

if (!file.open(QIODevice::ReadOnly | QIODevice::Unbuffered))
return;

QRectF pathRect(QPointF(ttl.x() - PATHS_EXTENT, ttl.y() - PATHS_EXTENT),
QPointF(ttl.x() + _rect.width() + PATHS_EXTENT, ttl.y() + _rect.height()
Expand All @@ -419,15 +423,15 @@ void RasterTile::fetchData(QList<MapData::Path> &paths,
_transform.img2proj(pathRect.bottomRight()));
RectD searchRectD(_transform.img2proj(searchRect.topLeft()),
_transform.img2proj(searchRect.bottomRight()));
_data->paths(searchRectD.toRectC(_proj, 20), pathRectD.toRectC(_proj, 20),
_zoom, &paths);
_data->paths(file, searchRectD.toRectC(_proj, 20),
pathRectD.toRectC(_proj, 20), _zoom, &paths);

QRectF pointRect(QPointF(ttl.x() - TEXT_EXTENT, ttl.y() - TEXT_EXTENT),
QPointF(ttl.x() + _rect.width() + TEXT_EXTENT, ttl.y() + _rect.height()
+ TEXT_EXTENT));
RectD pointRectD(_transform.img2proj(pointRect.topLeft()),
_transform.img2proj(pointRect.bottomRight()));
_data->points(pointRectD.toRectC(_proj, 20), _zoom, &points);
_data->points(file, pointRectD.toRectC(_proj, 20), _zoom, &points);
}

MatrixD RasterTile::elevation(int extend) const
Expand Down

0 comments on commit a11ba04

Please sign in to comment.