Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Use cache when reading OBF sections #797

Merged
merged 1 commit into from
Nov 4, 2024
Merged

Use cache when reading OBF sections #797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/Data/ObfMapSectionInfo_P.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "QtExtensions.h"
#include "ignore_warnings_on_external_includes.h"
#include <QMutex>
#include <QReadWriteLock>
#include <QSet>
#include <QHash>
#include <QMap>
Expand Down Expand Up @@ -44,6 +45,8 @@ namespace OsmAnd

bool hasChildrenDataBoxes;
uint32_t firstDataBoxInnerOffset;
mutable std::shared_ptr<const QList<ObfMapSectionDataBlockId>> subNodeIds;
mutable QMutex _subNodeIdsMutex;

friend class OsmAnd::ObfMapSectionReader_P;
};
Expand All @@ -57,6 +60,8 @@ namespace OsmAnd
mutable std::shared_ptr< const QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> > > _rootNodes;
mutable QAtomicInt _rootNodesLoaded;
mutable QMutex _rootNodesLoadMutex;
mutable QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>> _nodeCache;
mutable QReadWriteLock _nodeCacheAccessMutex;
public:
virtual ~ObfMapSectionLevel_P();

Expand Down
217 changes: 167 additions & 50 deletions src/Data/ObfMapSectionReader_P.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
#include "Logging.h"
#include "Utilities.h"

const int MAX_TREE_DEPTH_IN_CACHE = 7;

using google::protobuf::internal::WireFormatLite;

OsmAnd::ObfMapSectionReader_P::ObfMapSectionReader_P()
Expand Down Expand Up @@ -396,39 +398,139 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode,
QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>>& nodeCache,
QReadWriteLock& nodeCacheAccessMutex,
int treeDepth,
MapSurfaceType& outChildrenSurfaceType,
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
const std::shared_ptr<const IQueryController>& queryController,
ObfMapSectionReader_Metrics::Metric_loadMapObjects* const metric)
{
const auto cis = reader.getCodedInputStream().get();
QList<std::shared_ptr<const ObfMapSectionLevelTreeNode>> childNodes;

outChildrenSurfaceType = MapSurfaceType::Undefined;
for (;;)
bool fromCache = false;
if (!treeNode->subNodeIds)
{
const auto tag = cis->ReadTag();
switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
QMutexLocker scopedLocker(&treeNode->_subNodeIdsMutex);

if (!treeNode->subNodeIds)
{
case 0:
if (!ObfReaderUtilities::reachedDataEnd(cis))
return;
// Read all child nodes and make a lookup table
const std::shared_ptr<QList<ObfMapSectionDataBlockId>> subNodeIds(new QList<ObfMapSectionDataBlockId>);
const auto cis = reader.getCodedInputStream().get();
cis->Seek(treeNode->offset);
const auto oldLimit = cis->PushLimit(treeNode->length);
cis->Skip(treeNode->firstDataBoxInnerOffset);
outChildrenSurfaceType = MapSurfaceType::Undefined;
bool keepReading = true;
while (keepReading)
{
const auto tag = cis->ReadTag();
switch (gpb::internal::WireFormatLite::GetTagFieldNumber(tag))
{
case 0:
keepReading = false;
if (!ObfReaderUtilities::reachedDataEnd(cis))
break;

return;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
break;
case OBF::OsmAndMapIndex_MapDataBox::kBoxesFieldNumber:
{
const auto length = ObfReaderUtilities::readBigEndianInt(cis);
const auto offset = cis->CurrentPosition();
const auto prevLimit = cis->PushLimit(length);

std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(
new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->surfaceType = treeNode->surfaceType;
childNode->offset = offset;
childNode->length = length;
readTreeNode(reader, section, treeNode->area31, childNode);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(prevLimit);

// Update metric
if (metric)
metric->visitedNodes++;

ObfMapSectionDataBlockId subNodeId;
subNodeId.sectionRuntimeGeneratedId = section->runtimeGeneratedId;
subNodeId.offset = childNode->offset;

// Register node in lookup table
subNodeIds->append(subNodeId);

// Store node in cache
if (treeDepth < MAX_TREE_DEPTH_IN_CACHE)
{
QWriteLocker scopedLocker(&nodeCacheAccessMutex);

nodeCache.insert(subNodeId, childNode);
}

if (bbox31)
{
const auto shouldSkip =
!bbox31->contains(childNode->area31) &&
!childNode->area31.contains(*bbox31) &&
!bbox31->intersects(childNode->area31);
if (shouldSkip)
break;
}

// Update metric
if (metric)
metric->acceptedNodes++;

if (nodesWithData && childNode->dataOffset > 0)
nodesWithData->push_back(childNode);

if (childNode->hasChildrenDataBoxes)
childNodes.append(childNode);

if (childNode->surfaceType != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = childNode->surfaceType;
else if (outChildrenSurfaceType != childNode->surfaceType)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}

break;
}
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}
ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);

// Store lookup table
if (treeDepth < MAX_TREE_DEPTH_IN_CACHE)
{
const auto length = ObfReaderUtilities::readBigEndianInt(cis);
const auto offset = cis->CurrentPosition();
const auto oldLimit = cis->PushLimit(length);
treeNode->subNodeIds = subNodeIds;
}
}
else
fromCache = true;
}
else
fromCache = true;

const std::shared_ptr<ObfMapSectionLevelTreeNode> childNode(new ObfMapSectionLevelTreeNode(treeNode->level));
childNode->surfaceType = treeNode->surfaceType;
childNode->offset = offset;
childNode->length = length;
readTreeNode(reader, section, treeNode->area31, childNode);
if (fromCache)
{
// Use lookup table to access child nodes stored in cache
QReadLocker scopedLocker(&nodeCacheAccessMutex);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
for (const auto& subNodeId : constOf(*treeNode->subNodeIds))
{
const auto& citChildNode = nodeCache.constFind(subNodeId);
if (citChildNode != nodeCache.cend())
{
const auto& childNode = citChildNode.value();

// Update metric
if (metric)
Expand All @@ -441,7 +543,7 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
!childNode->area31.contains(*bbox31) &&
!bbox31->intersects(childNode->area31);
if (shouldSkip)
break;
continue;
}

// Update metric
Expand All @@ -451,33 +553,43 @@ void OsmAnd::ObfMapSectionReader_P::readTreeNodeChildren(
if (nodesWithData && childNode->dataOffset > 0)
nodesWithData->push_back(childNode);

auto subchildrenSurfaceType = MapSurfaceType::Undefined;
if (childNode->hasChildrenDataBoxes)
{
cis->Seek(childNode->offset);
const auto oldLimit = cis->PushLimit(childNode->length);

cis->Skip(childNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, childNode, subchildrenSurfaceType, nodesWithData, bbox31, queryController, metric);
childNodes.append(childNode);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
}

const auto surfaceTypeToMerge = (subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
if (childNode->surfaceType != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = surfaceTypeToMerge;
else if (outChildrenSurfaceType != surfaceTypeToMerge)
outChildrenSurfaceType = childNode->surfaceType;
else if (outChildrenSurfaceType != childNode->surfaceType)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}

break;
}
default:
ObfReaderUtilities::skipUnknownField(cis, tag);
break;
}
}

for (const auto& childNode : constOf(childNodes))
{
auto subchildrenSurfaceType = MapSurfaceType::Undefined;
readTreeNodeChildren(
reader,
section,
childNode,
nodeCache,
nodeCacheAccessMutex,
treeDepth + 1,
subchildrenSurfaceType,
nodesWithData,
bbox31,
queryController,
metric);
const auto surfaceTypeToMerge =
(subchildrenSurfaceType != MapSurfaceType::Undefined) ? subchildrenSurfaceType : childNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (outChildrenSurfaceType == MapSurfaceType::Undefined)
outChildrenSurfaceType = surfaceTypeToMerge;
else if (outChildrenSurfaceType != surfaceTypeToMerge)
outChildrenSurfaceType = MapSurfaceType::Mixed;
}
}
}
Expand Down Expand Up @@ -1102,17 +1214,22 @@ void OsmAnd::ObfMapSectionReader_P::loadMapObjects(
auto rootSubnodesSurfaceType = MapSurfaceType::Undefined;
if (rootNode->hasChildrenDataBoxes)
{
cis->Seek(rootNode->offset);
auto oldLimit = cis->PushLimit(rootNode->length);

cis->Skip(rootNode->firstDataBoxInnerOffset);
readTreeNodeChildren(reader, section, rootNode, rootSubnodesSurfaceType, &treeNodesWithData, bbox31, queryController, metric);

ObfReaderUtilities::ensureAllDataWasRead(cis);
cis->PopLimit(oldLimit);
readTreeNodeChildren(
reader,
section,
rootNode,
mapLevel->_p->_nodeCache,
mapLevel->_p->_nodeCacheAccessMutex,
0,
rootSubnodesSurfaceType,
&treeNodesWithData,
bbox31,
queryController,
metric);
}

const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined) ? rootSubnodesSurfaceType : rootNode->surfaceType;
const auto surfaceTypeToMerge = (rootSubnodesSurfaceType != MapSurfaceType::Undefined)
? rootSubnodesSurfaceType : rootNode->surfaceType;
if (surfaceTypeToMerge != MapSurfaceType::Undefined)
{
if (bboxOrSectionSurfaceType == MapSurfaceType::Undefined)
Expand Down
4 changes: 4 additions & 0 deletions src/Data/ObfMapSectionReader_P.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <QHash>
#include <QMap>
#include <QSet>
#include <QReadWriteLock>
#include "restore_internal_warnings.h"

#include "OsmAndCore.h"
Expand Down Expand Up @@ -81,6 +82,9 @@ namespace OsmAnd
const ObfReader_P& reader,
const std::shared_ptr<const ObfMapSectionInfo>& section,
const std::shared_ptr<const ObfMapSectionLevelTreeNode>& treeNode,
QHash<ObfMapSectionDataBlockId, std::shared_ptr<const ObfMapSectionLevelTreeNode>>& nodeCache,
QReadWriteLock& nodeCacheAccessMutex,
int treeDepth,
MapSurfaceType& outChildrenSurfaceType,
QList< std::shared_ptr<const ObfMapSectionLevelTreeNode> >* nodesWithData,
const AreaI* bbox31,
Expand Down