23#include <QJsonDocument>
28#include <QNetworkRequest>
42#include "qgspointcloudexpression.h"
48#define PROVIDER_KEY QStringLiteral( "ept" )
49#define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" )
51QgsEptPointCloudIndex::QgsEptPointCloudIndex()
56QgsEptPointCloudIndex::~QgsEptPointCloudIndex() =
default;
58void QgsEptPointCloudIndex::load(
const QString &urlString )
62 if ( url.isValid() && ( url.scheme() ==
"http" || url.scheme() ==
"https" ) )
68 QStringList splitUrl = mUri.split(
'/' );
70 mUrlDirectoryPart = splitUrl.join(
'/' );
75 QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );
91 if ( !f.open( QIODevice::ReadOnly ) )
93 mError = QObject::tr(
"Unable to open %1 for reading" ).arg( mUri );
97 content = f.readAll();
100 bool success = loadSchema( content );
104 const QString manifestPath = mUrlDirectoryPart + QStringLiteral(
"/ept-sources/manifest.json" );
105 QByteArray manifestJson;
108 QUrl manifestUrl( manifestPath );
110 QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
118 QFile manifestFile( manifestPath );
119 if ( manifestFile.open( QIODevice::ReadOnly ) )
120 manifestJson = manifestFile.readAll();
123 if ( !manifestJson.isEmpty() )
124 loadManifest( manifestJson );
129 QgsDebugError( QStringLiteral(
"Failed to load root EPT node" ) );
136void QgsEptPointCloudIndex::loadManifest(
const QByteArray &manifestJson )
140 const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
141 if ( err.error != QJsonParseError::NoError )
144 const QJsonArray manifestArray = manifestDoc.array();
145 if ( manifestArray.empty() )
149 const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
150 const QString metadataPath = sourceObject.value( QStringLiteral(
"metadataPath" ) ).toString();
151 const QString fullMetadataPath = mUrlDirectoryPart + QStringLiteral(
"/ept-sources/" ) + metadataPath;
153 QByteArray metadataJson;
156 QUrl metadataUrl( fullMetadataPath );
157 QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
166 QFile metadataFile( fullMetadataPath );
167 if ( ! metadataFile.open( QIODevice::ReadOnly ) )
169 metadataJson = metadataFile.readAll();
172 const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
173 if ( err.error != QJsonParseError::NoError )
176 const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral(
"metadata" ) ).toObject();
177 if ( metadataObject.empty() )
180 const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
181 mOriginalMetadata = sourceMetadata.toVariantMap();
184bool QgsEptPointCloudIndex::loadSchema(
const QByteArray &dataJson )
187 const QJsonDocument doc = QJsonDocument::fromJson( dataJson, &err );
188 if ( err.error != QJsonParseError::NoError )
190 const QJsonObject result = doc.object();
191 mDataType = result.value( QLatin1String(
"dataType" ) ).toString();
192 if ( mDataType != QLatin1String(
"laszip" ) && mDataType != QLatin1String(
"binary" ) && mDataType != QLatin1String(
"zstandard" ) )
195 const QString hierarchyType = result.value( QLatin1String(
"hierarchyType" ) ).toString();
196 if ( hierarchyType != QLatin1String(
"json" ) )
199 mSpan = result.value( QLatin1String(
"span" ) ).toInt();
200 mPointCount = result.value( QLatin1String(
"points" ) ).toDouble();
203 const QJsonObject srs = result.value( QLatin1String(
"srs" ) ).toObject();
204 mWkt = srs.value( QLatin1String(
"wkt" ) ).toString();
207 const QJsonArray bounds = result.value( QLatin1String(
"bounds" ) ).toArray();
208 if ( bounds.size() != 6 )
211 const QJsonArray boundsConforming = result.value( QLatin1String(
"boundsConforming" ) ).toArray();
212 if ( boundsConforming.size() != 6 )
214 mExtent.set( boundsConforming[0].toDouble(), boundsConforming[1].toDouble(),
215 boundsConforming[3].toDouble(), boundsConforming[4].toDouble() );
216 mZMin = boundsConforming[2].toDouble();
217 mZMax = boundsConforming[5].toDouble();
219 const QJsonArray schemaArray = result.value( QLatin1String(
"schema" ) ).toArray();
222 for (
const QJsonValue &schemaItem : schemaArray )
224 const QJsonObject schemaObj = schemaItem.toObject();
225 const QString name = schemaObj.value( QLatin1String(
"name" ) ).toString();
226 const QString type = schemaObj.value( QLatin1String(
"type" ) ).toString();
228 const int size = schemaObj.value( QLatin1String(
"size" ) ).toInt();
230 if ( name == QLatin1String(
"ClassFlags" ) && size == 1 )
237 else if ( type == QLatin1String(
"float" ) && ( size == 4 ) )
241 else if ( type == QLatin1String(
"float" ) && ( size == 8 ) )
245 else if ( size == 1 )
249 else if ( type == QLatin1String(
"unsigned" ) && size == 2 )
253 else if ( size == 2 )
257 else if ( size == 4 )
268 if ( schemaObj.contains( QLatin1String(
"scale" ) ) )
269 scale = schemaObj.value( QLatin1String(
"scale" ) ).toDouble();
272 if ( schemaObj.contains( QLatin1String(
"offset" ) ) )
273 offset = schemaObj.value( QLatin1String(
"offset" ) ).toDouble();
275 if ( name == QLatin1String(
"X" ) )
277 mOffset.set( offset, mOffset.y(), mOffset.z() );
278 mScale.set( scale, mScale.y(), mScale.z() );
280 else if ( name == QLatin1String(
"Y" ) )
282 mOffset.set( mOffset.x(), offset, mOffset.z() );
283 mScale.set( mScale.x(), scale, mScale.z() );
285 else if ( name == QLatin1String(
"Z" ) )
287 mOffset.set( mOffset.x(), mOffset.y(), offset );
288 mScale.set( mScale.x(), mScale.y(), scale );
292 AttributeStatistics stats;
293 bool foundStats =
false;
294 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
296 stats.count = schemaObj.value( QLatin1String(
"count" ) ).toInt();
299 if ( schemaObj.contains( QLatin1String(
"minimum" ) ) )
301 stats.minimum = schemaObj.value( QLatin1String(
"minimum" ) ).toDouble();
304 if ( schemaObj.contains( QLatin1String(
"maximum" ) ) )
306 stats.maximum = schemaObj.value( QLatin1String(
"maximum" ) ).toDouble();
309 if ( schemaObj.contains( QLatin1String(
"count" ) ) )
311 stats.mean = schemaObj.value( QLatin1String(
"mean" ) ).toDouble();
314 if ( schemaObj.contains( QLatin1String(
"stddev" ) ) )
316 stats.stDev = schemaObj.value( QLatin1String(
"stddev" ) ).toDouble();
319 if ( schemaObj.contains( QLatin1String(
"variance" ) ) )
321 stats.variance = schemaObj.value( QLatin1String(
"variance" ) ).toDouble();
325 mMetadataStats.insert( name, stats );
327 if ( schemaObj.contains( QLatin1String(
"counts" ) ) )
329 QMap< int, int > classCounts;
330 const QJsonArray counts = schemaObj.value( QLatin1String(
"counts" ) ).toArray();
331 for (
const QJsonValue &count : counts )
333 const QJsonObject countObj = count.toObject();
334 classCounts.insert( countObj.value( QLatin1String(
"value" ) ).toInt(), countObj.value( QLatin1String(
"count" ) ).toInt() );
336 mAttributeClasses.insert( name, classCounts );
339 setAttributes( attributes );
344 const double xmin = bounds[0].toDouble();
345 const double ymin = bounds[1].toDouble();
346 const double zmin = bounds[2].toDouble();
347 const double xmax = bounds[3].toDouble();
348 const double ymax = bounds[4].toDouble();
349 const double zmax = bounds[5].toDouble();
351 mRootBounds =
QgsBox3D( xmin, ymin, zmin, xmax, ymax, zmax );
354 double dx = xmax - xmin, dy = ymax - ymin, dz = zmax - zmin;
355 QgsDebugMsgLevel( QStringLiteral(
"lvl0 node size in CRS units: %1 %2 %3" ).arg( dx ).arg( dy ).arg( dz ), 2 );
356 QgsDebugMsgLevel( QStringLiteral(
"res at lvl0 %1" ).arg( dx / mSpan ), 2 );
357 QgsDebugMsgLevel( QStringLiteral(
"res at lvl1 %1" ).arg( dx / mSpan / 2 ), 2 );
358 QgsDebugMsgLevel( QStringLiteral(
"res at lvl2 %1 with node size %2" ).arg( dx / mSpan / 4 ).arg( dx / 4 ), 2 );
368 return std::unique_ptr<QgsPointCloudBlock>( cached );
371 std::unique_ptr<QgsPointCloudBlock> block;
374 std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
382 block = blockRequest->takeBlock();
385 QgsDebugError( QStringLiteral(
"Error downloading node %1 data, error : %2 " ).arg( n.
toString(), blockRequest->errorStr() ) );
393 QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
395 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
398 if ( mDataType == QLatin1String(
"binary" ) )
400 const QString filename = QStringLiteral(
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
401 block = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
403 else if ( mDataType == QLatin1String(
"zstandard" ) )
405 const QString filename = QStringLiteral(
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
406 block = QgsEptDecoder::decompressZStandard( filename, attributes(), request.
attributes(), scale(), offset(), filterExpression, filterRect );
408 else if ( mDataType == QLatin1String(
"laszip" ) )
410 const QString filename = QStringLiteral(
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
411 block = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
415 storeNodeDataToCache( block.get(), n, request );
424 scale(), offset(), mFilterExpression, request.
filterRect() );
430 if ( !loadNodeHierarchy( n ) )
434 if ( mDataType == QLatin1String(
"binary" ) )
436 fileUrl = QStringLiteral(
"%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.
toString() );
438 else if ( mDataType == QLatin1String(
"zstandard" ) )
440 fileUrl = QStringLiteral(
"%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.
toString() );
442 else if ( mDataType == QLatin1String(
"laszip" ) )
444 fileUrl = QStringLiteral(
"%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.
toString() );
454 QgsPointCloudExpression filterExpression = request.
ignoreIndexFilterEnabled() ? QgsPointCloudExpression() : mFilterExpression;
456 requestAttributes.
extend( attributes(), filterExpression.referencedAttributes() );
462 return loadNodeHierarchy( n );
470qint64 QgsEptPointCloudIndex::pointCount()
const
485 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot(
id );
486 for (
int i = pathToRoot.size() - 1; i >= 0; --i )
488 loadSingleNodeHierarchy( pathToRoot[i] );
490 QMutexLocker locker( &mHierarchyMutex );
491 qint64 pointCount = mHierarchy.value(
id, -1 );
492 if ( pointCount != -1 )
502 QMap<QString, QgsPointCloudAttributeStatistics> statsMap;
505 QString name = attribute.name();
506 const AttributeStatistics &stats = mMetadataStats[ name ];
507 if ( !stats.minimum.isValid() )
510 s.
minimum = stats.minimum.toDouble();
511 s.
maximum = stats.maximum.toDouble();
513 s.
stDev = stats.stDev;
514 s.
count = stats.count;
518 statsMap[ name ] = std::move( s );
523bool QgsEptPointCloudIndex::loadSingleNodeHierarchy(
const QgsPointCloudNodeId &nodeId )
const
525 mHierarchyMutex.lock();
526 const bool foundInHierarchy = mHierarchy.contains( nodeId );
527 const bool foundInHierarchyNodes = mHierarchyNodes.contains( nodeId );
528 mHierarchyMutex.unlock();
530 if ( foundInHierarchy )
533 if ( !foundInHierarchyNodes )
536 const QString filePath = QStringLiteral(
"%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, nodeId.
toString() );
538 QByteArray dataJsonH;
541 QNetworkRequest nr( filePath );
543 nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
544 nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute,
true );
552 if ( reply->error() != QNetworkReply::NoError )
554 QgsDebugError( QStringLiteral(
"Request failed: " ) + filePath );
558 dataJsonH = reply->data();
562 QFile file( filePath );
563 if ( ! file.open( QIODevice::ReadOnly ) )
565 QgsDebugError( QStringLiteral(
"Loading file failed: " ) + filePath );
568 dataJsonH = file.readAll();
571 QJsonParseError errH;
572 const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
573 if ( errH.error != QJsonParseError::NoError )
575 QgsDebugMsgLevel( QStringLiteral(
"QJsonParseError when reading hierarchy from file %1" ).arg( filePath ), 2 );
579 QMutexLocker locker( &mHierarchyMutex );
580 const QJsonObject rootHObj = docH.object();
581 for (
auto it = rootHObj.constBegin(); it != rootHObj.constEnd(); ++it )
583 const QString nodeIdStr = it.key();
584 const int nodePointCount = it.value().toInt();
586 if ( nodePointCount >= 0 )
587 mHierarchy[nodeId] = nodePointCount;
588 else if ( nodePointCount == -1 )
589 mHierarchyNodes.insert( nodeId );
595QVector<QgsPointCloudNodeId> QgsEptPointCloudIndex::nodePathToRoot(
const QgsPointCloudNodeId &nodeId )
const
597 QVector<QgsPointCloudNodeId> path;
601 path.push_back( currentNode );
604 while ( currentNode.
d() >= 0 );
613 QMutexLocker lock( &mHierarchyMutex );
614 found = mHierarchy.contains( nodeId );
619 QVector<QgsPointCloudNodeId> pathToRoot = nodePathToRoot( nodeId );
620 for (
int i = pathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
623 if ( !loadSingleNodeHierarchy( node ) )
628 QMutexLocker lock( &mHierarchyMutex );
629 found = mHierarchy.contains( nodeId );
636bool QgsEptPointCloudIndex::isValid()
const
647#undef PROVIDER_DESCRIPTION
PointCloudAccessType
The access type of the data, local is for local files and remote for remote files (over HTTP).
@ Local
Local means the source is a local file on the machine.
@ Remote
Remote means it's loaded through a protocol like HTTP.
virtual QgsPointCloudNode getNode(const QgsPointCloudNodeId &id) const
Returns object for a given node.
static QgsTileDownloadManager * tileDownloadManager()
Returns the application's tile download manager, used for download of map tiles when rendering.
A thread safe class for performing blocking (sync) network requests, with full support for QGIS proxy...
QString errorMessage() const
Returns the error message string, after a get(), post(), head() or put() request has been made.
ErrorCode get(QNetworkRequest &request, bool forceRefresh=false, QgsFeedback *feedback=nullptr, RequestFlags requestFlags=QgsBlockingNetworkRequest::RequestFlags())
Performs a "get" operation on the specified request.
@ NoError
No error was encountered.
QgsNetworkReplyContent reply() const
Returns the content of the network reply, after a get(), post(), head() or put() request has been mad...
A 3-dimensional box composed of x, y, z coordinates.
Class for handling a QgsPointCloudBlockRequest using existing cached QgsPointCloudBlock.
This class represents a coordinate reference system (CRS).
static QgsCoordinateReferenceSystem fromWkt(const QString &wkt)
Creates a CRS from a WKT spatial ref sys definition string.
Base class for handling loading QgsPointCloudBlock asynchronously from a remote EPT dataset.
QByteArray content() const
Returns the reply content.
Collection of point cloud attributes.
void push_back(const QgsPointCloudAttribute &attribute)
Adds extra attribute.
void extend(const QgsPointCloudAttributeCollection &otherCollection, const QSet< QString > &matchingNames)
Adds specific missing attributes from another QgsPointCloudAttributeCollection.
Attribute for point cloud data pair of name and size in bytes.
@ UShort
Unsigned short int 2 bytes.
@ Short
Short int 2 bytes.
@ UChar
Unsigned char 1 byte.
Base class for handling loading QgsPointCloudBlock asynchronously.
void finished()
Emitted when the request processing has finished.
Base class for storing raw data from point cloud nodes.
Represents a indexed point cloud node's position in octree.
static QgsPointCloudNodeId fromString(const QString &str)
Creates node from string.
QString toString() const
Encode node to string.
QgsPointCloudNodeId parentNode() const
Returns the parent of the node.
Keeps metadata for indexed point cloud node.
QList< QgsPointCloudNodeId > children() const
Returns IDs of child nodes.
qint64 pointCount() const
Returns number of points contained in node data.
float error() const
Returns node's error in map units (used to determine in whether the node has enough detail for the cu...
QgsBox3D bounds() const
Returns node's bounding cube in CRS coords.
Point cloud data request.
bool ignoreIndexFilterEnabled() const
Returns whether the request will ignore the point cloud index's filter expression,...
QgsPointCloudAttributeCollection attributes() const
Returns attributes.
QgsRectangle filterRect() const
Returns the rectangle from which points will be taken, in point cloud's crs.
Class used to store statistics of a point cloud dataset.
A rectangle specified with double values.
void finished()
Emitted when the reply has finished (either with a success or with a failure)
#define QgsDebugMsgLevel(str, level)
#define QgsDebugError(str)
#define QgsSetRequestInitiatorClass(request, _class)
Class used to store statistics of one attribute of a point cloud dataset.
QMap< int, int > classCount