Client status reporting feature. First iteration.

Signed-off-by: alex-z <blackslayer4@gmail.com>
This commit is contained in:
alex-z 2023-11-21 20:35:38 +01:00 коммит произвёл allexzander
Родитель d3562e6205
Коммит ff8db2674a
13 изменённых файлов: 544 добавлений и 16 удалений

Просмотреть файл

@ -0,0 +1,24 @@
/*
* Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#include "clientstatusreportingrecord.h"
namespace OCC
{
bool ClientStatusReportingRecord::isValid() const
{
return !_name.isEmpty() && _nameHash > 0 && _lastOccurence > 0;
}
}

Просмотреть файл

@ -0,0 +1,34 @@
/*
* Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#pragma once
#include "ocsynclib.h"
#include <QtGlobal>
#include <QByteArray>
namespace OCC
{
/**
* @brief The ClientStatusReportingRecord class
* @ingroup libsync
*/
struct OCSYNC_EXPORT ClientStatusReportingRecord {
QByteArray _name;
quint64 _nameHash = 0;
quint64 _numOccurences = 1;
quint64 _lastOccurence = 0;
[[nodiscard]] bool isValid() const;
};
}

Просмотреть файл

@ -4,6 +4,7 @@
set(common_SOURCES
${CMAKE_CURRENT_LIST_DIR}/checksums.cpp
${CMAKE_CURRENT_LIST_DIR}/checksumcalculator.cpp
${CMAKE_CURRENT_LIST_DIR}/clientstatusreportingrecord.cpp
${CMAKE_CURRENT_LIST_DIR}/filesystembase.cpp
${CMAKE_CURRENT_LIST_DIR}/ownsql.cpp
${CMAKE_CURRENT_LIST_DIR}/preparedsqlquerymanager.cpp

Просмотреть файл

@ -121,10 +121,10 @@ set(client_SRCS
navigationpanehelper.cpp
networksettings.h
networksettings.cpp
"${CMAKE_SOURCE_DIR}/src/libsync/ocsjob.h"
"${CMAKE_SOURCE_DIR}/src/libsync/ocsjob.cpp"
ocsnavigationappsjob.h
ocsnavigationappsjob.cpp
ocsjob.h
ocsjob.cpp
ocssharejob.h
ocssharejob.cpp
ocsshareejob.h

Просмотреть файл

@ -24,6 +24,7 @@ set(libsync_SRCS
capabilities.cpp
clientproxy.h
clientproxy.cpp
clientstatusreporting.cpp
cookiejar.h
cookiejar.cpp
discovery.h
@ -53,6 +54,10 @@ set(libsync_SRCS
owncloudpropagator.cpp
nextcloudtheme.h
nextcloudtheme.cpp
ocsjob.h
ocsjob.cpp
ocsclientstatusreportingjob.h
ocsclientstatusreportingjob.cpp
abstractpropagateremotedeleteencrypted.h
abstractpropagateremotedeleteencrypted.cpp
deletejob.h
@ -171,7 +176,7 @@ IF (NOT APPLE)
)
ENDIF(NOT APPLE)
find_package(Qt5 REQUIRED COMPONENTS WebSockets Xml)
find_package(Qt5 REQUIRED COMPONENTS WebSockets Xml Sql)
add_library(nextcloudsync SHARED ${libsync_SRCS})
add_library(Nextcloud::sync ALIAS nextcloudsync)
@ -186,6 +191,7 @@ target_link_libraries(nextcloudsync
Qt5::Network
Qt5::WebSockets
Qt5::Xml
Qt5::Sql
)
if (NOT TOKEN_AUTH_ONLY)

Просмотреть файл

@ -17,6 +17,7 @@
#include "accountfwd.h"
#include "capabilities.h"
#include "clientsideencryptionjobs.h"
#include "clientstatusreporting.h"
#include "configfile.h"
#include "cookiejar.h"
#include "creds/abstractcredentials.h"
@ -64,7 +65,8 @@ constexpr int checksumRecalculateRequestServerVersionMinSupportedMajor = 24;
constexpr auto isSkipE2eeMetadataChecksumValidationAllowedInClientVersion = MIRALL_VERSION_MAJOR == 3 && MIRALL_VERSION_MINOR == 8;
}
namespace OCC {
namespace OCC
{
Q_LOGGING_CATEGORY(lcAccount, "nextcloud.sync.account", QtInfoMsg)
const char app_password[] = "_app-password";
@ -87,7 +89,7 @@ AccountPtr Account::create()
return acc;
}
ClientSideEncryption* Account::e2e()
ClientSideEncryption *Account::e2e()
{
// Qt expects everything in the connect to be a pointer, so return a pointer.
return &_e2e;
@ -267,14 +269,10 @@ void Account::setCredentials(AbstractCredentials *cred)
if (proxy.type() != QNetworkProxy::DefaultProxy) {
_am->setProxy(proxy);
}
connect(_am.data(), &QNetworkAccessManager::sslErrors,
this, &Account::slotHandleSslErrors);
connect(_am.data(), &QNetworkAccessManager::proxyAuthenticationRequired,
this, &Account::proxyAuthenticationRequired);
connect(_credentials.data(), &AbstractCredentials::fetched,
this, &Account::slotCredentialsFetched);
connect(_credentials.data(), &AbstractCredentials::asked,
this, &Account::slotCredentialsAsked);
connect(_am.data(), &QNetworkAccessManager::sslErrors, this, &Account::slotHandleSslErrors);
connect(_am.data(), &QNetworkAccessManager::proxyAuthenticationRequired, this, &Account::proxyAuthenticationRequired);
connect(_credentials.data(), &AbstractCredentials::fetched, this, &Account::slotCredentialsFetched);
connect(_credentials.data(), &AbstractCredentials::asked, this, &Account::slotCredentialsAsked);
trySetupPushNotifications();
}
@ -284,6 +282,18 @@ void Account::setPushNotificationsReconnectInterval(int interval)
_pushNotificationsReconnectTimer.setInterval(interval);
}
void Account::trySetupClientStatusReporting()
{
_clientStatusReporting.reset(new ClientStatusReporting(this));
}
void Account::reportClientStatus(const int status)
{
if (_clientStatusReporting) {
_clientStatusReporting->reportClientStatus(static_cast<ClientStatusReporting::Status>(status));
}
}
void Account::trySetupPushNotifications()
{
// Stop the timer to prevent parallel setup attempts
@ -669,6 +679,8 @@ void Account::setCapabilities(const QVariantMap &caps)
setupUserStatusConnector();
trySetupPushNotifications();
trySetupClientStatusReporting();
}
void Account::setupUserStatusConnector()

Просмотреть файл

@ -55,6 +55,7 @@ class AbstractCredentials;
class Account;
using AccountPtr = QSharedPointer<Account>;
class AccessManager;
class ClientStatusReporting;
class SimpleNetworkJob;
class PushNotifications;
class UserStatusConnector;
@ -305,6 +306,10 @@ public:
[[nodiscard]] PushNotifications *pushNotifications() const;
void setPushNotificationsReconnectInterval(int interval);
void trySetupClientStatusReporting();
void reportClientStatus(const int status);
[[nodiscard]] std::shared_ptr<UserStatusConnector> userStatusConnector() const;
void setLockFileState(const QString &serverRelativePath,
@ -439,6 +444,8 @@ private:
PushNotifications *_pushNotifications = nullptr;
QScopedPointer<ClientStatusReporting> _clientStatusReporting;
std::shared_ptr<UserStatusConnector> _userStatusConnector;
QHash<QString, QVector<SyncFileItem::LockStatus>> _lockStatusChangeInprogress;

Просмотреть файл

@ -0,0 +1,283 @@
/*
* Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#include "clientstatusreporting.h"
#include "creds/abstractcredentials.h"
#include "account.h"
#include "common/clientstatusreportingrecord.h"
#include "common/syncjournaldb.h"
#include <configfile.h>
#include <networkjobs.h>
namespace
{
constexpr auto lastSentReportTimestamp = "lastClientStatusReportSentTime";
constexpr auto repordSendIntervalMs = 24 * 60 * 60 * 1000;
constexpr int clientStatusReportingSendTimerInterval = 1000 * 60 * 2;
}
namespace OCC
{
Q_LOGGING_CATEGORY(lcClientStatusReporting, "nextcloud.sync.clientstatusreporting", QtInfoMsg)
ClientStatusReporting::ClientStatusReporting(Account *account, QObject *parent)
: _account(account)
, QObject(parent)
{
init();
}
void ClientStatusReporting::init()
{
if (_isInitialized) {
qCDebug(lcClientStatusReporting) << "Double call to init";
return;
}
for (int i = 0; i < ClientStatusReporting::Count; ++i) {
const auto statusString = statusStringFromNumber(static_cast<Status>(i));
_statusNamesAndHashes[i] = {statusString, SyncJournalDb::getPHash(statusString)};
}
const auto databaseId = QStringLiteral("%1@%2").arg(_account->davUser(), _account->url().toString());
const auto databaseIdHash = QCryptographicHash::hash(databaseId.toUtf8(), QCryptographicHash::Md5);
const QString journalPath = ConfigFile().configPath() + QStringLiteral(".userdata_%1.db").arg(QString::fromLatin1(databaseIdHash.left(6).toHex()));
_database = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"));
_database.setDatabaseName(journalPath);
if (!_database.open()) {
qCDebug(lcClientStatusReporting) << "Could not setup client reporting, database connection error.";
return;
}
QSqlQuery query;
const auto prepareResult = query.prepare(
"CREATE TABLE IF NOT EXISTS clientstatusreporting("
"nHash INTEGER(8) PRIMARY KEY,"
"name VARCHAR(4096),"
"count INTEGER,"
"lastOccurrence INTEGER(8))");
if (!prepareResult || !query.exec()) {
qCDebug(lcClientStatusReporting) << "Could not setup client clientstatusreporting table:" << query.lastError().text();
return;
}
if (!query.prepare("CREATE TABLE IF NOT EXISTS keyvalue(key VARCHAR(4096), value VARCHAR(4096), PRIMARY KEY(key))") || !query.exec()) {
qCDebug(lcClientStatusReporting) << "Could not setup client keyvalue table:" << query.lastError().text();
return;
}
_clientStatusReportingSendTimer.setInterval(clientStatusReportingSendTimerInterval);
connect(&_clientStatusReportingSendTimer, &QTimer::timeout, this, &ClientStatusReporting::sendReportToServer);
_clientStatusReportingSendTimer.start();
_isInitialized = true;
reportClientStatus(Status::DownloadError_ConflictCaseClash);
reportClientStatus(Status::DownloadError_ConflictInvalidCharacters);
reportClientStatus(Status::UploadError_ServerError);
reportClientStatus(Status::UploadError_ServerError);
setLastSentReportTimestamp(QDateTime::currentDateTime().toMSecsSinceEpoch());
auto records = getClientStatusReportingRecords();
auto resDelete = deleteClientStatusReportingRecords();
records = getClientStatusReportingRecords();
auto res = getLastSentReportTimestamp();
}
QVector<ClientStatusReportingRecord> ClientStatusReporting::getClientStatusReportingRecords() const
{
QVector<ClientStatusReportingRecord> records;
QMutexLocker locker(&_mutex);
QSqlQuery query;
const auto prepareResult = query.prepare("SELECT * FROM clientstatusreporting");
if (!prepareResult || !query.exec()) {
const auto errorMessage = query.lastError().text();
qCDebug(lcClientStatusReporting) << "Could not get records from clientstatusreporting:" << errorMessage;
return records;
}
while (query.next()) {
ClientStatusReportingRecord record;
record._nameHash = query.value(query.record().indexOf("nHash")).toLongLong();
record._name = query.value(query.record().indexOf("name")).toByteArray();
record._numOccurences = query.value(query.record().indexOf("count")).toLongLong();
record._lastOccurence = query.value(query.record().indexOf("lastOccurrence")).toLongLong();
records.push_back(record);
}
return records;
}
bool ClientStatusReporting::deleteClientStatusReportingRecords()
{
QSqlQuery query;
const auto prepareResult = query.prepare("DELETE FROM clientstatusreporting");
if (!prepareResult || !query.exec()) {
const auto errorMessage = query.lastError().text();
qCDebug(lcClientStatusReporting) << "Could not get records from clientstatusreporting:" << errorMessage;
return false;
}
return true;
}
Result<void, QString> ClientStatusReporting::setClientStatusReportingRecord(const ClientStatusReportingRecord &record)
{
Q_ASSERT(record.isValid());
if (!record.isValid()) {
qCWarning(lcClientStatusReporting) << "Failed to set ClientStatusReportingRecord";
return {QStringLiteral("Invalid parameter")};
}
const auto recordCopy = record;
QMutexLocker locker(&_mutex);
QSqlQuery query;
const auto prepareResult = query.prepare(
"INSERT OR REPLACE INTO clientstatusreporting (nHash, name, count, lastOccurrence) VALUES(:nHash, :name, :count, :lastOccurrence) ON CONFLICT(nHash) "
"DO UPDATE SET count = count + 1, lastOccurrence = :lastOccurrence;");
query.bindValue(":nHash", recordCopy._nameHash);
query.bindValue(":name", recordCopy._name);
query.bindValue(":count", 1);
query.bindValue(":lastOccurrence", recordCopy._lastOccurence);
if (!prepareResult || !query.exec()) {
const auto errorMessage = query.lastError().text();
qCDebug(lcClientStatusReporting) << "Could not report client status:" << errorMessage;
return errorMessage;
}
return {};
}
void ClientStatusReporting::reportClientStatus(const Status status)
{
if (!_isInitialized) {
qCWarning(lcClientStatusReporting) << "Could not report status. Status reporting is not initialized";
return;
}
Q_ASSERT(status >= 0 && status < Count);
if (status < 0 || status >= Status::Count) {
qCWarning(lcClientStatusReporting) << "Trying to report invalid status:" << status;
return;
}
ClientStatusReportingRecord record;
record._name = _statusNamesAndHashes[status].first;
record._nameHash = _statusNamesAndHashes[status].second;
record._lastOccurence = QDateTime::currentDateTimeUtc().toMSecsSinceEpoch();
const auto result = setClientStatusReportingRecord(record);
if (!result.isValid()) {
qCWarning(lcClientStatusReporting) << "Could not report client status:" << result.error();
}
}
void ClientStatusReporting::sendReportToServer()
{
if (!_isInitialized) {
qCWarning(lcClientStatusReporting) << "Could not send report to server. Status reporting is not initialized";
return;
}
const auto lastSentReportTime = setLastSentReportTimestamp(0);
if (QDateTime::currentDateTimeUtc().toMSecsSinceEpoch() - lastSentReportTime < repordSendIntervalMs) {
return;
}
const auto records = getClientStatusReportingRecords();
if (!records.isEmpty()) {
// send to server ->
const auto clientStatusReportingJob = new JsonApiJob(_account->sharedFromThis(), QStringLiteral("ocs/v2.php/apps/security_guard/diagnostics"));
clientStatusReportingJob->setBody({});
clientStatusReportingJob->setVerb(SimpleApiJob::Verb::Put);
connect(clientStatusReportingJob, &JsonApiJob::jsonReceived, [this](const QJsonDocument &json) {
const QJsonObject data = json.object().value("ocs").toObject().value("data").toObject();
slotSendReportToserverFinished();
});
clientStatusReportingJob->start();
}
}
void ClientStatusReporting::slotSendReportToserverFinished()
{
if (!deleteClientStatusReportingRecords()) {
qCWarning(lcClientStatusReporting) << "Error deleting client status report.";
}
setLastSentReportTimestamp(QDateTime::currentDateTimeUtc().toMSecsSinceEpoch());
}
qulonglong ClientStatusReporting::getLastSentReportTimestamp() const
{
QMutexLocker locker(&_mutex);
QSqlQuery query;
const auto prepareResult = query.prepare("SELECT value FROM keyvalue WHERE key = (:key)");
query.bindValue(":key", lastSentReportTimestamp);
if (!prepareResult || !query.exec()) {
qCDebug(lcClientStatusReporting) << "Could not get last sent report timestamp from keyvalue table. No such record:" << lastSentReportTimestamp;
return 0;
}
if (!query.next()) {
qCDebug(lcClientStatusReporting) << "Could not get last sent report timestamp from keyvalue table:" << query.lastError().text();
return 0;
}
int valueIndex = query.record().indexOf("value");
return query.value(valueIndex).toULongLong();
}
bool ClientStatusReporting::setLastSentReportTimestamp(const qulonglong timestamp)
{
QMutexLocker locker(&_mutex);
QSqlQuery query;
const auto prepareResult = query.prepare("INSERT OR REPLACE INTO keyvalue (key, value) VALUES(:key, :value);");
query.bindValue(":key", lastSentReportTimestamp);
query.bindValue(":value", timestamp);
if (!prepareResult || !query.exec()) {
qCDebug(lcClientStatusReporting) << "Could not set last sent report timestamp from keyvalue table. No such record:" << lastSentReportTimestamp;
return false;
}
return true;
}
QByteArray ClientStatusReporting::statusStringFromNumber(const Status status)
{
Q_ASSERT(status >= 0 && status < Count);
if (status < 0 || status >= Status::Count) {
qCWarning(lcClientStatusReporting) << "Invalid status:" << status;
return {};
}
switch (status) {
case DownloadError_ConflictInvalidCharacters:
return QByteArrayLiteral("DownloadError.CONFLICT_INVALID_CHARACTERS");
case DownloadError_ConflictCaseClash:
return QByteArrayLiteral("DownloadError.CONFLICT_CASECLASH");
case UploadError_ServerError:
return QByteArrayLiteral("UploadError.SERVER_ERROR");
case Count:
return {};
};
return {};
}
}

Просмотреть файл

@ -0,0 +1,71 @@
/*
* Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#pragma once
#include "owncloudlib.h"
#include "accountfwd.h"
#include <common/result.h>
#include <QObject>
#include <QHash>
#include <QByteArray>
#include <QTimer>
#include <QtSql>
#include <QPair>
#include <QRecursiveMutex>
namespace OCC {
class Account;
struct ClientStatusReportingRecord;
class OWNCLOUDSYNC_EXPORT ClientStatusReporting : public QObject
{
Q_OBJECT
public:
enum Status {
DownloadError_ConflictInvalidCharacters = 0,
DownloadError_ConflictCaseClash,
UploadError_ServerError,
Count,
};
explicit ClientStatusReporting(Account *account, QObject *parent = nullptr);
~ClientStatusReporting() = default;
void reportClientStatus(const Status status);
void init();
private:
[[nodiscard]] Result<void, QString> setClientStatusReportingRecord(const ClientStatusReportingRecord &record);
[[nodiscard]] QVector<ClientStatusReportingRecord> getClientStatusReportingRecords() const;
[[nodiscard]] bool deleteClientStatusReportingRecords();
[[nodiscard]] bool setLastSentReportTimestamp(const qulonglong timestamp);
[[nodiscard]] qulonglong getLastSentReportTimestamp() const;
private slots:
void sendReportToServer();
void slotSendReportToserverFinished();
private:
static QByteArray statusStringFromNumber(const Status status);
Account *_account = nullptr;
QHash<int, QPair<QByteArray, qint64>> _statusNamesAndHashes;
QSqlDatabase _database;
bool _isInitialized = false;
QTimer _clientStatusReportingSendTimer;
mutable QRecursiveMutex _mutex;
};
}

Просмотреть файл

@ -0,0 +1,46 @@
/*
* Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#include "ocsclientstatusreportingjob.h"
#include "networkjobs.h"
#include "account.h"
#include <QBuffer>
#include <QJsonDocument>
namespace OCC {
OcsClientStatusReportingJob::OcsClientStatusReportingJob(AccountPtr account)
: OcsJob(account)
{
setPath(QStringLiteral("ocs/v2.php/apps/security_guard/diagnostics"));
connect(this, &OcsJob::jobFinished, this, &OcsClientStatusReportingJob::jobDone);
}
void OcsClientStatusReportingJob::sendStatusReport(const QVariant &jsonData)
{
setVerb("PUT");
addRawHeader("Ocs-APIREQUEST", "true");
addRawHeader("Content-Type", "application/json");
const auto url = Utility::concatUrlPath(account()->url(), path());
sendRequest(_verb, url, _request, QJsonDocument::fromVariant(jsonData.toMap()).toJson());
AbstractNetworkJob::start();
}
void OcsClientStatusReportingJob::jobDone(QJsonDocument reply)
{
emit jobFinished(reply, {});
}
}

Просмотреть файл

@ -0,0 +1,42 @@
/*
* Copyright (C) 2023 by Oleksandr Zolotov <alex@nextcloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#pragma once
#include "ocsjob.h"
#include <QJsonDocument>
#include <QVariant>
namespace OCC {
/**
* @brief The OcsClientStatusReportingJob class
* @ingroup gui
*
* Handle sending client status reports via OCS Diagnostics API.
*/
class OcsClientStatusReportingJob : public OcsJob
{
Q_OBJECT
public:
explicit OcsClientStatusReportingJob(AccountPtr account);
void sendStatusReport(const QVariant &jsonData);
signals:
void jobFinished(QJsonDocument reply, QVariant value);
private slots:
void jobDone(QJsonDocument reply);
};
}

Просмотреть файл

Просмотреть файл

@ -148,11 +148,13 @@ signals:
private slots:
bool finished() override;
private:
protected:
QByteArray _verb;
QHash<QString, QString> _params;
QVector<int> _passStatusCodes;
QNetworkRequest _request;
private:
QVector<int> _passStatusCodes;
QHash<QString, QString> _params;
};
}