diff --git a/lib/api/Logger.cpp b/lib/api/Logger.cpp index df97eb3f..88805193 100644 --- a/lib/api/Logger.cpp +++ b/lib/api/Logger.cpp @@ -375,9 +375,9 @@ namespace ARIASDK_NS_BEGIN } record.iKey = m_iKey; - return m_baseDecorator.decorate(record) && m_semanticContextDecorator.decorate(record) - && m_eventPropertiesDecorator.decorate(record, latency, properties); - + return m_baseDecorator.decorate(record) + && m_semanticContextDecorator.decorate(record) + && m_eventPropertiesDecorator.decorate(record, latency, properties); } void Logger::submit(::CsProtocol::Record& record, const EventProperties& props) diff --git a/lib/decorators/EventPropertiesDecorator.hpp b/lib/decorators/EventPropertiesDecorator.hpp index 0a40481b..f0fffb57 100644 --- a/lib/decorators/EventPropertiesDecorator.hpp +++ b/lib/decorators/EventPropertiesDecorator.hpp @@ -15,10 +15,58 @@ namespace ARIASDK_NS_BEGIN { class EventPropertiesDecorator : public DecoratorBase { + std::string randomLocalId; public: EventPropertiesDecorator(ILogManager& owner) : - DecoratorBase(owner) {}; + DecoratorBase(owner) + { + // + // Random local deviceId must be used for events tagged with Pii DROP EventTag. + // Since we generate a new random id for every SDK session - products tagging + // their telemetry events with Pii DROP tag cannot use ext.device.localId field + // in Kusto for DAU/MAU user engagement metrics. + // + // It would have been more logical to adjust the Pii flags spec to require all + // devices running with Pii DROP flag to the same identical bogus {deadbeef-...} + // id. That would have allowed the apps to continue using ext.device.localId for + // engagement metrics, estimating their population size using that field. + // + randomLocalId = "r:"; + randomLocalId+= PAL::generateUuidString(); + }; + + void dropPiiPartA(::CsProtocol::Record& record) + { + // MICROSOFT_EVENTTAG_DROP_PII tag functionality reference: + // https://osgwiki.com/wiki/Telemetry#De-Identification_of_Telemetry_Events + // + // Drop Pii EventTag scrubs Part A Pii data client-side. + // The flag has no effect on Part C Pii data. + // + + // clear tickets because these provide a way to identify the end-user + record.extProtocol[0].ticketKeys.clear(); + + // clean Pii from Device extension + record.extDevice[0].localId = randomLocalId; + record.extDevice[0].authId.clear(); + record.extDevice[0].authSecId.clear(); + record.extDevice[0].id.clear(); + + // clean Pii from User extension + record.extUser[0].localId.clear(); + record.extUser[0].authId.clear(); + record.extUser[0].id.clear(); + + // clean epoch and seq + installId + record.extSdk[0].seq = 0; + record.extSdk[0].epoch.clear(); + record.extSdk[0].installId.clear(); + + // clear correlation vector + record.cV = ""; + } bool decorate(::CsProtocol::Record& record, EventLatency& latency, EventProperties const& eventProperties) { @@ -50,6 +98,10 @@ namespace ARIASDK_NS_BEGIN { record.popSample = eventProperties.GetPopSample(); int64_t flags = eventProperties.GetPolicyBitFlags(); + + // Caller asked us to drop Pii from Part A of that event + bool tagDropPii = bool(flags & MICROSOFT_EVENTTAG_DROP_PII); + // Pack flags the same way as Win 10 UTC is doing this flags = (flags & 0xffff) | ((flags & 0xffffffffffff0000) >> 8); @@ -344,30 +396,36 @@ namespace ARIASDK_NS_BEGIN { } } - if (extPartB.size() > 0) + if (extPartB.size() > 0) + { + ::CsProtocol::Data partBdata; + partBdata.properties = extPartB; + record.baseData.push_back(partBdata); + } + + // special case of CorrelationVector value + if (ext.count(CorrelationVector::PropertyName) > 0) + { + CsProtocol::Value cvValue = ext[CorrelationVector::PropertyName]; + + if (cvValue.type == ::CsProtocol::ValueKind::ValueString) { - ::CsProtocol::Data partBdata; - partBdata.properties = extPartB; - record.baseData.push_back(partBdata); + record.cV = cvValue.stringValue; } - // special case of CorrelationVector value - if (ext.count(CorrelationVector::PropertyName) > 0) + else { - CsProtocol::Value cvValue = ext[CorrelationVector::PropertyName]; - - if (cvValue.type == ::CsProtocol::ValueKind::ValueString) - { - record.cV = cvValue.stringValue; - } - else - { - LOG_TRACE("CorrelationVector value type is invalid %u", cvValue.type); - } - + LOG_TRACE("CorrelationVector value type is invalid %u", cvValue.type); + } ext.erase(CorrelationVector::PropertyName); - } + } - return true; + // scrub if MICROSOFT_EVENTTAG_DROP_PII is set + if (tagDropPii) + { + dropPiiPartA(record); + } + + return true; } }; diff --git a/tests/functests/APITest.cpp b/tests/functests/APITest.cpp index 0871accb..e4a788a4 100644 --- a/tests/functests/APITest.cpp +++ b/tests/functests/APITest.cpp @@ -1,4 +1,3 @@ -// #ifdef _WIN32 #ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS #endif @@ -26,11 +25,15 @@ #include "json.hpp" #endif +#include "CorrelationVector.hpp" + using namespace MAT; LOGMANAGER_INSTANCE -#define TEST_TOKEN "6d084bbf6a9644ef83f40a77c9e34580-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322" +// 1DSCppSdkTest sandbox key +#define TEST_TOKEN "7c8b1796cbc44bd5a03803c01c2b9d61-b6e370dd-28d9-4a52-9556-762543cf7aa7-6991" + #define KILLED_TOKEN "deadbeefdeadbeefdeadbeefdeadbeef-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322" #define TEST_TOKEN2 "0ae6cd22d8264818933f4857dd3c1472-eea5f30e-e0ed-4ab0-8ed0-4dc0f5e156e0-7385" @@ -669,7 +672,7 @@ TEST(APITest, C_API_Test) }, "name" : "C-API-Client-0", "version" : "1.0.0", - "primaryToken" : "6d084bbf6a9644ef83f40a77c9e34580-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322", + "primaryToken" : "7c8b1796cbc44bd5a03803c01c2b9d61-b6e370dd-28d9-4a52-9556-762543cf7aa7-6991", "maxTeardownUploadTimeInSec" : 5, "hostMode" : false, "minimumTraceLevel" : 0, @@ -844,6 +847,84 @@ TEST(APITest, UTC_Callback_Test) LogManager::FlushAndTeardown(); LogManager::RemoveEventListener(EVT_LOG_EVENT, debugListener); } + +TEST(APITest, Pii_DROP_Test) +{ + TestDebugEventListener debugListener; + + auto config = LogManager::GetLogConfiguration(); + config[CFG_INT_SDK_MODE] = SdkModeTypes::SdkModeTypes_CS; + config["stats"]["interval"] = 0; // avoid sending stats for this test + config[CFG_INT_MAX_TEARDOWN_TIME] = 1; // give enough time to upload + + // register a listener + LogManager::AddEventListener(EVT_LOG_EVENT, debugListener); + auto logger = LogManager::Initialize(TEST_TOKEN); + unsigned totalEvents = 0; + std::string realDeviceId; + + // verify that we get one regular event with real device id, + // as well as more events with anonymous random device id. + debugListener.OnLogX = [&](::CsProtocol::Record & record) + { + totalEvents++; + if (record.name == "Regular.Event") + { + // usual event with proper SDK-obtained localId + realDeviceId = record.extDevice[0].localId; + EXPECT_STREQ(record.extUser[0].localId.c_str(), "c:1234567890"); + return; + } + + ASSERT_EQ(record.extProtocol[0].ticketKeys.size(), 0); + // more events with random device id + EXPECT_STRNE(record.extDevice[0].localId.c_str(), realDeviceId.c_str()); + EXPECT_STREQ(record.extDevice[0].authId.c_str(), ""); + EXPECT_STREQ(record.extDevice[0].authSecId.c_str(), ""); + EXPECT_STREQ(record.extDevice[0].id.c_str(), ""); + // ext.user.localId stripped + EXPECT_STREQ(record.extUser[0].localId.c_str(), ""); + EXPECT_STREQ(record.extUser[0].authId.c_str(), ""); + EXPECT_STREQ(record.extUser[0].id.c_str(), ""); + // SDK tracking cookies stripped + EXPECT_EQ(record.extSdk[0].seq, 0); + EXPECT_STREQ(record.extSdk[0].epoch.c_str(), ""); + EXPECT_STREQ(record.extSdk[0].installId.c_str(), ""); + // cV stripped + EXPECT_STREQ(record.cV.c_str(), ""); + }; + + auto context = logger->GetSemanticContext(); + context->SetUserId("c:1234567890"); + + // Set some random cV + CorrelationVector m_appCV; + m_appCV.SetValue("jj9XLhDw7EuXoC2L"); + // Extend that value. + m_appCV.Extend(); + // Get the next value, log it and/or pass it to your downstream dependency. + std::string curCV = m_appCV.GetNextValue(); + + logger->LogEvent("Regular.Event"); + + for (size_t i = 0; i < 3; i++) + { + EventProperties event("PiiDrop.Event", + { + { "strKey", "some string" } + }); + + event.SetPolicyBitFlags(MICROSOFT_EVENTTAG_DROP_PII); + event.SetProperty(CorrelationVector::PropertyName, curCV); + logger->LogEvent(event); + } + + LogManager::FlushAndTeardown(); + ASSERT_EQ(totalEvents, 4); + LogManager::RemoveEventListener(EVT_LOG_EVENT, debugListener); + +} + #endif static void logBenchMark(const char * label) @@ -1134,7 +1215,6 @@ TEST(APITest, Pii_Kind_E2E_Test) // Verify that contents get hashed by server } -// #endif #endif // HAVE_MAT_DEFAULT_HTTP_CLIENT // TEST_PULL_ME_IN(APITest) diff --git a/tests/functests/BasicFuncTests.cpp b/tests/functests/BasicFuncTests.cpp index 639253b7..65e47245 100644 --- a/tests/functests/BasicFuncTests.cpp +++ b/tests/functests/BasicFuncTests.cpp @@ -34,7 +34,9 @@ using namespace MAT; char const* const TEST_STORAGE_FILENAME = "BasicFuncTests.db"; -#define TEST_TOKEN "6d084bbf6a9644ef83f40a77c9e34580-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322" +// 1DSCppSdktest sandbox key +#define TEST_TOKEN "7c8b1796cbc44bd5a03803c01c2b9d61-b6e370dd-28d9-4a52-9556-762543cf7aa7-6991" + #define KILLED_TOKEN "deadbeefdeadbeefdeadbeefdeadbeef-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322" #define HTTP_PORT 19000 diff --git a/tests/functests/BondDecoderTests.cpp b/tests/functests/BondDecoderTests.cpp index f6a27beb..85ae7ad5 100644 --- a/tests/functests/BondDecoderTests.cpp +++ b/tests/functests/BondDecoderTests.cpp @@ -20,7 +20,8 @@ using namespace MAT; using namespace std; -#define TOKEN "6d084bbf6a9644ef83f40a77c9e34580-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322" +// 1DSCppSdkTest sandbox key +#define TOKEN "7c8b1796cbc44bd5a03803c01c2b9d61-b6e370dd-28d9-4a52-9556-762543cf7aa7-6991" void Configure(ILogConfiguration& config) { diff --git a/wrappers/go/main.go b/wrappers/go/main.go index dac97200..f66e35ca 100644 --- a/wrappers/go/main.go +++ b/wrappers/go/main.go @@ -48,7 +48,8 @@ func main() { // trap SIGABRT and SIGTERM registerSignalHandler() - token := "6d084bbf6a9644ef83f40a77c9e34580-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322" + // 1DSCppSdkTest sandbox key + token := "7c8b1796cbc44bd5a03803c01c2b9d61-b6e370dd-28d9-4a52-9556-762543cf7aa7-6991" fmt.Println("Hello from Microsoft Telemetry-Go!\n") logger := EventLogger.NewEventLogger() diff --git a/wrappers/obj-c/main.mm b/wrappers/obj-c/main.mm index ed73d7fc..25575232 100644 --- a/wrappers/obj-c/main.mm +++ b/wrappers/obj-c/main.mm @@ -5,7 +5,8 @@ int main(int argc, char** argv){ @autoreleasepool{ - NSString* token = @"6d084bbf6a9644ef83f40a77c9e34580-c2d379e0-4408-4325-9b4d-2a7d78131e14-7322"; + // 1DSCppSdkTest sandbox key. Replace with your own iKey! + NSString* token = @"7c8b1796cbc44bd5a03803c01c2b9d61-b6e370dd-28d9-4a52-9556-762543cf7aa7-6991"; WLogger* myLogger = [WLogManager initForTenant: token]; if(myLogger){