From 5aa9d43c870094f41efffe404b675840cbff2411 Mon Sep 17 00:00:00 2001 From: mogiligarimidi23 Date: Tue, 23 Jun 2026 00:22:16 +0530 Subject: [PATCH 01/30] Surface per-batch record count (param2) in HTTP response debug events (#1487) --- lib/http/HttpResponseDecoder.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/http/HttpResponseDecoder.cpp b/lib/http/HttpResponseDecoder.cpp index 11e9d4096..6014cec19 100644 --- a/lib/http/HttpResponseDecoder.cpp +++ b/lib/http/HttpResponseDecoder.cpp @@ -91,6 +91,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_OK; evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); evt.data = static_cast(request.GetBody().data()); evt.size = request.GetBody().size(); DispatchEvent(evt); @@ -112,6 +113,7 @@ namespace MAT_NS_BEGIN { // This is to be addressed with ETW trace API that can send // a detailed error context to ETW provider. evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); evt.data = static_cast(request.GetBody().data()); evt.size = request.GetBody().size(); DispatchEvent(evt); @@ -127,6 +129,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_FAILURE; evt.param1 = 0; // response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); DispatchEvent(evt); } ctx->httpResponse = nullptr; @@ -144,6 +147,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_FAILURE; evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); DispatchEvent(evt); } temporaryServerFailure(ctx); @@ -157,6 +161,7 @@ namespace MAT_NS_BEGIN { DebugEvent evt; evt.type = DebugEventType::EVT_HTTP_FAILURE; evt.param1 = response.GetStatusCode(); + evt.param2 = ctx->recordIdsAndTenantIds.size(); DispatchEvent(evt); } temporaryNetworkFailure(ctx); @@ -253,4 +258,3 @@ namespace MAT_NS_BEGIN { } } MAT_NS_END - From 62cef16907d2cf3070b0f72cc052dc2cea463ba7 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Mon, 22 Jun 2026 16:23:08 -0500 Subject: [PATCH 02/30] Fix GUID_t::operator< to be a valid strict-weak-ordering (#1490) GUID_t::operator< used a non-lexicographic chained-|| comparison with a `Data3 == other.Data3` typo (should be `<`): return Data1 < other.Data1 || Data2 < other.Data2 || Data3 == other.Data3 || (memcmp(Data4,...) < 0); Two defects: (1) the Data3 line uses == instead of <, so GUIDs differing only in Data3 compare equivalent; (2) the chained-|| form is not lexicographic and violates antisymmetry (e.g. {1,5,..} and {2,3,..} can report both a / std::map (the default std::less calls it; the comment says it is "needed for maps") is undefined behavior -- container corruption, infinite loops, or crashes. GUID_t is a public header type (EventProperty.hpp), so SDK consumers hit this with default ordered containers. Replace with proper lexicographic comparison (Data1, then Data2, then Data3, then memcmp(Data4)). Test: GuidTests.OperatorLess_IsStrictWeakOrdering (added) fails on the old operator (c --- lib/system/EventProperty.cpp | 11 +++++++---- tests/unittests/GuidTests.cpp | 23 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/system/EventProperty.cpp b/lib/system/EventProperty.cpp index 3bf9c3f3c..6d0582440 100644 --- a/lib/system/EventProperty.cpp +++ b/lib/system/EventProperty.cpp @@ -307,10 +307,13 @@ namespace MAT_NS_BEGIN { // How to sort 2 objects (needed for maps) bool GUID_t::operator<(GUID_t const& other) const { - return Data1 < other.Data1 || - Data2 < other.Data2 || - Data3 == other.Data3 || - (memcmp(Data4, other.Data4, sizeof(Data4)) < 0); + if (Data1 != other.Data1) + return Data1 < other.Data1; + if (Data2 != other.Data2) + return Data2 < other.Data2; + if (Data3 != other.Data3) + return Data3 < other.Data3; + return memcmp(Data4, other.Data4, sizeof(Data4)) < 0; } void EventProperty::copydata(EventProperty const* source) diff --git a/tests/unittests/GuidTests.cpp b/tests/unittests/GuidTests.cpp index a0fc9fdb5..4a45d880c 100644 --- a/tests/unittests/GuidTests.cpp +++ b/tests/unittests/GuidTests.cpp @@ -7,6 +7,8 @@ #include "utils/Utils.hpp" #include "EventProperties.hpp" +#include + using namespace testing; using namespace MAT; @@ -84,4 +86,23 @@ TEST(GuidTests, MoveAssignment_ValidInput_MovesCorrectly) GUID_t second{"BEE391C8-72B0-464F-93C3-1B27879AD103"}; second = std::move(first); ASSERT_EQ("9D016D64-372E-4DCE-9FA3-0D0772217C54", second.to_string()); -} \ No newline at end of file +} +TEST(GuidTests, OperatorLess_IsStrictWeakOrdering) +{ + // a and b differ in Data1/Data2 such that a non-lexicographic chained-|| operator + // reported BOTH a < b and b < a (antisymmetry violation). + GUID_t a{ "00000001-0005-0000-0000-000000000000" }; + GUID_t b{ "00000002-0003-0000-0000-000000000000" }; + EXPECT_TRUE(a < b); + EXPECT_FALSE(b < a); + + // c and d differ ONLY in Data3; a '==' in that position made them compare equivalent. + GUID_t c{ "00000001-0001-0001-0000-000000000000" }; + GUID_t d{ "00000001-0001-0002-0000-000000000000" }; + EXPECT_TRUE(c < d); + EXPECT_FALSE(d < c); + + // A std::set keyed on operator< must keep four distinct GUIDs distinct. + std::set s{ a, b, c, d }; + EXPECT_EQ(static_cast(4), s.size()); +} From beba0e8e440e97f20f50e5fb51b2c9cc76cf19dc Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Mon, 22 Jun 2026 16:45:10 -0500 Subject: [PATCH 03/30] Fix inverted oneds_memcpy_s result in CompactBinaryProtocolReader::ReadBlob (#1492) Under USE_ONEDS_BOUNDCHECK_METHODS, ReadBlob assigned the errno_t return of MAT::BoundCheckFunctions::oneds_memcpy_s (0 == success) directly to a bool: bool result = MAT::BoundCheckFunctions::oneds_memcpy_s(...); so `result` is false on success and true on failure -- inverted. ReadBlob (and thus ReadFloat/ReadDouble, which call it) returns "failure" for every successful read, breaking decode of any payload containing a blob/float/double field whenever the SDK is built with bound-check methods. The non-boundcheck branch one line below already does it correctly: `bool result = (memcpy_s(...) == 0);`. Compare the result against 0, matching the memcpy_s branch. Validated (TDD) with a standalone reader round-trip compiled with -DHAVE_ONEDS_BOUNDCHECK_METHODS: ReadDouble of a written double returns false before the fix and true (correct value) after. The bug is gated behind the non-default USE_ONEDS_BOUNDCHECK_METHODS option; the default build is unaffected (the changed line is #ifdef'd out) and the existing bondlite CompactBinaryProtocolTests assert ReadBlob == true under that option. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/bond/CompactBinaryProtocolReader.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bond/CompactBinaryProtocolReader.hpp b/lib/bond/CompactBinaryProtocolReader.hpp index 612970421..708aba421 100644 --- a/lib/bond/CompactBinaryProtocolReader.hpp +++ b/lib/bond/CompactBinaryProtocolReader.hpp @@ -69,7 +69,7 @@ class CompactBinaryProtocolReader { return false; } #ifdef HAVE_ONEDS_BOUNDCHECK_METHODS - bool result = MAT::BoundCheckFunctions::oneds_memcpy_s(static_cast(data), size, &(m_input[m_ofs]), size); + bool result = (MAT::BoundCheckFunctions::oneds_memcpy_s(static_cast(data), size, &(m_input[m_ofs]), size) == 0); #else bool result = (memcpy_s(static_cast(data), size, &(m_input[m_ofs]), size) == 0); #endif From c093ac17d0820024dbf27c30b6e5fba9d560125f Mon Sep 17 00:00:00 2001 From: Max Golovanov Date: Tue, 23 Jun 2026 12:37:03 -0700 Subject: [PATCH 04/30] New flag to scrub IP addresses (#1161) * Update EventPropertiesDecorator.hpp Scrub IP addresses by default. * Gate IP scrubbing behind CFG_BOOL_ENABLE_IP_SCRUBBING (default on) The initial change set RECORD_FLAGS_EVENTTAG_SCRUB_IP unconditionally for every event, forcing IP scrubbing on all SDK consumers in direct-upload mode -- a breaking change for apps that need client IP (e.g. geo-location). Make scrubbing the default but opt-out: the decorator sets the SCRUB_IP record flag unless CFG_BOOL_ENABLE_IP_SCRUBBING is explicitly set to false. record.flags is forwarded on the cross-platform/direct-upload path, so this redacts client IP at the collector without relying on ext.metadata privacy tags. - ILogConfiguration.hpp: add CFG_BOOL_ENABLE_IP_SCRUBBING config key - EventPropertiesDecorator.hpp: gate SCRUB_IP flag behind the config - EventPropertiesDecoratorTests.cpp: add default-on, opt-out, explicit-enable tests with a per-instance ConfigurableLogManager helper Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Cover SDK stats events and add wrapper-parity config keys for IP scrubbing - Statistics.cpp: SDK statistics/metastats events bypass EventPropertiesDecorator, so apply the same collector-side client-IP scrub (gated by CFG_BOOL_ENABLE_IP_SCRUBBING, on by default) to those records too. Closes the gap identified in PR review. - LogConfigurationKey.java + ODWLogConfiguration.{h,mm}: expose CFG_BOOL_ENABLE_IP_SCRUBBING ('enableIpScrubbing') to the Android (Java) and Apple (Obj-C) wrappers for API parity. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Clarify CFG_BOOL_ENABLE_IP_SCRUBBING docstring (Copilot round 1) The doc implied the setting only applies in direct-upload mode, but the scrub flag is set for all events and modes. Clarify that the flag is honored by the OneCollector direct-upload path while UTC mode applies its own client-privacy handling. No behavior change -- the flag is intentionally mode-agnostic and is ignored by the UTC pipeline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot round 2 on #1161: self-contained header + default-config docs - EventPropertiesDecorator.hpp: include ILogManager.hpp so the header is self-contained for the ILogManager (m_owner) and ILogConfiguration types / CFG_BOOL_ENABLE_IP_SCRUBBING used inline, instead of relying on the includer. - Clarify CFG_BOOL_ENABLE_IP_SCRUBBING docs (C++ / Java / Obj-C): scrubbing is applied unless explicitly set to false (on by default) and the key is not present in the default configuration, so GetDefaultConfiguration() does not surface it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot round 3 on #1161: extract on-wire record flags to a shared header Move the RECORD_FLAGS_EVENTTAG_* on-wire bits out of EventPropertiesDecorator.hpp into a dedicated decorators/RecordFlagConstants.hpp, exposed as static constexpr std::int64_t in the MAT namespace (no longer #define macros). This lets the stats pipeline reference RECORD_FLAGS_EVENTTAG_SCRUB_IP via the small shared header instead of pulling in the full decorator header, and avoids macro pollution. - New: lib/decorators/RecordFlagConstants.hpp - EventPropertiesDecorator.hpp: include the shared header; drop the macros - Statistics.cpp: include the shared header instead of EventPropertiesDecorator.hpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Harden OfflineStorage_Room against null JNI objects (no-crash on null m_room/element) Defensive follow-up to the #1227 family of Android Room crashes ("java_object == null in call to GetObjectClass" under GetAndReserveRecords). #1417 fixed the stale-local-ref root cause but left two null paths unguarded, which can still hard-abort the host process: * m_room is null when the Room DB failed to open or was torn down (the destructor already guards with `if (s_vm && m_room)`, but ~10 other methods dereferenced it unconditionally). Add `if (!m_room) return ;` guards to DeleteRecords(x2), GetAndReserveRecords, ReleaseRecords, StoreRecords, DeleteSetting, StoreSetting, GetSizeInternal, GetRecordCount, ResizeDbInternal, and GetRecords. GetSetting already had this guard. * a null element in the getAndReserve/releaseRecords result array (observed to be androidx.room-version sensitive) was passed to GetObjectClass. Guard both loops: GetAndReserveRecords pops the frame and stops (the existing index < limit path releases the rest for retry); ReleaseRecords skips it. Turns a process-killing JNI abort into graceful degradation. No functional change on the healthy path. Compiles on Android only (JNI/Room) -- validated by CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * docs: note the androidx.room version the SDK is built against (#1227 follow-up) cpp-start-android.md covered Room setup but said nothing about the Room version. The GetAndReserveRecords native crash (#1227, Room-version-sensitive) and the null-guard hardening in this PR make this worth documenting: the maesdk AAR brings androidx.room transitively (pinned in maesdk/build.gradle, currently 2.8.4); since Gradle resolves one Room version app-wide, consumers should not force a version below what the SDK is built against and should prefer aligning on the bundled (or a compatible newer) version. Points at build.gradle as the source of truth so the doc doesn't drift on future bumps. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Room: guard null JNIEnv in 5 JNI methods; skip releaseUnconsumed on null element Address the latest Copilot review (6 comments) on lib/offline/OfflineStorage_Room.cpp. ConnectedEnv null-env guards (5): DeleteByToken, ReleaseRecords, DeleteSetting, StoreSetting and GetRecords created ConnectedEnv env(s_vm) and dereferenced env->... behind only an if(!m_room) guard. ConnectedEnv::operator! can report a null JNIEnv (null s_vm / thread-attach failure), and sibling methods already guard with if(!env); added the matching early return (void/false/records) so a null env no longer crashes. releaseUnconsumed on null element: the null-array-element path broke out and fell through to releaseUnconsumed(selected, index). The Java StorageRecordDao.releaseUnconsumed (pre-existing since 2020, commit c81d46aa) reads selected[0..unconsumed-1] ignoring the offset, so on the null path it could index the null element and throw, or release the wrong rows. A sawNullElement flag now skips releaseUnconsumed on that path; the reserved records expire and are retried (no data loss), and the normal end-early path is unchanged. Validated: NDK aarch64-linux-android23 -fsyntax-only clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Lalit Kumar Bhasin Co-authored-by: bmehta001 Co-authored-by: Bhagirath Mehta Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/cpp-start-android.md | 2 + .../events/LogConfigurationKey.java | 3 + lib/decorators/EventPropertiesDecorator.hpp | 19 ++-- lib/decorators/RecordFlagConstants.hpp | 30 ++++++ lib/include/public/ILogConfiguration.hpp | 10 ++ lib/offline/OfflineStorage_Room.cpp | 98 ++++++++++++++++++- lib/stats/Statistics.cpp | 8 ++ .../EventPropertiesDecoratorTests.cpp | 51 ++++++++++ wrappers/obj-c/ODWLogConfiguration.h | 5 + wrappers/obj-c/ODWLogConfiguration.mm | 5 + 10 files changed, 217 insertions(+), 14 deletions(-) create mode 100644 lib/decorators/RecordFlagConstants.hpp diff --git a/docs/cpp-start-android.md b/docs/cpp-start-android.md index 0f979bcda..2281a5e31 100644 --- a/docs/cpp-start-android.md +++ b/docs/cpp-start-android.md @@ -14,6 +14,8 @@ The Gradle wrapper in ```android_build``` builds two modules, ```app``` and ```m On Android, there are two database implementations to choose from. By default (the main branch on Github), the SDK will use the Android-supported androidx.Room database package. This reduces APK size because we don't need to compile and link in a copy of SQLite in native code (SQLite is hundreds of kB per ABI of APK file size). Room does have a slight CPU performance disadvantage since database transactions cross the JNI boundary when native code uses it. If you wish to change from Room to the native SQLite implementation, you should change the two module ```build.gradle``` files (app and maesdk). In those files, you will see an argument to CMake to select Room: ```"-DUSE_ROOM=1"```. Change this to ```"-DUSE_ROOM=0``` to select the native SQLite. +When using the Room implementation, the ```maesdk``` AAR brings ```androidx.room``` as a transitive dependency, pinned in ```lib/android_build/maesdk/build.gradle``` (currently ```2.8.4```). The SDK's native (JNI) code is compiled and tested against this version and the Room-generated schema. Because Gradle resolves a single ```androidx.room``` version for the entire app, if your app (or one of its dependencies) selects a different version, the SDK's native code runs against it. **Do not force ```androidx.room``` below the version the SDK is built against**, and prefer aligning your app on the bundled version (or a compatible newer one). A significantly different Room version can change the shape of query results that cross the JNI boundary and has historically caused native crashes in record retrieval (issue #1227); the SDK now guards against null results defensively, but version alignment avoids subtle behavior differences. + The Room database implementation adds one additional initialization requirement, since it needs a pointer to the JVM and an object reference to the application context. See below (4.5) for the required call to either ```connectContext``` (in Java) or ```ConnectJVM``` (in C++) to set this up. If you are building on Windows, this helper script [build-android.cmd](../build-android.cmd) is provided to illustrate how to deploy the necessary SDK and NDK dependencies. Once you installed the necessary dependencies, you may use Android Studio IDE for local builds. See [ide.cmd](../lib/android_build/ide.cmd) that shows how to build the project from IDE. The `app` project (`maesdktest`) allows to build and run all SDK tests on either emulator or real Android device. While the tests are running, you can monitor the test results in logcat output. diff --git a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java index 329f17680..0ce2881ef 100644 --- a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java +++ b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/LogConfigurationKey.java @@ -29,6 +29,9 @@ public enum LogConfigurationKey { /** Enable network detector. */ CFG_BOOL_ENABLE_NET_DETECT("enableNetworkDetector", Boolean.class), + /** Scrub (obfuscate) the client IP address at the collector. Applied unless explicitly set to false (on by default; not present in the default configuration). */ + CFG_BOOL_ENABLE_IP_SCRUBBING("enableIpScrubbing", Boolean.class), + CFG_BOOL_TPM_CLOCK_SKEW_ENABLED("clockSkewEnabled", Boolean.class), /** Parameter that allows to check if the SDK is running on UTC mode */ diff --git a/lib/decorators/EventPropertiesDecorator.hpp b/lib/decorators/EventPropertiesDecorator.hpp index 5bb3e927a..800dc1635 100644 --- a/lib/decorators/EventPropertiesDecorator.hpp +++ b/lib/decorators/EventPropertiesDecorator.hpp @@ -6,6 +6,8 @@ #define EVENTPROPERTIESDECORATOR_HPP #include "IDecorator.hpp" +#include "ILogManager.hpp" +#include "RecordFlagConstants.hpp" #include "EventProperties.hpp" #include "CorrelationVector.hpp" #include "utils/Utils.hpp" @@ -16,15 +18,6 @@ namespace MAT_NS_BEGIN { -// Bit remapping has to happen on bits passed via API surface. -// Ref CS2.1+ : https://osgwiki.com/wiki/CommonSchema/flags -// #define MICROSOFT_EVENTTAG_MARK_PII 0x08000000 -#define RECORD_FLAGS_EVENTTAG_MARK_PII 0x00080000 -// #define MICROSOFT_EVENTTAG_HASH_PII 0x04000000 -#define RECORD_FLAGS_EVENTTAG_HASH_PII 0x00100000 -// #define MICROSOFT_EVENTTAG_DROP_PII 0x02000000 -#define RECORD_FLAGS_EVENTTAG_DROP_PII 0x00200000 - class EventPropertiesDecorator : public IDecorator { protected: @@ -125,6 +118,14 @@ namespace MAT_NS_BEGIN { int64_t tags = eventProperties.GetPolicyBitFlags(); int64_t flags = 0; + // Scrub/obfuscate the client IP address at the collector by default. + // Hosts that require the client IP (e.g. for geo-location enrichment) + // can opt out by setting CFG_BOOL_ENABLE_IP_SCRUBBING = false. + ILogConfiguration& config = m_owner.GetLogConfiguration(); + if (!config.HasConfig(CFG_BOOL_ENABLE_IP_SCRUBBING) || config[CFG_BOOL_ENABLE_IP_SCRUBBING]) + { + flags |= RECORD_FLAGS_EVENTTAG_SCRUB_IP; + } // We must remap from one bitfield set to another, no way to bit-shift :( // At the moment 1DS SDK in direct upload mode supports DROP and MARK tags only: flags |= (tags & MICROSOFT_EVENTTAG_MARK_PII) ? RECORD_FLAGS_EVENTTAG_MARK_PII : 0; diff --git a/lib/decorators/RecordFlagConstants.hpp b/lib/decorators/RecordFlagConstants.hpp new file mode 100644 index 000000000..8c0fe56d3 --- /dev/null +++ b/lib/decorators/RecordFlagConstants.hpp @@ -0,0 +1,30 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +#ifndef RECORDFLAGCONSTANTS_HPP +#define RECORDFLAGCONSTANTS_HPP + +#include "ctmacros.hpp" + +#include + +namespace MAT_NS_BEGIN { + + // On-wire CS protocol record.flags bits. These are distinct from the + // API-surface MICROSOFT_EVENTTAG_* policy flags and are remapped onto + // record.flags by EventPropertiesDecorator. Kept in a dedicated header so + // other components (e.g. the stats pipeline) can reference a single bit + // definition without depending on the decorator's inline implementation. + // Ref CS2.1+: https://osgwiki.com/wiki/CommonSchema/flags + // (API-surface MICROSOFT_EVENTTAG_MARK_PII 0x08000000) + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_MARK_PII = 0x00080000; + // (API-surface MICROSOFT_EVENTTAG_HASH_PII 0x04000000) + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_HASH_PII = 0x00100000; + // (API-surface MICROSOFT_EVENTTAG_DROP_PII 0x02000000) + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_DROP_PII = 0x00200000; + static constexpr std::int64_t RECORD_FLAGS_EVENTTAG_SCRUB_IP = 0x00400000; + +} MAT_NS_END + +#endif // RECORDFLAGCONSTANTS_HPP diff --git a/lib/include/public/ILogConfiguration.hpp b/lib/include/public/ILogConfiguration.hpp index 1cb8103b8..af1bc44c2 100644 --- a/lib/include/public/ILogConfiguration.hpp +++ b/lib/include/public/ILogConfiguration.hpp @@ -104,6 +104,16 @@ namespace MAT_NS_BEGIN /// static constexpr const char* const CFG_BOOL_ENABLE_NET_DETECT = "enableNetworkDetector"; + /// + /// Request collector-side scrubbing (obfuscation) of the client IP address. + /// Applied unless explicitly set to false (on by default; the key is not + /// present in the default configuration). Opt out when the client IP is + /// needed, e.g. for geo-location enrichment. Honored by the OneCollector + /// direct-upload path; in UTC mode client privacy is governed by the OS UTC + /// pipeline instead. + /// + static constexpr const char* const CFG_BOOL_ENABLE_IP_SCRUBBING = "enableIpScrubbing"; + /// /// Parameter that allows to check if the SDK is running on UTC mode /// diff --git a/lib/offline/OfflineStorage_Room.cpp b/lib/offline/OfflineStorage_Room.cpp index 72a04d0ed..d052e7a9d 100644 --- a/lib/offline/OfflineStorage_Room.cpp +++ b/lib/offline/OfflineStorage_Room.cpp @@ -240,6 +240,14 @@ namespace MAT_NS_BEGIN MATSDK_THROW(std::logic_error("whereFilter not implemented")); } + if (!env) + { + return; + } + if (!m_room) + { + return; + } auto room_class = env->GetObjectClass(m_room); auto deleteByToken = env->GetMethodID(room_class, "deleteByToken", @@ -274,6 +282,10 @@ namespace MAT_NS_BEGIN { return; } + if (!m_room) + { + return; + } auto room_class = env->GetObjectClass(m_room); auto method = env->GetMethodID(room_class, "deleteById", "([J)J"); ThrowLogic(env, "Unable to get deleteById method"); @@ -377,6 +389,10 @@ namespace MAT_NS_BEGIN { return false; } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); auto reserve = env->GetMethodID(room_class, "getAndReserve", "(IJJJ)[Lcom/microsoft/applications/events/StorageRecord;"); @@ -424,11 +440,26 @@ namespace MAT_NS_BEGIN int persist_lb = static_cast(EventPersistence_Normal); int persist_ub = static_cast(EventPersistence_DoNotStoreOnDisk); + // Set if a null array element is hit below, so the early-release + // path skips releaseUnconsumed (which would index into the null). + bool sawNullElement = false; for (index = 0; index < limit; ++index) { env.pushLocalFrame(32); auto record = env->GetObjectArrayElement(selected, index); ThrowLogic(env, "getAndReserve element"); + if (!record) + { + // Null array element (observed with some androidx.room + // versions): pop this frame and stop rather than + // dereferencing null in GetObjectClass. We cannot safely + // release the tail here (it contains this null and Java + // releaseUnconsumed indexes from 0), so leave the + // remaining reservations to expire and be retried. + sawNullElement = true; + env.popLocalFrame(); + break; + } if (!record_class) { // Promote to a global ref so it survives popLocalFrame on @@ -518,11 +549,14 @@ namespace MAT_NS_BEGIN if (index < limit) { // we did not consume all these events - auto release = env->GetMethodID(room_class, "releaseUnconsumed", - "([Lcom/microsoft/applications/events/StorageRecord;I)V"); - ThrowLogic(env, "releaseUnconsumed"); - env->CallVoidMethod(m_room, release, selected, static_cast(index)); - ThrowRuntime(env, "call ru"); + if (!sawNullElement) + { + auto release = env->GetMethodID(room_class, "releaseUnconsumed", + "([Lcom/microsoft/applications/events/StorageRecord;I)V"); + ThrowLogic(env, "releaseUnconsumed"); + env->CallVoidMethod(m_room, release, selected, static_cast(index)); + ThrowRuntime(env, "call ru"); + } break; // break out of the request > collected loop--end early by request } } @@ -633,6 +667,14 @@ namespace MAT_NS_BEGIN try { ConnectedEnv env(s_vm); + if (!env) + { + return; + } + if (!m_room) + { + return; + } auto room_class = env->GetObjectClass(m_room); ThrowLogic(env, "GetObjectClass for m_room"); auto release = env->GetMethodID(room_class, @@ -700,6 +742,12 @@ namespace MAT_NS_BEGIN env.pushLocalFrame(8); auto byTenant = env->GetObjectArrayElement(results, index); ThrowRuntime(env, "Exception fetching element from results"); + if (!byTenant) + { + // Skip a null array element rather than dereference null. + env.popLocalFrame(); + continue; + } if (!bt_class) { // Promote to a global ref so it survives popLocalFrame. @@ -794,6 +842,10 @@ namespace MAT_NS_BEGIN static constexpr char newRecordSignature[] = "(JIIJIJ[B)Lcom/microsoft/applications/events/StorageRecord;"; + if (!m_room) + { + return 0; + } auto room_class = env->GetObjectClass(m_room); size_t count = std::min(records.size(), INT32_MAX); @@ -924,6 +976,14 @@ namespace MAT_NS_BEGIN try { ConnectedEnv env(s_vm); + if (!env) + { + return false; + } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); auto delete_method = env->GetMethodID(room_class, "deleteSetting", "(Ljava/lang/String;)V"); @@ -964,6 +1024,14 @@ namespace MAT_NS_BEGIN try { ConnectedEnv env(s_vm); + if (!env) + { + return false; + } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); jmethodID store_setting = env->GetMethodID( room_class, @@ -1081,6 +1149,10 @@ namespace MAT_NS_BEGIN size_t OfflineStorage_Room::GetSizeInternal(ConnectedEnv& env) const { + if (!m_room) + { + return 0; + } auto room_class = env->GetObjectClass(m_room); auto method = env->GetMethodID(room_class, "totalSize", "()J"); if (!method) @@ -1107,6 +1179,10 @@ namespace MAT_NS_BEGIN { return 0; } + if (!m_room) + { + return 0; + } auto room_class = env->GetObjectClass(m_room); auto count_id = env->GetMethodID(room_class, "getRecordCount", "(I)J"); ThrowLogic(env, "getRecordCount"); @@ -1162,6 +1238,10 @@ namespace MAT_NS_BEGIN { return false; } + if (!m_room) + { + return false; + } auto room_class = env->GetObjectClass(m_room); auto trim_id = env->GetMethodID(room_class, "trim", "(J)J"); ThrowLogic(env, "trim"); @@ -1192,6 +1272,14 @@ namespace MAT_NS_BEGIN { ConnectedEnv env(s_vm); + if (!env) + { + return records; + } + if (!m_room) + { + return records; + } auto room_class = env->GetObjectClass(m_room); auto method = env->GetMethodID(room_class, "getRecords", "(ZIJ)[Lcom/microsoft/applications/events/StorageRecord;"); diff --git a/lib/stats/Statistics.cpp b/lib/stats/Statistics.cpp index 773dd4d13..a1377ac37 100644 --- a/lib/stats/Statistics.cpp +++ b/lib/stats/Statistics.cpp @@ -9,6 +9,7 @@ #include "ILogManager.hpp" #include "mat/config.h" #include "utils/Utils.hpp" +#include "decorators/RecordFlagConstants.hpp" #include namespace MAT_NS_BEGIN { @@ -83,6 +84,13 @@ namespace MAT_NS_BEGIN { result &= m_baseDecorator.decorate(record); // Allow stats to capture Part A common properties, but not the custom result &= m_semanticContextDecorator.decorate(record, true); + // Stats events bypass EventPropertiesDecorator, so apply the same + // collector-side client-IP scrub here (on by default; opt out via + // CFG_BOOL_ENABLE_IP_SCRUBBING = false). + if (!m_config.HasConfig(CFG_BOOL_ENABLE_IP_SCRUBBING) || m_config[CFG_BOOL_ENABLE_IP_SCRUBBING]) + { + record.flags |= RECORD_FLAGS_EVENTTAG_SCRUB_IP; + } if (result) { IncomingEventContext evt(PAL::generateUuidString(), tenantToken, EventLatency_Normal, EventPersistence_Normal, &record); diff --git a/tests/unittests/EventPropertiesDecoratorTests.cpp b/tests/unittests/EventPropertiesDecoratorTests.cpp index f38444fd0..348354604 100644 --- a/tests/unittests/EventPropertiesDecoratorTests.cpp +++ b/tests/unittests/EventPropertiesDecoratorTests.cpp @@ -28,6 +28,19 @@ class TestEventPropertiesDecorator : public EventPropertiesDecorator } }; +// NullLogManager hands out a single shared static ILogConfiguration, which would +// leak configuration across tests. This subclass owns a per-instance configuration +// so the IP-scrubbing opt-out can be exercised in isolation. +class ConfigurableLogManager : public NullLogManager +{ +public: + ILogConfiguration localConfig; + ILogConfiguration& GetLogConfiguration() override + { + return localConfig; + } +}; + static std::unique_ptr PopulateRecordForDropPii() { auto record = std::unique_ptr(new Record{}); @@ -545,3 +558,41 @@ TEST(EventPropertiesDecoratorTests, DropPiiPartA_StripsValues) EXPECT_THAT(record->extSdk[0].installId, Eq("")); EXPECT_THAT(record->cV, Eq("")); } + +TEST(EventPropertiesDecoratorTests, Decorate_ScrubIp_EnabledByDefault) +{ + ConfigurableLogManager logManager; // CFG_BOOL_ENABLE_IP_SCRUBBING not set + EventPropertiesDecorator decorator(logManager); + Record record; + EventProperties props {"TestEvent"}; + EventLatency latency = EventLatency::EventLatency_Normal; + + EXPECT_TRUE(decorator.decorate(record, latency, props)); + EXPECT_TRUE(record.flags & RECORD_FLAGS_EVENTTAG_SCRUB_IP); +} + +TEST(EventPropertiesDecoratorTests, Decorate_ScrubIp_OptOutViaConfig) +{ + ConfigurableLogManager logManager; + logManager.localConfig[CFG_BOOL_ENABLE_IP_SCRUBBING] = false; + EventPropertiesDecorator decorator(logManager); + Record record; + EventProperties props {"TestEvent"}; + EventLatency latency = EventLatency::EventLatency_Normal; + + EXPECT_TRUE(decorator.decorate(record, latency, props)); + EXPECT_FALSE(record.flags & RECORD_FLAGS_EVENTTAG_SCRUB_IP); +} + +TEST(EventPropertiesDecoratorTests, Decorate_ScrubIp_ExplicitlyEnabled) +{ + ConfigurableLogManager logManager; + logManager.localConfig[CFG_BOOL_ENABLE_IP_SCRUBBING] = true; + EventPropertiesDecorator decorator(logManager); + Record record; + EventProperties props {"TestEvent"}; + EventLatency latency = EventLatency::EventLatency_Normal; + + EXPECT_TRUE(decorator.decorate(record, latency, props)); + EXPECT_TRUE(record.flags & RECORD_FLAGS_EVENTTAG_SCRUB_IP); +} diff --git a/wrappers/obj-c/ODWLogConfiguration.h b/wrappers/obj-c/ODWLogConfiguration.h index 3cbdaaa61..6e3f77946 100644 --- a/wrappers/obj-c/ODWLogConfiguration.h +++ b/wrappers/obj-c/ODWLogConfiguration.h @@ -44,6 +44,11 @@ extern NSString * _Nonnull const ODWCFG_BOOL_ENABLE_WAL_JOURNAL; */ extern NSString * _Nonnull const ODWCFG_BOOL_ENABLE_NET_DETECT; +/*! + Scrub (obfuscate) the client IP address at the collector. Applied unless explicitly set to false (on by default; not present in the default configuration). +*/ +extern NSString * _Nonnull const ODWCFG_BOOL_ENABLE_IP_SCRUBBING; + /*! The event collection URI. */ diff --git a/wrappers/obj-c/ODWLogConfiguration.mm b/wrappers/obj-c/ODWLogConfiguration.mm index d69ddf70c..611b92940 100644 --- a/wrappers/obj-c/ODWLogConfiguration.mm +++ b/wrappers/obj-c/ODWLogConfiguration.mm @@ -50,6 +50,11 @@ */ NSString *const ODWCFG_BOOL_ENABLE_NET_DETECT = @"enableNetworkDetector"; +/*! + Scrub (obfuscate) the client IP address at the collector. Applied unless explicitly set to false (on by default; not present in the default configuration). +*/ +NSString *const ODWCFG_BOOL_ENABLE_IP_SCRUBBING = @"enableIpScrubbing"; + /*! The event collection URI. */ From 6c37993c3bc4debb8e6101d6e99765a27631bf71 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Tue, 23 Jun 2026 17:46:40 -0500 Subject: [PATCH 05/30] vcpkg port: release-bump workflow, overlay port bump, and consumer binary-footprint reductions (#1475) * Add vcpkg-release-bump workflow to automate port version bumps On a published version release, open a PR to microsoft/vcpkg bumping the cpp-client-telemetry port (REF -> tag, recomputed SHA512, version, then x-add-version). Runs only on published, non-prerelease version tags (vX.Y.Z.W) or manual dispatch, and opens no PR when the port already matches the release. Requires repo variable VCPKG_FORK_REPO and secret VCPKG_BUMP_TOKEN. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Bump in-repo overlay port to v3.10.161.1 tag Repoint tools/ports/cpp-client-telemetry REF from the pre-release commit to the published v3.10.161.1 tag (SHA512 updated) for exact parity with the official microsoft/vcpkg port. Version was already 3.10.161.1. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * docs(vcpkg): add manifest-mode overlay fallback for pre-registry installs building-with-vcpkg.md only told manifest-mode users to add `cpp-client-telemetry` to vcpkg.json, which fails with an unknown-port error until the port is accepted into the official vcpkg registry. Document the `vcpkg-configuration.json` `overlay-ports` fallback that points manifest mode at the in-repo overlay port, giving parity with the classic-mode `--overlay-ports` instructions already in the doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg-bump: address Copilot force-with-lease comment + drop docs note .github/workflows/vcpkg-release-bump.yml (Copilot): `git push --force-with-lease` could fail on reruns because a fresh clone has no remote-tracking ref for an already-existing bump branch, so the workflow couldn't refresh an open bump PR (contradicting the "force-pushed branch refreshes it" intent). Fetch the branch into refs/remotes/origin/${BR} (|| true on the first run, when it doesn't exist yet) before the force-with-lease push so the lease has a ref to compare against. Verified at .github/workflows/vcpkg-release-bump.yml:146-152. docs/building-with-vcpkg.md: remove the manifest-mode overlay-ports note added in 136e0100, per maintainer request. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Update vcpkg docs: port is now live in the official registry The cpp-client-telemetry port was merged into the upstream vcpkg registry (microsoft/vcpkg#52316, version 3.10.161.1), so docs/building-with-vcpkg.md no longer needs the conditional "once the port is accepted" phrasing. - Intro: state the port is published in the official registry and consumable directly; drop the stale "build recipe / CONTROL file" wording (vcpkg uses vcpkg.json, and the port is registry-resolved now). - "Installing from the vcpkg registry": present tense, link to the upstream ports/cpp-client-telemetry directory. - "Installing from the overlay port": reframe as development-only (test local port changes or a newer SDK revision before they reach the registry). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Align overlay port with the merged upstream vcpkg port The cpp-client-telemetry port merged into microsoft/vcpkg (ports/cpp-client-telemetry) ships only portfile.cmake + vcpkg.json. Bring the in-repo overlay back in sync so testing the overlay validates exactly what is published. - Drop the custom 'usage' file and its install step in portfile.cmake. The two lines it printed (find_package(MSTelemetry CONFIG REQUIRED) + target_link_libraries ... MSTelemetry::mat) duplicate vcpkg's auto-generated heuristic usage, and the upstream port carries no usage file. - Reorder vcpkg.json dependencies to vcpkg format-manifest canonical (alphabetical) order; same dependency set, no resolution change. After this, the overlay portfile.cmake and vcpkg.json are byte-identical to the upstream port blobs (cfdab236 / a74cc08b). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix vcpkg-bump PR existence check: jq null skipped PR creation Copilot review (round): the open-PR existence guard used --jq '.[0].number'. On the first run, when no PR exists yet, gh pr list returns [] and .[0].number evaluates to null, which gh prints as the literal string "null". [ -n "null" ] is true, so the workflow wrongly logged "An open PR already exists" and skipped 'gh pr create' -- the release-bump PR would never be opened on a clean run. Fix: --jq '.[0].number // empty' yields empty output when no PR exists (guard false -> PR created) and the PR number when one does (guard true -> skipped). Verified jq semantics (jq 1.x): '[] | .[0].number' -> null (prints "null"); '[] | .[0].number // empty' -> no output; '[{number:42}] | .[0].number // empty' -> 42. Confirmed at .github/workflows/vcpkg-release-bump.yml:158. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Harden vcpkg-bump workflow: graceful no-op on non-version tags, no token in URL Address Copilot round comments on the release-bump workflow. vcpkg-release-bump.yml:77 - non-version tag handling was contradictory: the message said "skipping" but the step ran exit 1, failing the workflow. A release published with a non-4-part tag (the SDK has historical 3-part tags like v3.3.8) would mark the automatic run red for what should be a no-op. Now: manual workflow_dispatch with a bad tag still fails loudly (user error), but the automatic release trigger emits a notice, sets a 'skip' output, and exits 0. All downstream steps are gated on steps.ver.outputs.skip != 'true'. vcpkg-release-bump.yml:100 - the PAT was embedded in the clone URL, which persists it in .git/config and risks leaking if git echoes the remote. Switch to 'gh auth setup-git' (writes a credential helper to the global gitconfig) plus a tokenless https clone; the later push step reuses that helper via its GH_TOKEN env. No token appears in any URL or on disk. Validated: workflow YAML parses (PyYAML) and every embedded run block passes 'bash -n'. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg-bump workflow: pass release tag via env to prevent shell injection Code review flagged a GitHub Actions script-injection vector: the "Resolve tag and version" step interpolated the untrusted release tag directly into the run: shell, before any validation. A tag containing shell metacharacters (e.g. v1.0.0.0";id;") would execute at assignment time, before the version regex runs. Injected code could write to GITHUB_ENV/GITHUB_PATH, which persist into the later Clone and push/PR steps that carry the VCPKG_BUMP_TOKEN PAT, enabling token exfiltration. Fix: pass the tag values through env: (RELEASE_TAG/INPUT_TAG) and reference them as quoted shell variables (TAG="${RELEASE_TAG:-$INPUT_TAG}"). Env values are not parsed as shell, so metacharacters can no longer inject. All downstream steps already use the regex-validated steps.ver.outputs.* values. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Enable function-level linking in the CMake build so consumers can dead-strip The compiler-flags block in CMakeLists.txt is wrapped entirely in `if(NOT MATSDK_USE_VCPKG_DEPS)`, and its MSVC branch only sets warning flags -- never /Gy. So the CMake/vcpkg build (the one packaged for downstream consumers) compiles every TU without function-level COMDATs, and referencing one symbol pulls the whole .obj into the consumer image. Add a block, applied in BOTH vendored and vcpkg modes, that splits functions/data into COMDATs/sections (MSVC /Gy /Gw; GCC/Clang -ffunction-sections -fdata-sections; AppleClang -ffunction-sections). This lets a consumer's linker dead-strip unreferenced SDK code (/OPT:REF + /OPT:ICF, --gc-sections, -dead_strip) and matches the MSBuild Release projects, which already enable FunctionLevelLinking + OptimizeReferences + EnableCOMDATFolding. No source or ABI change. Bundled into the vcpkg PR since it directly improves the footprint of the vcpkg-packaged library. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Reword function-level-linking comment to justify the vcpkg-mode exception Copilot review: the new comment read as conflicting with the earlier "let the toolchain manage compiler flags" note. Clarify that these section/COMDAT flags are a deliberate exception -- they are not optimization/dependency choices the toolchain owns; the toolchain doesn't set them, and without them the vcpkg-packaged library links whole .obj files. No code change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Consolidate section-splitting flags into the global block (drop REL_FLAGS dupes) Copilot review: the non-vcpkg REL_FLAGS already injected -ffunction-sections (and -fdata-sections for GNU), duplicating the new global block on Release builds. Remove them from REL_FLAGS so the global add_compile_options block is the single source of truth for section splitting across both dependency modes (it now also covers MSVC /Gy /Gw and AppleClang, which CI confirms build clean). No behavior change; eliminates redundant flags and future drift. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg: request sqlite3 without default features (drop json1) + docs The SDK uses SQLite only for offline event storage (plain tables/indexes, no JSON/FTS/RTREE/vtab), so the port now declares sqlite3 with default-features:false instead of pulling json1. This is the necessary floor for footprint-conscious consumers: vcpkg unions features across the dependency graph and ignores default-features:false on transitive deps, so without this the SDK's own edge forces json1 on and no consumer can opt out. Measured: a consumer that also sets {"name":"sqlite3","default-features":false} in its root manifest links ~52 KB smaller (SQLITE_OMIT_JSON) on x64-windows-static Release; the vcpkg integration test stays 10/10. - tools/ports/cpp-client-telemetry/vcpkg.json: sqlite3 default-features:false; port-version 1 (port-only change over the published 3.10.161.1#0) - docs/building-with-vcpkg.md: document the required root-manifest opt-out Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * docs(vcpkg): add consumer linker dead-strip guidance; clarify json1 opt-out Address Copilot round comments on docs/building-with-vcpkg.md and document the footprint lever consumers actually need (validated end-to-end on a real downstream DLL, which shrank substantially once /OPT:REF,ICF were on). - Add "Enable linker dead-stripping" section: the SDK's /Gy /Gw only enable stripping; the consumer's link must set /OPT:REF + /OPT:ICF (with the /DEBUG gotcha that silently disables them) + /INCREMENTAL:NO, or --gc-sections / -dead_strip on GNU/Clang/Apple. Note static-link vs DLL-reexport caveat. - json1 section: clarify this is the in-repo OVERLAY port (registry port to follow upstream), addressing the "registry still pulls defaults" comment. - Reword the resolution explanation around vcpkg's union model instead of "ignores transitive default-features:false". Verified by dry-run: SDK edge opt-out alone keeps sqlite3[core,json1]; adding the same at the root yields sqlite3 (no json1); any edge requesting defaults restores json1 -- so the consumer must opt out at the root too. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg(overlay): drop port-version The in-repo overlay is only used for local testing (overlays ignore the version database), and the vcpkg-release-bump workflow deletes port-version on every bump, so it served no purpose here. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * docs(vcpkg): precise /DEBUG wording for /OPT:REF,ICF Address Copilot comment on the linker-stripping guidance. Per the MSVC /OPT docs, /DEBUG changes the /OPT default from REF/ICF to NOREF/NOICF (it does disable them by default, contrary to the comment's premise). Reword to the exact behavior ("flips their default to off, /OPT:NOREF,NOICF") and note that /OPT:REF is also incompatible with incremental linking (hence /INCREMENTAL:NO). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * cmake: clarify AppleClang dead-strip comment (ld64 atomizes per symbol) Address Copilot comment: reword the AppleClang branch to state the actual mechanism -- clang emits .subsections_via_symbols on Mach-O, so ld64's -dead_strip removes unreferenced code at per-symbol (function) granularity without -ffunction-sections (which we add only for cross-toolchain consistency). -fdata-sections stays omitted due to the historical bitcode conflict. No behavior change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * build: hide non-public symbols on non-Windows (-fvisibility=hidden) Add -fvisibility=hidden -fvisibility-inlines-hidden for non-MSVC builds and make MATSDK_LIBABI = __attribute__((visibility("default"))) on GCC/Clang, so only the MATSDK_LIBABI-decorated public API (the 56 public classes + the EVTSDK_LIBABI C API in mat.h) is exported; SDK internals and the bundled sqlite3/zlib are hidden. Non-Windows analog of the __declspec(dllexport)-gated export on Windows and of /Gy + the consumer's /OPT:REF: a much smaller dynamic symbol table -> faster dynamic linking/loading, smaller shared binaries, and more inlining/dead-code elimination. No behavior change for static consumers (hidden symbols remain usable within the same link); for shared-lib builds it restricts exports to the public API. Validated (NDK aarch64): compiling EventProperties.cpp with the flag yields the decorated public methods (EventProperties::SetType/GetType) as GLOBAL DEFAULT while internals (EventPropertiesStorage) are WEAK HIDDEN -- 64 exported vs 300 hidden in that one TU. Full cross-platform validation (build a shared lib and link a separate consumer on Linux/macOS/iOS/Android to confirm no public symbol is missing) should run in CI before merge. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * address Copilot round on #1475: clang-cl /Gw gating + workflow_dispatch input - CMakeLists.txt: if(MSVC) is also true for the ClangCL toolset, which supports /Gy but not /Gw. Apply /Gy for all MSVC-like compilers and gate /Gw to real cl.exe (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") so ClangCL builds don't choke on /Gw. - vcpkg-release-bump.yml: read the workflow_dispatch tag via github.event.inputs.tag (concurrency key + resolve step) so manual dispatches resolve the tag unambiguously. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Windows shared-lib: add a dllimport path and tie export decoration to linkage The SDK has no .def file, so __declspec(dllexport)/(dllimport) via MATSDK_LIBABI is the sole Windows export mechanism, but it only had export and static states -- no dllimport path for a consumer of a shared mat.dll. And the cmake build hard-defined MATSDK_SHARED_LIB=1 for every Win32 build regardless of BUILD_SHARED_LIBS, so a static build still decorated the public API with dllexport (which also re-exports SDK symbols from any consumer DLL that absorbs the static lib). - ctmacros.hpp: add a MATSDK_IMPORT_LIB branch -> __declspec(dllimport). - lib/CMakeLists.txt: drop the hard-coded MATSDK_SHARED_LIB=1; instead set it on the mat target by linkage. Shared: PRIVATE MATSDK_SHARED_LIB (export from the SDK) + INTERFACE MATSDK_IMPORT_LIB (carried by the installed MSTelemetry::mat target, so find_package() consumers get dllimport automatically). Static: MATSDK_STATIC_LIB so nothing is decorated. This makes the C++ public API safe to consume from a single shared mat.dll (and fixes the missing-dllimport gap). The MSBuild/.vcxproj projects are unaffected (they define MATSDK_SHARED_LIB themselves). Add docs/sharing-a-single-sdk-runtime.md: how to ship one shared mat runtime that multiple modules in a process import (stable C ABI recommended; C++ shared-DLL path with its ABI-matching requirements), per-port vcpkg linkage, single LogManager lifetime ownership, one-copy-on-the-loader-path, and validation. Validated: MSVC preprocessor expands MATSDK_LIBABI to dllexport / dllimport / empty for the shared / import / static cases; cmake configures and the static Linux build is unaffected (change is Windows-guarded). Windows shared export/import is exercised by CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Scope -fvisibility-inlines-hidden to C++ to avoid -Werror C failures -fvisibility-inlines-hidden is a C++-only option. Applying it to all languages via add_compile_options meant the legacy Android path's bundled C sources (sqlite3_bundled, zlib_bundled) received it too; under Clang this emits an unused-argument warning that becomes an error with the project's -Werror. Scope it to CXX via a COMPILE_LANGUAGE generator expression while keeping -fvisibility=hidden for both C and C++. Files changed: - CMakeLists.txt: -fvisibility-inlines-hidden gated to \$ Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Re-export ObjC wrapper classes in shared Apple builds The global -fvisibility=hidden (root CMakeLists.txt) hides Objective-C class symbols (_OBJC_CLASS_\*) as well as C++ internals. With the default BUILD_OBJC_WRAPPER=YES, a shared libmat.dylib on Apple would therefore export none of the public ODW* wrapper classes, and consumers linking against them would fail with undefined _OBJC_CLASS_\... symbols. Collect the ObjC wrapper translation units into OBJC_WRAPPER_SRCS and, for shared Apple builds, compile just those units with -fvisibility=default so the public Objective-C API is re-exported while the C++ core stays hidden. Files changed: - lib/CMakeLists.txt: OBJC_WRAPPER_SRCS variable + per-source -fvisibility=default for shared Apple builds Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Use portable OR generator expression in footprint docs snippet The multi-config form \$ only matches on CMake >= 3.19; on older CMake it compares the literal string and never matches, so a consumer copy-pasting the snippet would silently fail to enable /OPT:REF, /OPT:ICF, and /INCREMENTAL:NO. Switch to \$,\$>, which is valid across all supported CMake versions. Files changed: - docs/building-with-vcpkg.md: OR-based CONFIG generator expression in the dead-strip snippet Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Gate visibility on NOT WIN32; clarify C-API dllimport doc wording Two review fixes: - CMakeLists.txt: the hidden-visibility block is described as non-Windows but was gated on NOT MSVC, which also matches MinGW/Clang-GNU Windows and would apply ELF-style -fvisibility=hidden to a PE/COFF target. Gate it on NOT WIN32 so all Windows toolchains rely on __declspec(dllexport) as intended. - docs/sharing-a-single-sdk-runtime.md: the C-API bullet said it 'links without __declspec(dllimport)', which contradicted the new MATSDK_IMPORT_LIB interface define. Reworded to: dllimport is not required (a C function resolves via the import-lib thunk) but is applied automatically to shared consumers as a harmless optimization. Files changed: - CMakeLists.txt: NOT MSVC -> NOT WIN32 for the visibility block - docs/sharing-a-single-sdk-runtime.md: C-API dllimport wording Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Scope footprint guidance to static-link scenarios The 'Reducing binary footprint' section assumed the SDK is always linked statically, but vcpkg's default triplets (e.g. x64-windows) build dynamic libraries and the port also supports BUILD_SHARED_LIBS=ON. Clarify that the consumer-side dead-stripping guidance applies to static linkage, and note that a dynamic mat ships its own runtime whose export table is already trimmed by the SDK's -fvisibility=hidden and /Gy /Gw. Files changed: - docs/building-with-vcpkg.md: scope footprint section to static-link case Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Gate non-Windows default-visibility on shared builds (mirror Windows) On non-Windows, MATSDK_LIBABI unconditionally expanded to __attribute__((visibility("default"))), so even in a static build the public API kept default visibility despite -fvisibility=hidden. A consumer that statically absorbs libmat.a into its own .so/.dylib would then unintentionally re-export the SDK's public API (larger dynamic symbol table, leaked SDK surface) -- the non-Windows analog of the Windows re-export that MATSDK_STATIC_LIB already prevents. Gate the visibility attribute on MATSDK_SHARED_LIB so it mirrors the __declspec(dllexport) gating: shared builds export the API; static builds omit the attribute, letting the public symbols inherit -fvisibility=hidden. Define MATSDK_SHARED_LIB PRIVATE for all shared builds in lib/CMakeLists.txt (keeping the Windows-only INTERFACE MATSDK_IMPORT_LIB and MATSDK_STATIC_LIB). Verified on Linux (readelf): static build -> evt_api_call_default is GLOBAL HIDDEN (still resolvable by static linking -- a consumer links+runs against libmat.a -- but not re-exported); shared build -> GLOBAL DEFAULT (exported from libmat.so). Files changed: - lib/include/public/ctmacros.hpp: non-Windows MATSDK_LIBABI gated on MATSDK_SHARED_LIB - lib/CMakeLists.txt: define MATSDK_SHARED_LIB for all shared builds Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/vcpkg-release-bump.yml | 193 ++++++++++++++++++ CMakeLists.txt | 60 +++++- docs/building-with-vcpkg.md | 74 ++++++- docs/sharing-a-single-sdk-runtime.md | 164 +++++++++++++++ lib/CMakeLists.txt | 45 +++- lib/include/public/ctmacros.hpp | 22 +- .../ports/cpp-client-telemetry/portfile.cmake | 7 +- tools/ports/cpp-client-telemetry/usage | 4 - tools/ports/cpp-client-telemetry/vcpkg.json | 11 +- 9 files changed, 550 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/vcpkg-release-bump.yml create mode 100644 docs/sharing-a-single-sdk-runtime.md delete mode 100644 tools/ports/cpp-client-telemetry/usage diff --git a/.github/workflows/vcpkg-release-bump.yml b/.github/workflows/vcpkg-release-bump.yml new file mode 100644 index 000000000..d09706242 --- /dev/null +++ b/.github/workflows/vcpkg-release-bump.yml @@ -0,0 +1,193 @@ +name: Vcpkg release bump + +# Opens a version-bump pull request against microsoft/vcpkg for the +# `cpp-client-telemetry` port whenever a new SDK release is published. +# +# It runs ONLY when a new version is cut: +# * automatically on a published, non-draft, non-prerelease GitHub Release +# whose tag looks like a version (vMAJOR.MINOR.PATCH.BUILD), or +# * manually via workflow_dispatch for a specific tag (recovery / re-run). +# It never runs on ordinary pushes, and it opens no PR if the port already +# matches the release (no version change). +# +# One-time setup required in this repository: +# * Variable VCPKG_FORK_REPO -> the vcpkg fork to push branches to, +# e.g. "your-org/vcpkg". +# * Secret VCPKG_BUMP_TOKEN -> a PAT (classic: repo+workflow, or +# fine-grained: Contents+Pull requests RW on +# the fork) able to push to VCPKG_FORK_REPO and +# open pull requests on microsoft/vcpkg. + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Release tag to bump the vcpkg port to (e.g. v3.10.161.1)" + required: true + type: string + +permissions: + contents: read + +concurrency: + group: vcpkg-release-bump-${{ github.event.release.tag_name || github.event.inputs.tag }} + cancel-in-progress: false + +jobs: + bump: + name: Bump cpp-client-telemetry port + # Skip drafts and pre-releases; always allow manual dispatch. + if: >- + ${{ github.event_name == 'workflow_dispatch' || + (github.event.release.draft == false && github.event.release.prerelease == false) }} + runs-on: ubuntu-latest + env: + UPSTREAM_REPO: ${{ github.repository }} # microsoft/cpp_client_telemetry + VCPKG_UPSTREAM: microsoft/vcpkg + VCPKG_FORK_REPO: ${{ vars.VCPKG_FORK_REPO }} + PORT: cpp-client-telemetry + steps: + - name: Validate configuration + env: + VCPKG_BUMP_TOKEN: ${{ secrets.VCPKG_BUMP_TOKEN }} + run: | + set -euo pipefail + if [ -z "${VCPKG_FORK_REPO}" ]; then + echo "::error::Repository variable VCPKG_FORK_REPO is not set (e.g. 'your-org/vcpkg')." + exit 1 + fi + if [ -z "${VCPKG_BUMP_TOKEN}" ]; then + echo "::error::Secret VCPKG_BUMP_TOKEN is not set. Provide a token that can push to ${VCPKG_FORK_REPO} and open PRs on ${VCPKG_UPSTREAM}." + exit 1 + fi + + - name: Resolve tag and version + id: ver + env: + # Pass untrusted tag values through the environment instead of + # interpolating ${{ ... }} directly into the script body, so a tag + # containing shell metacharacters cannot inject commands into this + # step (which shares a runner with later PAT-bearing steps). + RELEASE_TAG: ${{ github.event.release.tag_name }} + INPUT_TAG: ${{ github.event.inputs.tag }} + run: | + set -euo pipefail + TAG="${RELEASE_TAG:-$INPUT_TAG}" + if [ -z "${TAG}" ]; then echo "::error::No release tag could be resolved."; exit 1; fi + # Only act on version tags: vMAJOR.MINOR.PATCH.BUILD. A non-matching + # tag from the automatic release trigger is a clean no-op (the SDK also + # has historical 3-part tags such as v3.3.8); a non-matching tag from a + # manual workflow_dispatch is user error and fails loudly. + if ! printf '%s' "${TAG}" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "::error::Tag '${TAG}' is not a version tag (expected vX.Y.Z.W)." + exit 1 + fi + echo "::notice::Tag '${TAG}' is not a version tag (expected vX.Y.Z.W); nothing to bump." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + VERSION="${TAG#v}" + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "branch=port/${PORT}-${VERSION}" >> "$GITHUB_OUTPUT" + echo "Bumping ${PORT} -> tag=${TAG} version=${VERSION}" + + - name: Compute source archive SHA512 + id: sha + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + URL="https://github.com/${UPSTREAM_REPO}/archive/${{ steps.ver.outputs.tag }}.tar.gz" + echo "Downloading ${URL}" + curl -fsSL --retry 3 "${URL}" -o source.tar.gz + SHA512="$(sha512sum source.tar.gz | cut -d' ' -f1)" + echo "sha512=${SHA512}" >> "$GITHUB_OUTPUT" + echo "SHA512=${SHA512}" + + - name: Clone vcpkg fork and branch off upstream master + if: ${{ steps.ver.outputs.skip != 'true' }} + env: + GH_TOKEN: ${{ secrets.VCPKG_BUMP_TOKEN }} + run: | + set -euo pipefail + # Authenticate git via gh's credential helper instead of embedding the + # token in the clone URL (which would persist it in .git/config and + # risk leaking it if git echoes the remote). The helper is written to + # the global gitconfig and reused by the later push step. + gh auth setup-git + git clone --depth 1 "https://github.com/${VCPKG_FORK_REPO}.git" vcpkg + cd vcpkg + git remote add upstream "https://github.com/${VCPKG_UPSTREAM}.git" + git fetch --depth 1 upstream master + git checkout -B "${{ steps.ver.outputs.branch }}" upstream/master + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Bootstrap vcpkg + if: ${{ steps.ver.outputs.skip != 'true' }} + run: cd vcpkg && ./bootstrap-vcpkg.sh -disableMetrics + + - name: Update port REF, SHA512 and version + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + cd vcpkg + PORTFILE="ports/${PORT}/portfile.cmake" + MANIFEST="ports/${PORT}/vcpkg.json" + if [ ! -f "${PORTFILE}" ] || [ ! -f "${MANIFEST}" ]; then + echo "::error::${PORT} port not found in ${VCPKG_UPSTREAM}. The port must already be in the registry before it can be bumped." + exit 1 + fi + sed -i -E "s|^([[:space:]]*REF[[:space:]]+).*$|\1${{ steps.ver.outputs.tag }}|" "${PORTFILE}" + sed -i -E "s|^([[:space:]]*SHA512[[:space:]]+).*$|\1${{ steps.sha.outputs.sha512 }}|" "${PORTFILE}" + jq --arg v "${{ steps.ver.outputs.version }}" '.version = $v | del(."port-version")' "${MANIFEST}" > "${MANIFEST}.tmp" + mv "${MANIFEST}.tmp" "${MANIFEST}" + ./vcpkg format-manifest "${MANIFEST}" + + - name: Detect change + id: diff + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + cd vcpkg + if git diff --quiet -- "ports/${PORT}"; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "No change: ${PORT} is already at ${{ steps.ver.outputs.version }} with this REF/SHA512. Nothing to do." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit, update version DB, push and open PR + if: ${{ steps.ver.outputs.skip != 'true' && steps.diff.outputs.changed == 'true' }} + env: + GH_TOKEN: ${{ secrets.VCPKG_BUMP_TOKEN }} + run: | + set -euo pipefail + cd vcpkg + # gh auth setup-git ran in the clone step; reuse that credential helper + # so 'git push' authenticates without a token in the remote URL. + BR="${{ steps.ver.outputs.branch }}" + git add "ports/${PORT}" + git commit -m "[${PORT}] Update to ${{ steps.ver.outputs.version }}" + ./vcpkg x-add-version "${PORT}" --overwrite-version + git add versions + git commit -m "[${PORT}] Update version database" + # Ensure a remote-tracking ref exists so --force-with-lease has a lease + # to compare against on reruns: the bump branch may already exist on the + # fork but be absent from this fresh clone. Ignore failure on the first + # run, when the branch does not exist remotely yet. + git fetch origin "+refs/heads/${BR}:refs/remotes/origin/${BR}" || true + git push --force-with-lease origin "${BR}" + if [ -n "$(gh pr list --repo "${VCPKG_UPSTREAM}" --head "$(printf '%s' "${VCPKG_FORK_REPO}" | cut -d/ -f1):${BR}" --state open --json number --jq '.[0].number // empty' 2>/dev/null)" ]; then + echo "An open PR already exists for ${BR}; the force-pushed branch refreshes it." + else + gh pr create \ + --repo "${VCPKG_UPSTREAM}" \ + --base master \ + --head "$(printf '%s' "${VCPKG_FORK_REPO}" | cut -d/ -f1):${BR}" \ + --title "[${PORT}] Update to ${{ steps.ver.outputs.version }}" \ + --body "Automated port bump to [\`${UPSTREAM_REPO}@${{ steps.ver.outputs.tag }}\`](https://github.com/${UPSTREAM_REPO}/releases/tag/${{ steps.ver.outputs.tag }}). Generated by the \`vcpkg-release-bump\` workflow." + fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a0ba0e82..360bf7436 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -149,16 +149,17 @@ else() endif() if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - # Using GCC with -s and -Wl linker flags - set(REL_FLAGS "-s -Wl,--gc-sections -Os ${WARN_FLAGS} -ffunction-sections -fdata-sections -fmerge-all-constants") + # Using GCC with -s and -Wl linker flags. -ffunction-sections/-fdata-sections + # are set once for all dep modes by the global block further below. + set(REL_FLAGS "-s -Wl,--gc-sections -Os ${WARN_FLAGS} -fmerge-all-constants") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") set(REL_FLAGS "${WARN_FLAGS}") elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") - # AppleClang does not support -ffunction-sections and -fdata-sections with the -fembed-bitcode and -fembed-bitcode-marker set(REL_FLAGS "-Os ${WARN_FLAGS} -fmerge-all-constants") else() - # Using clang - strip unsupported GCC options - set(REL_FLAGS "-Os ${WARN_FLAGS} -ffunction-sections -fmerge-all-constants") + # Using clang - strip unsupported GCC options (-ffunction-sections is set by + # the global block further below). + set(REL_FLAGS "-Os ${WARN_FLAGS} -fmerge-all-constants") endif() ## Uncomment this to reduce the volume of note warnings on RPi4 w/gcc-8 Ref. https://gcc.gnu.org/ml/gcc/2017-05/msg00073.html @@ -206,6 +207,55 @@ endif() endif() # NOT MATSDK_USE_VCPKG_DEPS (compiler flags) +# --- Dead-strip enablement (applies in BOTH vendored and vcpkg modes) --------- +# Deliberate exception to the "let the toolchain manage compiler flags" note +# above (the NOT MATSDK_USE_VCPKG_DEPS block): these flags are NOT optimization +# or dependency choices the vcpkg toolchain owns -- they only split functions and +# data into separate COMDATs/sections so a *consumer's* linker can drop +# unreferenced SDK code (MSVC /OPT:REF + /OPT:ICF, GNU/Clang --gc-sections, Apple +# ld -dead_strip). The toolchain does not set them, and the vcpkg-packaged +# library (and every MSVC build, which never gets /Gy from the block above) would +# otherwise link whole .obj files instead of individual functions. Applying them +# here in both modes closes that gap and matches the MSBuild Release projects, +# which already enable FunctionLevelLinking + OptimizeReferences + COMDATFolding. +if(MSVC) + # /Gy (function-level linking) is supported by both cl.exe and clang-cl. + add_compile_options(/Gy) + # /Gw (whole-program global data) is cl.exe-only; the ClangCL toolset (for + # which MSVC is also true) does not support it. + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + add_compile_options(/Gw) + endif() +elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") + # On Mach-O, clang emits .subsections_via_symbols, so ld64's -dead_strip + # already removes unreferenced code at per-symbol (function) granularity + # without -ffunction-sections; we add it only for cross-toolchain + # consistency. -fdata-sections is omitted because it historically conflicted + # with bitcode on AppleClang. + add_compile_options(-ffunction-sections) +else() + # GCC / Clang (Linux, Android, MinGW) + add_compile_options(-ffunction-sections -fdata-sections) +endif() + +# Hidden symbol visibility (non-Windows): export only the MATSDK_LIBABI-decorated +# public API (classes + the C API), hiding SDK internals and the bundled +# sqlite3/zlib. This shrinks the dynamic symbol table (faster dynamic +# linking/loading, smaller binaries) and enables more inlining + dead-code +# elimination -- the non-Windows analog of what /Gy plus the consumer's /OPT:REF +# achieve on MSVC. All Windows toolchains (MSVC, MinGW, ClangCL) restrict exports +# via __declspec(dllexport) on MATSDK_LIBABI (lib/include/public/ctmacros.hpp), +# so this is gated on NOT WIN32 (not NOT MSVC, which would also catch MinGW/ +# Clang-GNU Windows builds and apply ELF-style visibility that does not belong on +# a PE/COFF target). +if(NOT WIN32) + # -fvisibility=hidden applies to C and C++; -fvisibility-inlines-hidden is a + # C++-only option, so scope it to CXX. (Applying it to C sources -- e.g. the + # bundled sqlite3/zlib on the legacy Android path -- makes Clang emit an + # "unused argument" warning that becomes an error under the project's -Werror.) + add_compile_options(-fvisibility=hidden $<$:-fvisibility-inlines-hidden>) +endif() + include(tools/Utils.cmake) include(GNUInstallDirs) include(CMakePackageConfigHelpers) diff --git a/docs/building-with-vcpkg.md b/docs/building-with-vcpkg.md index b736ba3c5..7e5cff7d0 100644 --- a/docs/building-with-vcpkg.md +++ b/docs/building-with-vcpkg.md @@ -1,6 +1,6 @@ # Building 1DS C++ SDK with vcpkg -[vcpkg](https://vcpkg.io/) is a Microsoft cross-platform open source C++ package manager. Onboarding instructions for Windows, Linux and Mac OS X [available here](https://docs.microsoft.com/en-us/cpp/build/vcpkg). This document assumes that the customer build system is already configured to use vcpkg ([getting started guide](https://learn.microsoft.com/en-us/vcpkg/get_started/overview)). 1DS C++ SDK maintainers provide a build recipe, `cpp-client-telemetry` port or CONTROL file for vcpkg. The mainline vcpkg repo is refreshed to point to latest stable open source release of 1DS C++ SDK. +[vcpkg](https://vcpkg.io/) is a Microsoft cross-platform open source C++ package manager. Onboarding instructions for Windows, Linux and Mac OS X [available here](https://docs.microsoft.com/en-us/cpp/build/vcpkg). This document assumes that the customer build system is already configured to use vcpkg ([getting started guide](https://learn.microsoft.com/en-us/vcpkg/get_started/overview)). The `cpp-client-telemetry` port is published in the official vcpkg registry, so it can be consumed directly with no overlay or extra configuration. Maintainers refresh the registry to point to the latest stable open source release of the 1DS C++ SDK on each release. The port provides the core SDK — the `MSTelemetry::mat` target and its public C++ headers. The optional Microsoft-proprietary modules (Privacy Guard, @@ -16,7 +16,8 @@ git clone --recurse-submodules https://github.com/microsoft/cpp_client_telemetry ### Installing from the vcpkg registry -Once a new port has been accepted into the official vcpkg registry, install with: +The `cpp-client-telemetry` port is available in the [official vcpkg registry](https://github.com/microsoft/vcpkg/tree/master/ports/cpp-client-telemetry), +so you can install it directly — no overlay or extra configuration required: ```console vcpkg install cpp-client-telemetry @@ -26,8 +27,9 @@ That's it! The package should be compiled for the current OS. ### Installing from the overlay port (development / pre-release) -Before the port is published, or to test local changes, use the overlay port -shipped in this repository: +The overlay port shipped in this repository is for **development only** — use it +to test local changes to the port, or a newer SDK revision, before they are +published to the registry: ```console git clone https://github.com/microsoft/cpp_client_telemetry @@ -190,6 +192,70 @@ will automatically use the optimized zlib-ng build. > zlib. When using `ZLIB_COMPAT=ON`, ensure all dependencies resolve to > zlib-ng rather than mixing stock zlib and zlib-ng. +## Reducing binary footprint + +This section applies when the SDK is linked **statically** into your binary +(the default for the `*-static` vcpkg triplets) — most footprint control then +lives on *your* side of the link. If you instead consume a **dynamic** `mat` +(e.g. the default `x64-windows` triplet, or `BUILD_SHARED_LIBS=ON`), the runtime +ships as its own `mat.dll` / `libmat.so` / `libmat.dylib`; the SDK's own +`-fvisibility=hidden` and `/Gy /Gw` already trim its exported symbol table, and +the consumer-side linker options below are specific to the static-link case. + +### Enable linker dead-stripping (largest lever) + +The SDK is compiled with function-level linking (`/Gy /Gw` on MSVC, +`-ffunction-sections -fdata-sections` on GCC/Clang) so that **your** linker can +discard SDK code you never reference. Make sure your final link enables it: + +- **MSVC:** `/OPT:REF` (drop unreferenced functions/data) and `/OPT:ICF` (fold + identical COMDATs). These are on by default for Release, **but `/DEBUG` flips + their default to off** (`/OPT:NOREF,NOICF`, per the MSVC `/OPT` docs) — so if + you ship PDBs, re-enable them explicitly. `/OPT:REF` is also incompatible with + incremental linking, so set `/INCREMENTAL:NO`: + + ```cmake + target_link_options(your_target PRIVATE + $<$,$>:/OPT:REF> + $<$,$>:/OPT:ICF> + $<$,$>:/INCREMENTAL:NO>) + ``` + +- **GCC / Clang:** link with `-Wl,--gc-sections`. +- **Apple (clang):** link with `-Wl,-dead_strip`. + +This is by far the largest lever — on a static `x64-windows-static` Release link +it can roughly halve the binary. The SDK's `/Gy /Gw` flags only *enable* this; +the stripping happens at your link. Keep the SDK a static dependency linked +*into* your binary: if you re-export its API across your own DLL boundary, the +export table pins its symbols and defeats `/OPT:REF`. + +### Drop unused SQLite features (json1) + +The SDK uses SQLite only for offline event storage — plain tables and indexes, +with no JSON, FTS, R*Tree, or virtual-table features. This in-repo overlay port +already requests `sqlite3` with `default-features: false` on its dependency edge +(the published registry port will follow once this change is upstreamed). + +vcpkg unions feature requests across the whole dependency graph, and a +transitive opt-out alone is **not** enough: you must **also** request `sqlite3` +with `default-features: false` in your own top-level manifest to actually omit +`json1` (which compiles SQLite with `SQLITE_OMIT_JSON`, ~50 KB smaller on a +static `x64-windows-static` Release build): + +```json +{ + "dependencies": [ + "cpp-client-telemetry", + { "name": "sqlite3", "default-features": false } + ] +} +``` + +If any package in your build (or your own code) needs SQLite's JSON functions, +request `sqlite3[json1]` instead and the extension is restored for the whole +graph. + ## How It Works: MATSDK_USE_VCPKG_DEPS When the SDK detects it is being built via vcpkg (by checking for diff --git a/docs/sharing-a-single-sdk-runtime.md b/docs/sharing-a-single-sdk-runtime.md new file mode 100644 index 000000000..10e69746d --- /dev/null +++ b/docs/sharing-a-single-sdk-runtime.md @@ -0,0 +1,164 @@ +# Sharing one SDK runtime across several modules in a process + +When more than one module in a single process links this SDK — for example an +application that loads several plug-ins or libraries, each of which uses 1DS — +the easy default (every module statically embeds the SDK) has two costs: + +1. **Size.** The SDK (plus its bundled SQLite/zlib) is duplicated once per module. +2. **Duplicated global state.** Each static copy has its *own* default + `LogManager`, HTTP transport, offline SQLite cache, and upload threads. They do + not share a pipeline, and multiple writers to the same offline-cache path will + corrupt it. + +This document describes how to ship **one** shared SDK runtime (`mat.dll` / +`libmat.so` / `libmat.dylib`) that every module imports, so there is a single +copy on disk and a single set of process-global state. + +There are two ways to consume the shared runtime. **The C API is strongly +recommended** because it removes the fragile C++/CRT ABI coupling between modules. + +--- + +## Option 1 (recommended): consume the stable C API + +The SDK ships a flat **C ABI** in [`mat.h`](../lib/include/public/mat.h). Every +`evt_*` entry point (`evt_open`, `evt_log`, `evt_flush`, `evt_upload`, +`evt_pause`, `evt_resume`, `evt_close`, `evt_configure`, …) is a `static inline` +wrapper that marshals its arguments into a POD struct and calls through a single +exported `__cdecl` symbol, `evt_api_call_default`. + +Consequences that make this the robust choice: + +* **Only one symbol crosses the module boundary, and no C++/STL type does.** The + request is a plain C struct, so there is *no* requirement that the modules and + the shared runtime agree on the C++ standard library ABI (`/MD`, + `_ITERATOR_DEBUG_LEVEL`, MSVC toolset/STL version, libstdc++ vs libc++, + `_GLIBCXX_USE_CXX11_ABI`, …). A 1DS version bump does not force every module to + rebuild in lockstep against an identical toolchain. +* **It does not *require* `__declspec(dllimport)` to link.** A plain C function + resolves through the shared library's import lib even without `dllimport`, so + the C API works across the boundary regardless. Consumers that link the shared + `MSTelemetry::mat` target do get `dllimport` applied automatically (via the + `MATSDK_IMPORT_LIB` interface define this PR adds); for a C function that is a + harmless calling-convention optimization, not a requirement. + +Each module includes `mat.h`, links the one shared runtime, and uses its own +tenant/source. You still pin the **same SDK version** in every module (so the +request/struct layout matches), but you avoid the C++ ABI lockstep entirely. + +## Option 2: consume the C++ API from a shared library + +All modules `find_package(MSTelemetry CONFIG REQUIRED)` and link +`MSTelemetry::mat` (resolving to the import lib); none statically embed the SDK. + +The C++ public API passes C++ standard-library types (`std::string`, `std::map`, +…) across the module boundary, so **every module and the shared runtime must +share one C++ ABI**. If they do not, you get heap corruption / undefined +behavior. Pin all of the following identically: + +| Axis | Requirement | +|------|-------------| +| **CRT linkage (Windows)** | Dynamic CRT (`/MD`, `/MDd` for Debug) everywhere — never `/MT`, and never mix Debug/Release CRT across the boundary. (vcpkg: `VCPKG_CRT_LINKAGE dynamic`.) | +| **STL / iterator debug** | One compiler + STL, one build config. `_ITERATOR_DEBUG_LEVEL` must match (Release `0` vs Debug `2`) — a Release consumer + Debug runtime is a silent layout mismatch. | +| **Toolset** | One MSVC toolset across all binaries (the v14x toolsets share an STL ABI, but don't mix major versions); or one libstdc++/libc++ with the same `_GLIBCXX_USE_CXX11_ABI`. | +| **Language / model** | Same `/std:c++NN`, same `/EHsc` exception model, same architecture, no overridden struct packing. | +| **SDK build options** | Same SDK feature/version selection in every module's manifest — different features mean different headers, hence a different ABI even at the same version. | + +Because the C++ ABI must match exactly across separately built and separately +versioned modules, this option is materially more brittle than the C API. Prefer +Option 1 unless you specifically need the C++ surface and control all modules' +toolchains. + +--- + +## Building the shared runtime with vcpkg (per-port linkage) + +A common requirement is "share *this* SDK, but keep everything else statically +linked (no DLL forest)". Override the library linkage **per port** in your +triplet so only this SDK goes dynamic: + +```cmake +set(VCPKG_CRT_LINKAGE dynamic) +if(PORT STREQUAL "cpp-client-telemetry") + set(VCPKG_LIBRARY_LINKAGE dynamic) # mat.dll / libmat.so / libmat.dylib + import lib +else() + set(VCPKG_LIBRARY_LINKAGE static) +endif() +``` + +The port honors `VCPKG_LIBRARY_LINKAGE` / `BUILD_SHARED_LIBS` and emits the +shared `mat` plus its import lib and the `MSTelemetry` CMake config package. + +### Pin one version across all modules + +All modules must compile against identical SDK headers. Pin the same +`cpp-client-telemetry` version and `builtin-baseline` (or a shared version +override) in every module's manifest, and ideally build all artifacts in the same +CI job/container with the same toolchain image. Most ABI drift comes from +separate modules quietly building on different agents. + +--- + +## How the SDK decorates its public symbols + +The SDK has no `.def` file; on Windows, exporting/importing is driven entirely by +`MATSDK_LIBABI` in [`ctmacros.hpp`](../lib/include/public/ctmacros.hpp), which the +build ties to the actual linkage: + +* **Shared build:** the SDK is compiled with `MATSDK_SHARED_LIB` + (`__declspec(dllexport)`), and the installed `MSTelemetry::mat` target carries + an `INTERFACE` definition of `MATSDK_IMPORT_LIB`, so consumers that + `find_package` + link automatically get `__declspec(dllimport)` — no + consumer-side configuration required. +* **Static build:** nothing is decorated, so the SDK's public symbols are not + re-exported by a consumer DLL that absorbs the static lib. + +On non-Windows platforms the SDK is built with `-fvisibility=hidden` and the +public API is marked `__attribute__((visibility("default")))`, so only the public +API (including the C API) is exported from the shared object. + +--- + +## Coordinate the single runtime's lifetime + +One shared runtime means **one** set of process-global state. Decide ownership: + +* **Recommended — single owner.** The top-level module initializes and tears down + the SDK (`LogManager::Initialize` / `FlushAndTeardown`, or `evt_open` / + `evt_close`). Other modules obtain loggers (their own tenant/source) but never + initialize or tear down. This avoids teardown-ordering crashes. +* **Alternative — named instances.** `LogManagerProvider::CreateLogManager(id)` + gives each module its own instance/tenant/config sharing the one transport; then + you need a last-one-out teardown refcount and **distinct offline-cache paths** + (one shared path with multiple writers corrupts it). +* Teardown must happen exactly once, **last**, after every module has stopped + logging. + +--- + +## Ship exactly one copy on the loader path + +Place a single runtime where every module finds it: + +* **Windows:** the same directory as the consumers (or side-by-side assembly). +* **Linux:** `RPATH=$ORIGIN` so every module resolves the one copy. +* **macOS:** a stable install name, `@rpath/libmat.dylib`. + +Make exactly one package own and ship the SDK runtime; the others declare a +dependency rather than bundling their own. If several packages each ship their +own copy, which one loads is path-order luck — and if their versions differ, you +are back to an ABI mismatch even with "one" DLL. + +--- + +## Validate + +* **Dependency present, definitions absent.** `dumpbin /dependents` (Windows), + `ldd` (Linux), `otool -L` (macOS) on each consumer should show a dependency on + the one `mat` module; `dumpbin /exports` (or `nm -D`) on a consumer should show + it imports — not defines — the SDK symbols. +* **One copy at runtime.** Process Explorer / `/proc//maps` / `vmmap` should + map the `mat` module exactly once; there should be one offline-cache file. +* **(C++ option) CRT/STL smoke test.** Have a consumer pass a `std::string` event + property into the SDK and read it back. A `/MD` vs `/MT` or `_ITERATOR_DEBUG_LEVEL` + mismatch typically crashes immediately (especially in Debug). diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 584c678ec..0d4161811 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -216,7 +216,7 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") endif() if(APPLE AND BUILD_OBJC_WRAPPER) message(STATUS "Include ObjC Wrappers") - list(APPEND SRCS + set(OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWLogger.mm ../wrappers/obj-c/ODWLogManager.mm ../wrappers/obj-c/ODWEventProperties.mm @@ -227,22 +227,23 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") ../wrappers/obj-c/ODWSanitizerInitConfig.mm ) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer/") - list(APPEND SRCS + list(APPEND OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWDiagnosticDataViewer.mm ) endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard/" AND BUILD_PRIVACYGUARD) set(MATSDK_OBJC_PRIVACYGUARD_AVAILABLE ON) - list(APPEND SRCS + list(APPEND OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWPrivacyGuard.mm ) endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer/" AND BUILD_SANITIZER) set(MATSDK_OBJC_SANITIZER_AVAILABLE ON) - list(APPEND SRCS + list(APPEND OBJC_WRAPPER_SRCS ../wrappers/obj-c/ODWSanitizer.mm ) endif() + list(APPEND SRCS ${OBJC_WRAPPER_SRCS}) endif() if(APPLE AND BUILD_SWIFT_WRAPPER) @@ -271,7 +272,7 @@ elseif(PAL_IMPLEMENTATION STREQUAL "WIN32") if(NOT MATSDK_USE_VCPKG_DEPS) include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../zlib ${CMAKE_CURRENT_SOURCE_DIR}/../sqlite) endif() -add_definitions(-D_UNICODE -DUNICODE -DWIN32 -DMATSDK_PLATFORM_WINDOWS=1 -DMATSDK_SHARED_LIB=1 -D_UTC_SDK -DUSE_BOND -D_WINDOWS -D_USRDLL -DWINVER=_WIN32_WINNT_WIN7) +add_definitions(-D_UNICODE -DUNICODE -DWIN32 -DMATSDK_PLATFORM_WINDOWS=1 -D_UTC_SDK -DUSE_BOND -D_WINDOWS -D_USRDLL -DWINVER=_WIN32_WINNT_WIN7) remove_definitions(-D_MBCS) list(APPEND SRCS http/HttpClient_WinInet.cpp @@ -323,6 +324,29 @@ else() add_library(mat STATIC ${SRCS}) endif() +# Public-API export decoration (MATSDK_LIBABI in lib/include/public/ctmacros.hpp). +# The SDK has no .def file, so __declspec(dllexport)/(dllimport) on Windows and +# __attribute__((visibility("default"))) elsewhere are the sole export mechanisms, +# and the decoration must follow the actual linkage: +# * shared: the SDK's own translation units export the public API +# (MATSDK_SHARED_LIB, PRIVATE). On Windows, consumers must additionally import +# it -- the INTERFACE MATSDK_IMPORT_LIB is carried by the installed +# MSTelemetry::mat target, so find_package() + link gives consumers dllimport +# automatically with no consumer-side configuration. Non-Windows consumers need +# nothing: they call symbols exported by libmat.so/.dylib. +# * static: decorate nothing, so the SDK's public symbols are NOT re-exported by +# a consumer DLL/.so that statically absorbs this library. Windows needs an +# explicit (empty) MATSDK_STATIC_LIB; elsewhere the empty MATSDK_LIBABI default +# plus -fvisibility=hidden (root CMakeLists.txt) already hides them. +if(BUILD_SHARED_LIBS) + target_compile_definitions(mat PRIVATE MATSDK_SHARED_LIB=1) + if(WIN32) + target_compile_definitions(mat INTERFACE MATSDK_IMPORT_LIB=1) + endif() +elseif(WIN32) + target_compile_definitions(mat PUBLIC MATSDK_STATIC_LIB=1) +endif() + # Target-based include paths for vcpkg / install workflow. # PUBLIC propagates to consumers; PRIVATE is SDK-internal only. # BUILD_INTERFACE is used during the SDK build; INSTALL_INTERFACE is used @@ -339,6 +363,17 @@ target_include_directories(mat ) if(APPLE AND BUILD_OBJC_WRAPPER) + if(BUILD_SHARED_LIBS AND OBJC_WRAPPER_SRCS) + # The root CMakeLists.txt applies -fvisibility=hidden globally to shrink the + # exported symbol table of the core C++ SDK. For Objective-C that also hides + # the wrapper class symbols (_OBJC_CLASS_$_ODW*), which are public API on + # Apple: a shared libmat.dylib would export no ODW* classes and consumers + # would fail to link (undefined _OBJC_CLASS_$_...). Re-export just the wrapper + # translation units with default visibility; the C++ core stays hidden. + set_source_files_properties(${OBJC_WRAPPER_SRCS} + PROPERTIES COMPILE_FLAGS "-fvisibility=default") + endif() + if(MATSDK_OBJC_PRIVACYGUARD_AVAILABLE) target_compile_definitions(mat PRIVATE MATSDK_OBJC_PRIVACYGUARD_AVAILABLE=1) else() diff --git a/lib/include/public/ctmacros.hpp b/lib/include/public/ctmacros.hpp index 42547e41d..cabd36f5f 100644 --- a/lib/include/public/ctmacros.hpp +++ b/lib/include/public/ctmacros.hpp @@ -28,9 +28,14 @@ #define MATSDK_LIBABI_CDECL __cdecl # if defined(MATSDK_SHARED_LIB) # define MATSDK_LIBABI __declspec(dllexport) +# elif defined(MATSDK_IMPORT_LIB) +// Consumer importing the public API from a shared mat.dll. The installed +// MSTelemetry::mat CMake target propagates this automatically when the SDK was +// built shared (see lib/CMakeLists.txt). +# define MATSDK_LIBABI __declspec(dllimport) # elif defined(MATSDK_STATIC_LIB) # define MATSDK_LIBABI -# else // Header file included by client +# else // Header file included by client; linkage unspecified # ifndef MATSDK_LIBABI # define MATSDK_LIBABI # endif @@ -47,8 +52,19 @@ #define MATSDK_LIBABI_CDECL #endif -#ifndef MATSDK_LIBABI -#define MATSDK_LIBABI +#ifndef MATSDK_LIBABI +// Mark the public API as default-visibility ONLY in shared builds, so it stays +// exported when the SDK is compiled with -fvisibility=hidden (see CMakeLists.txt). +// This mirrors the __declspec(dllexport) gating above: in a static build the +// attribute is omitted, so the public symbols inherit -fvisibility=hidden and are +// NOT re-exported when a consumer .so/.dylib statically absorbs libmat. (When a +// consumer includes this header, MATSDK_SHARED_LIB is not defined either, which +// is fine: the symbols are exported by the shared libmat they link against.) +# if (defined(__GNUC__) || defined(__clang__)) && defined(MATSDK_SHARED_LIB) +# define MATSDK_LIBABI __attribute__((visibility("default"))) +# else +# define MATSDK_LIBABI +# endif #endif // TODO: [MG] - ideally we'd like to use __attribute__((unused)) with gcc/clang diff --git a/tools/ports/cpp-client-telemetry/portfile.cmake b/tools/ports/cpp-client-telemetry/portfile.cmake index b0ce77107..cfdab2368 100644 --- a/tools/ports/cpp-client-telemetry/portfile.cmake +++ b/tools/ports/cpp-client-telemetry/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO microsoft/cpp_client_telemetry - REF 4485b82005abf1d24336ace99b11df88dd578eb0 - SHA512 1f3ee1c26f1ae9e7323262c9b4c8796efba2c6addcde432d6c6c77b8c1c2f254cb8ff334b1dd0a72dc8ecfbfbae04ab374ec5ac7e5d286d6042953d53e50fd5b + REF v3.10.161.1 + SHA512 4664b34ddce601d6a95669df4a59d11a6cc67de1f23de132192f791a275edc6a10b8498d340e6cf7d120d9e7a22c494d7517b24fc0954bf9e236e84a8800589a HEAD_REF main ) @@ -46,8 +46,5 @@ vcpkg_cmake_config_fixup(PACKAGE_NAME MSTelemetry CONFIG_PATH lib/cmake/MSTeleme file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -# Install usage instructions -file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") - # Install license vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/tools/ports/cpp-client-telemetry/usage b/tools/ports/cpp-client-telemetry/usage deleted file mode 100644 index 736d289f6..000000000 --- a/tools/ports/cpp-client-telemetry/usage +++ /dev/null @@ -1,4 +0,0 @@ -cpp-client-telemetry provides CMake targets: - - find_package(MSTelemetry CONFIG REQUIRED) - target_link_libraries(main PRIVATE MSTelemetry::mat) diff --git a/tools/ports/cpp-client-telemetry/vcpkg.json b/tools/ports/cpp-client-telemetry/vcpkg.json index d721df65a..2ee6d9bc3 100644 --- a/tools/ports/cpp-client-telemetry/vcpkg.json +++ b/tools/ports/cpp-client-telemetry/vcpkg.json @@ -6,9 +6,6 @@ "license": "Apache-2.0", "supports": "((windows & !mingw) | linux | osx | ios | android) & !uwp", "dependencies": [ - "nlohmann-json", - "sqlite3", - "zlib", { "name": "curl", "default-features": false, @@ -17,6 +14,11 @@ ], "platform": "linux | android" }, + "nlohmann-json", + { + "name": "sqlite3", + "default-features": false + }, { "name": "vcpkg-cmake", "host": true @@ -24,6 +26,7 @@ { "name": "vcpkg-cmake-config", "host": true - } + }, + "zlib" ] } From edf33f80035575b82f1fafd5f9bd0dc0d2064e94 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Tue, 23 Jun 2026 18:06:43 -0500 Subject: [PATCH 06/30] Prepare for new release - 3.10.173.1 (#1489) * Prepare for new release - 3.10.170.1 Regenerated lib/include/public/Version.hpp via tools/gen-version (date-derived version: 3.<(year-2020)+4>..1 = 3.10.170.1). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Update version to 3.10.173.1 --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/include/public/Version.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/include/public/Version.hpp b/lib/include/public/Version.hpp index bb114f5a5..cf7ec9b99 100644 --- a/lib/include/public/Version.hpp +++ b/lib/include/public/Version.hpp @@ -6,8 +6,8 @@ #define MAT_VERSION_HPP // WARNING: DO NOT MODIFY THIS FILE! // This file has been automatically generated, manual changes will be lost. -#define BUILD_VERSION_STR "3.10.161.1" -#define BUILD_VERSION 3,10,161,1 +#define BUILD_VERSION_STR "3.10.173.1" +#define BUILD_VERSION 3,10,173,1 #ifndef RESOURCE_COMPILER_INVOKED #include "ctmacros.hpp" @@ -18,7 +18,7 @@ namespace MAT_NS_BEGIN { uint64_t const Version = ((uint64_t)3 << 48) | ((uint64_t)10 << 32) | - ((uint64_t)161 << 16) | + ((uint64_t)173 << 16) | ((uint64_t)1); } MAT_NS_END From 38b7dfa5ebed3557c4bef640c722a1ff7370b03c Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Tue, 30 Jun 2026 22:14:15 -0500 Subject: [PATCH 07/30] Harden Windows command-line builds: VS2026/v145 support, CI failure propagation (#981), and docs (#1458) * Document supported Windows build toolsets Clarify the VS2019/VS2022/VS2026 command-line build entry points, avoid legacy .NET Framework 4.0 projects in VS2022+ wrappers, and make solution-level :Build targets work through RunMsBuild. Files changed: - docs/cpp-start-windows.md - build-all.bat - build-all-v143.bat - build-all-v145.bat - tools/RunMsBuild.bat - tools/setup-buildtools.cmd - tools/vcvars.cmd - tools/.vsconfig.vs2022 - tools/.vsconfig.vs2026 - Solutions/before.targets Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Windows build script review comments Clarify the Windows Visual Studio build entry point by moving the build matrix implementation to build-all-windows.bat while keeping build-all.bat as a compatibility wrapper. Update version-specific wrappers, CI, helper scripts, and docs to call the clearer name. Also clarify that MFC/ATL Visual Studio components remain intentional because SampleCppMini uses static MFC. Validation: - git diff --check - build-all-v142.bat Solutions\build.compact-dll.props smoke with all build legs skipped confirmed custom props forwarding - material self-review found no remaining issues Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * ci(windows): make the VS2022 build gate fail correctly Addresses code-review findings on the new Windows build CI: - build-windows-vs2022.yaml ran build-all-windows.bat with v143/vs2022 but without SKIP_NET40_BUILD, so it tried to build the legacy net40 / SampleCsNet40 projects that VS2022 cannot build (and that the docs say are skipped). Add SKIP_NET40_BUILD: 1 to match the canonical build-all-v143.bat. - build-all-windows.bat called tools\RunMsBuild.bat ~12 times with no errorlevel check between them, so only the last build's exit code reached the caller; an intermediate config failure was swallowed and the gating CI job could report success on a broken build. Add 'if errorlevel 1 exit /b 1' after each call to fail fast. (Pre-existing in build-all.bat, but newly load-bearing now that a CI gate runs this script; verified the swallowing and the fix with an isolated batch repro.) - Remove continue-on-error: true from the Checkout step so a failed checkout fails the job. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * build(win): add concrete Windows 11 SDK 10.0.22621 to vs2022/vs2026 .vsconfig The vs2022/vs2026 component configs only listed the bare Microsoft.VisualStudio.Component.Windows10SDK; since setup-buildtools.cmd applies them with --quiet and no --includeRecommended, a fresh install could end up with no concrete Windows SDK and fail to build. Add the explicit Windows11SDK.22621 component (the 10.0.22621 platform SDK the repo's CodeQL workflow already pins via WindowsSDKVersion: 10.0.22621.0), mirroring how .vsconfig.vs2019 pairs the bare component with a concrete Windows10SDK.18362. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * address Copilot review: doc project list + robust cd in build-all-windows.bat - docs/cpp-start-windows.md: drop 'win32-cs' from the list of legacy .NET 4.0 projects the VS2022/2026 wrappers skip. MSTelemetrySDK.sln only contains net40 and SampleCsNet40 (gated by SKIP_NET40_BUILD); win32-cs lives in a separate Solutions/win32-cs solution and is not part of the SDK build matrix. - build-all-windows.bat: use 'cd /d "%~dp0"' so the script reliably changes drive and tolerates spaces when invoked from another drive/working directory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * address Copilot review: locate sibling scripts via %~dp0, not CWD The VS-version wrappers (build-all-v142/v143/v145.bat) called build-all-windows.bat by bare relative name, and tools\build-Win10-compact-exp.cmd used 'cd ..' + a CWD-relative call. Invoked from another drive/working directory (e.g. via an absolute path), the sibling script wouldn't be found. Call it via a %~dp0-relative absolute path instead so the wrappers work from any CWD: - build-all-v142/v143/v145.bat: call "%~dp0build-all-windows.bat" %* - build-Win10-compact-exp.cmd: drop the fragile 'cd ..'/%CD% and call "%~dp0..\build-all-windows.bat" with the props file as a %~dp0-relative path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * build(win): fail with a clear message when Visual Studio is not detected vcvars.cmd sets VSTOOLS_NOTFOUND and exits 0 when it can't find any Visual Studio install, but RunMsBuild.bat didn't check it and went straight to msbuild, so a developer with no/undetected VS only saw a cryptic "'msbuild' is not recognized". Check VSTOOLS_NOTFOUND after calling vcvars and print an actionable error (install VS with the C++ workload, or run setup-buildtools.cmd, or set VSTOOLS_VERSION) before aborting. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * build(win): warn when VS detection falls back to a different version vcvars.cmd's label cascade silently moves forward when the requested Visual Studio isn't installed (e.g. a vs2022 request ends up on vs2026). Combined with the version wrappers pinning PlatformToolset (v143/v145), that mismatch surfaces later as a confusing toolset error from msbuild. Capture the explicitly requested version and, once configuration succeeds, print a clear warning when the detected VS differs from what was asked for. The configured path now returns a deterministic exit 0 (callers key off VSTOOLS_NOTFOUND, not the exit code). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * build(win): make VSTOOLS_NOTFOUND robust to stale shell state Copilot round: vcvars.cmd set VSTOOLS_NOTFOUND=1 on the not-found path but never cleared it on success, so a stale value left in the shell by a previous failed run could make RunMsBuild.bat's new guard abort a build even when VS is present. Clear VSTOOLS_NOTFOUND at vcvars.cmd entry (only the not-found path sets it now) and check the explicit ==1 value in RunMsBuild.bat instead of mere existence. Verified at tools/vcvars.cmd entry and tools/RunMsBuild.bat:17. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * build(win): clear stale VS output vars at vcvars.cmd entry Copilot round: clearing only VSTOOLS_NOTFOUND wasn't enough. Callers such as tools\setup-buildtools.cmd gate on if exist "%VSINSTALLDIR%" and use %VSVERSION% for the .vsconfig path, so a stale VSINSTALLDIR/VSVERSION/VSDEVCMD left in the shell (or caller environment) could make them act on the wrong install after a failed detection. Reset VSINSTALLDIR, VSDEVCMD and VSVERSION alongside VSTOOLS_NOTFOUND at entry so each run starts from a fully clean detection state; only the matching detection path repopulates them. Verified: stale VSINSTALLDIR is replaced with the real path on success and left empty when detection doesn't match (tools/setup-buildtools.cmd:42,44). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-windows-vs2022.yaml | 4 +- build-all-v142.bat | 2 +- build-all-v143.bat | 3 +- build-all-v145.bat | 6 + build-all-windows.bat | 111 ++++++++++++++++ build-all.bat | 92 +------------ docs/building-custom-SKU.md | 4 +- docs/cpp-start-windows.md | 13 +- tools/.vsconfig.vs2022 | 23 ++++ tools/.vsconfig.vs2026 | 23 ++++ tools/RunMsBuild.bat | 32 ++++- tools/build-Win10-compact-exp.cmd | 3 +- tools/setup-buildtools.cmd | 4 +- tools/vcvars.cmd | 135 ++++++++++++++++++++ 14 files changed, 350 insertions(+), 105 deletions(-) create mode 100644 build-all-v145.bat create mode 100644 build-all-windows.bat create mode 100644 tools/.vsconfig.vs2022 create mode 100644 tools/.vsconfig.vs2026 diff --git a/.github/workflows/build-windows-vs2022.yaml b/.github/workflows/build-windows-vs2022.yaml index b28aa1515..222e32e67 100644 --- a/.github/workflows/build-windows-vs2022.yaml +++ b/.github/workflows/build-windows-vs2022.yaml @@ -29,13 +29,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 - continue-on-error: true - name: Build env: SKIP_ARM_BUILD: 1 SKIP_ARM64_BUILD: 1 + SKIP_NET40_BUILD: 1 PlatformToolset: v143 VSTOOLS_VERSION: vs2022 shell: cmd - run: build-all.bat + run: build-all-windows.bat diff --git a/build-all-v142.bat b/build-all-v142.bat index 4ae0364bc..72776a978 100644 --- a/build-all-v142.bat +++ b/build-all-v142.bat @@ -2,4 +2,4 @@ set VSTOOLS_VERSION=vs2019 set PlatformToolset=v142 -call build-all.bat +call "%~dp0build-all-windows.bat" %* diff --git a/build-all-v143.bat b/build-all-v143.bat index 6ff46738d..8d5ebbfa9 100644 --- a/build-all-v143.bat +++ b/build-all-v143.bat @@ -2,4 +2,5 @@ set VSTOOLS_VERSION=vs2022 set PlatformToolset=v143 -call build-all.bat +set SKIP_NET40_BUILD=1 +call "%~dp0build-all-windows.bat" %* diff --git a/build-all-v145.bat b/build-all-v145.bat new file mode 100644 index 000000000..54f6b4e9a --- /dev/null +++ b/build-all-v145.bat @@ -0,0 +1,6 @@ +@echo off + +set VSTOOLS_VERSION=vs2026 +set PlatformToolset=v145 +set SKIP_NET40_BUILD=1 +call "%~dp0build-all-windows.bat" %* diff --git a/build-all-windows.bat b/build-all-windows.bat new file mode 100644 index 000000000..4ea3808e9 --- /dev/null +++ b/build-all-windows.bat @@ -0,0 +1,111 @@ +@echo off + +cd /d "%~dp0" +@setlocal ENABLEEXTENSIONS + +set CUSTOM_PROPS= +if not "%~1"=="" ( + if not exist "%~f1" ( + goto custom_props_missing + ) + if /I not "%~x1"==".props" ( + if /I not "%~x1"==".targets" ( + goto custom_props_invalid_type + ) + ) + set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%~f1" + echo Using custom properties file for the build: + echo "/p:ForceImportBeforeCppTargets=%~f1" +) + +goto after_custom_props_validation + +:custom_props_missing +echo ERROR: Custom build input not found: %~1 +echo Pass an existing MSBuild .props or .targets file to ForceImportBeforeCppTargets. +exit /b 1 + +:custom_props_invalid_type +echo ERROR: Custom build input must be an MSBuild .props or .targets file: %~1 +echo Pass the MSBuild import file, not the CONFIG_CUSTOM_H header. +exit /b 1 + +:after_custom_props_validation +call tools\gen-version.cmd + +set NET40_MD_TARGETS=,net40:Rebuild +set NET40_SAMPLE_TARGETS=,Samples\cs\SampleCsNet40:Rebuild +if DEFINED SKIP_NET40_BUILD ( + echo Skipping legacy .NET Framework 4.0 targets. + set NET40_MD_TARGETS= + set NET40_SAMPLE_TARGETS= +) + +echo Update all public submodules... +git -c submodule."lib/modules".update=none submodule update --init --recursive + +if DEFINED GIT_PULL_TOKEN ( + rd /s /q lib\modules + git clone https://%GIT_PULL_TOKEN%:x-oauth-basic@github.com/microsoft/cpp_client_telemetry_modules.git lib\modules +) + +set GTEST_PATH=third_party\googletest +if NOT EXIST %GTEST_PATH%\CMakeLists.txt ( + git clone --depth 1 --branch release-1.12.1 https://github.com/google/googletest %GTEST_PATH% +) + +if NOT DEFINED SKIP_MD_BUILD ( + REM DLL and static /MD build + REM Release + call tools\RunMsBuild.bat Win32 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild%NET40_SAMPLE_TARGETS%" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild%NET40_SAMPLE_TARGETS%" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + REM Debug + if NOT DEFINED SKIP_DEBUG_BUILD ( + call tools\RunMsBuild.bat Win32 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild%NET40_MD_TARGETS%,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) + +if NOT DEFINED SKIP_MT_BUILD ( + REM Static /MT build + REM Release + call tools\RunMsBuild.bat Win32 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + REM Debug + if NOT DEFINED SKIP_DEBUG_BUILD ( + call tools\RunMsBuild.bat Win32 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + call tools\RunMsBuild.bat x64 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) + +if NOT DEFINED SKIP_ARM_BUILD ( + REM ARM DLL build + REM Release + call tools\RunMsBuild.bat ARM Release "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + if NOT DEFINED SKIP_DEBUG_BUILD ( + REM Debug + call tools\RunMsBuild.bat ARM Debug "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) + +if NOT DEFINED SKIP_ARM64_BUILD ( + REM ARM64 DLL build + REM Release + call tools\RunMsBuild.bat ARM64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + if NOT DEFINED SKIP_DEBUG_BUILD ( + REM Debug + call tools\RunMsBuild.bat ARM64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% + if errorlevel 1 exit /b 1 + ) +) diff --git a/build-all.bat b/build-all.bat index 5b535bc08..547594ff6 100644 --- a/build-all.bat +++ b/build-all.bat @@ -1,91 +1,3 @@ @echo off - -cd %~dp0 -@setlocal ENABLEEXTENSIONS - -set CUSTOM_PROPS= -if not "%~1"=="" ( - if not exist "%~f1" ( - goto custom_props_missing - ) - if /I not "%~x1"==".props" ( - if /I not "%~x1"==".targets" ( - goto custom_props_invalid_type - ) - ) - set CUSTOM_PROPS="/p:ForceImportBeforeCppTargets=%~f1" - echo Using custom properties file for the build: - echo %CUSTOM_PROPS% -) - -goto after_custom_props_validation - -:custom_props_missing -echo ERROR: Custom build input not found: %~1 -echo Pass an existing MSBuild .props or .targets file to ForceImportBeforeCppTargets. -exit /b 1 - -:custom_props_invalid_type -echo ERROR: Custom build input must be an MSBuild .props or .targets file: %~1 -echo Pass the MSBuild import file, not the CONFIG_CUSTOM_H header. -exit /b 1 - -:after_custom_props_validation -call tools\gen-version.cmd - -echo Update all public submodules... -git -c submodule."lib/modules".update=none submodule update --init --recursive - -if DEFINED GIT_PULL_TOKEN ( - rd /s /q lib\modules - git clone https://%GIT_PULL_TOKEN%:x-oauth-basic@github.com/microsoft/cpp_client_telemetry_modules.git lib\modules -) - -set GTEST_PATH=third_party\googletest -if NOT EXIST %GTEST_PATH%\CMakeLists.txt ( - git clone --depth 1 --branch release-1.12.1 https://github.com/google/googletest %GTEST_PATH% -) - -if NOT DEFINED SKIP_MD_BUILD ( - REM DLL and static /MD build - REM Release - call tools\RunMsBuild.bat Win32 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild,Samples\cs\SampleCsNet40:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild,Samples\cs\SampleCsNet40:Rebuild" %CUSTOM_PROPS% - REM Debug - if NOT DEFINED SKIP_DEBUG_BUILD ( - call tools\RunMsBuild.bat Win32 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,net40:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild,Tests\gmock:Rebuild,Tests\gtest:Rebuild,Tests\UnitTests:Rebuild,Tests\FuncTests:Rebuild" %CUSTOM_PROPS% - ) -) - -if NOT DEFINED SKIP_MT_BUILD ( - REM Static /MT build - REM Release - call tools\RunMsBuild.bat Win32 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Release.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - REM Debug - if NOT DEFINED SKIP_DEBUG_BUILD ( - call tools\RunMsBuild.bat Win32 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - call tools\RunMsBuild.bat x64 Debug.vc14x.MT-sqlite "sqlite:Rebuild,zlib:Rebuild,win32-lib:Rebuild" %CUSTOM_PROPS% - ) -) - -if NOT DEFINED SKIP_ARM_BUILD ( - REM ARM DLL build - REM Release - call tools\RunMsBuild.bat ARM Release "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% - if NOT DEFINED SKIP_DEBUG_BUILD ( - REM Debug - call tools\RunMsBuild.bat ARM Debug "zlib:Rebuild,sqlite-uwp:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild" %CUSTOM_PROPS% - ) -) - -if NOT DEFINED SKIP_ARM64_BUILD ( - REM ARM64 DLL build - REM Release - call tools\RunMsBuild.bat ARM64 Release "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% - if NOT DEFINED SKIP_DEBUG_BUILD ( - REM Debug - call tools\RunMsBuild.bat ARM64 Debug "sqlite:Rebuild,zlib:Rebuild,sqlite-uwp:Rebuild,win32-dll:Rebuild,win32-lib:Rebuild,win10-cs:Rebuild,win10-dll:Rebuild,win10-lib:Rebuild" %CUSTOM_PROPS% - ) -) +echo build-all.bat is a compatibility wrapper. Use build-all-windows.bat for Windows Visual Studio builds. +call "%~dp0build-all-windows.bat" %* diff --git a/docs/building-custom-SKU.md b/docs/building-custom-SKU.md index 0668fbe3f..a68d6a681 100644 --- a/docs/building-custom-SKU.md +++ b/docs/building-custom-SKU.md @@ -39,12 +39,12 @@ Build recipe must contain the following preprocessor definitions: Command: ```console -build-all.bat %CD%\Solutions\build.compact.props +build-all-windows.bat %CD%\Solutions\build.compact.props ``` produces a custom compact SDK build. -The argument passed to `build-all.bat` must be an MSBuild `.props` or `.targets` file that sets the required preprocessor definitions. Do not pass the `config-*.h` header directly to `ForceImportBeforeCppTargets`. +The argument passed to `build-all-windows.bat` must be an MSBuild `.props` or `.targets` file that sets the required preprocessor definitions. Do not pass the `config-*.h` header directly to `ForceImportBeforeCppTargets`. `build-all.bat` remains as a compatibility wrapper for existing automation. How it works: diff --git a/docs/cpp-start-windows.md b/docs/cpp-start-windows.md index ef5850d84..6f6189056 100644 --- a/docs/cpp-start-windows.md +++ b/docs/cpp-start-windows.md @@ -16,17 +16,24 @@ If your project requires the Universal Telemetry Client (a.k.a. UTC) to send tel ## **Windows prerequisites and dependencies for building from source** -* Visual Studio 2019 or 2022 (2022 is recommended). +* Visual Studio 2019, 2022, or 2026 (2022 or newer is recommended). * C++ Dev Tools ## **Option 1: Build the SDK from source using Visual Studio** * Open the *cpp_client_telemetry/Solutions/MSTelemetrySDK.sln* solution in Visual Studio. -* Alternatively you can use *build-all.bat* located in workspace root folder to build from command line +* Alternatively, build from the workspace root with the script that matches your Visual Studio toolset: + * Visual Studio 2019: `build-all-v142.bat` + * Visual Studio 2022: `build-all-v143.bat` + * Visual Studio 2026: `build-all-v145.bat` + +The version-specific scripts set `VSTOOLS_VERSION` and `PlatformToolset` before calling `build-all-windows.bat`, which builds the Windows Visual Studio solution matrix. `build-all.bat` remains as a compatibility wrapper for existing automation; if you call either script directly, set both values yourself so `tools\vcvars.cmd` selects the same Visual Studio installation as your requested toolset. + +Visual Studio 2022 and newer may report the legacy .NET Framework 4.0 projects (`net40` and `SampleCsNet40`) as unsupported. They are only needed for the legacy .NET Framework wrapper; the VS2022 and VS2026 command-line wrappers skip those projects, and you can unload them in the IDE when building the native SDK. If your build fails, then you most likely missing the following optional Visual Studio components: -* ATL support +* MFC/ATL support (for example, `SampleCppMini` uses static MFC in its Visual Studio project) * ARM64 support * Spectre mitigation libraries diff --git a/tools/.vsconfig.vs2022 b/tools/.vsconfig.vs2022 new file mode 100644 index 000000000..48e609352 --- /dev/null +++ b/tools/.vsconfig.vs2022 @@ -0,0 +1,23 @@ +{ + "version": "1.0", + "components": [ + "Microsoft.VisualStudio.Component.Windows10SDK", + "Microsoft.VisualStudio.Component.Windows11SDK.22621", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "Microsoft.VisualStudio.Component.VC.Tools.ARM", + "Microsoft.VisualStudio.Component.VC.Tools.ARM64", + "Microsoft.VisualStudio.Component.VC.Redist.14.Latest", + "Microsoft.VisualStudio.Component.VC.CMake.Project", + "Microsoft.VisualStudio.Component.VC.ATL", + "Microsoft.VisualStudio.Component.VC.ATL.ARM", + "Microsoft.VisualStudio.Component.VC.ATL.ARM64", + "Microsoft.VisualStudio.Component.VC.ATLMFC", + "Microsoft.VisualStudio.Component.VC.CLI.Support", + "Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset", + "Microsoft.VisualStudio.Component.VC.Llvm.Clang", + "Microsoft.Component.VC.Runtime.UCRTSDK", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.ComponentGroup.UWP.VC", + "Microsoft.VisualStudio.Workload.Universal" + ] +} diff --git a/tools/.vsconfig.vs2026 b/tools/.vsconfig.vs2026 new file mode 100644 index 000000000..48e609352 --- /dev/null +++ b/tools/.vsconfig.vs2026 @@ -0,0 +1,23 @@ +{ + "version": "1.0", + "components": [ + "Microsoft.VisualStudio.Component.Windows10SDK", + "Microsoft.VisualStudio.Component.Windows11SDK.22621", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "Microsoft.VisualStudio.Component.VC.Tools.ARM", + "Microsoft.VisualStudio.Component.VC.Tools.ARM64", + "Microsoft.VisualStudio.Component.VC.Redist.14.Latest", + "Microsoft.VisualStudio.Component.VC.CMake.Project", + "Microsoft.VisualStudio.Component.VC.ATL", + "Microsoft.VisualStudio.Component.VC.ATL.ARM", + "Microsoft.VisualStudio.Component.VC.ATL.ARM64", + "Microsoft.VisualStudio.Component.VC.ATLMFC", + "Microsoft.VisualStudio.Component.VC.CLI.Support", + "Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset", + "Microsoft.VisualStudio.Component.VC.Llvm.Clang", + "Microsoft.Component.VC.Runtime.UCRTSDK", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.ComponentGroup.UWP.VC", + "Microsoft.VisualStudio.Workload.Universal" + ] +} diff --git a/tools/RunMsBuild.bat b/tools/RunMsBuild.bat index 38bab739c..2e6b9bf7a 100644 --- a/tools/RunMsBuild.bat +++ b/tools/RunMsBuild.bat @@ -4,18 +4,46 @@ set TARGETPLATFORM=%1 set CONFIGURATION=%2 set TARGETS=%~3 +call :NormalizeTargets "%TARGETS%" set CUSTOM_PROPS= -if ("%~4" == "") goto endCustomProps +if "%~4" == "" goto endCustomProps set CUSTOM_PROPS=%4 echo Using custom properties file for the build: echo %CUSTOM_PROPS% :endCustomProps call tools\vcvars.cmd +if "%VSTOOLS_NOTFOUND%"=="1" ( + echo. + echo ERROR: Visual Studio was not detected, so the build cannot continue. + echo Install Visual Studio 2019, 2022, or 2026 with the Desktop development with C++ workload, + echo or run tools\setup-buildtools.cmd to install the required build tools and components. + echo To target a specific installed version, set VSTOOLS_VERSION first, for example: + echo set VSTOOLS_VERSION=vs2022 + echo. + exit /b 1 +) set MAXCPUCOUNT=%NUMBER_OF_PROCESSORS% set platform= set SOLUTION=Solutions\MSTelemetrySDK.sln -msbuild %SOLUTION% /target:%TARGETS% /p:BuildProjectReferences=true /maxcpucount:%MAXCPUCOUNT% /detailedsummary /p:Configuration=%CONFIGURATION% /p:Platform=%TARGETPLATFORM% %CUSTOM_PROPS% \ No newline at end of file +msbuild %SOLUTION% /target:%TARGETS% /p:BuildProjectReferences=true /maxcpucount:%MAXCPUCOUNT% /detailedsummary /p:Configuration=%CONFIGURATION% /p:Platform=%TARGETPLATFORM% %CUSTOM_PROPS% +exit /b %ERRORLEVEL% + +:NormalizeTargets +setlocal ENABLEDELAYEDEXPANSION +set "TARGETS_IN=%~1" +set "NORMALIZED_TARGETS=" +for %%T in ("!TARGETS_IN:,=" "!") do ( + set "TARGET=%%~T" + if /I "!TARGET:~-6!"==":Build" set "TARGET=!TARGET:~0,-6!" + if defined NORMALIZED_TARGETS ( + set "NORMALIZED_TARGETS=!NORMALIZED_TARGETS!,!TARGET!" + ) else ( + set "NORMALIZED_TARGETS=!TARGET!" + ) +) +endlocal & set "TARGETS=%NORMALIZED_TARGETS%" +exit /b 0 \ No newline at end of file diff --git a/tools/build-Win10-compact-exp.cmd b/tools/build-Win10-compact-exp.cmd index 96b7f41ac..9b2509aad 100644 --- a/tools/build-Win10-compact-exp.cmd +++ b/tools/build-Win10-compact-exp.cmd @@ -1,3 +1,2 @@ @echo off -cd .. -build-all.bat %CD%\Solutions\build.compact-exp.props +call "%~dp0..\build-all-windows.bat" "%~dp0..\Solutions\build.compact-exp.props" diff --git a/tools/setup-buildtools.cmd b/tools/setup-buildtools.cmd index 890467256..d8a143697 100644 --- a/tools/setup-buildtools.cmd +++ b/tools/setup-buildtools.cmd @@ -38,10 +38,10 @@ if NOT exist "%VSINSTALLER%" ( echo Visual Studio installer: echo %VSINSTALLER% -REM Install optional components required for ARM build - vs2017-BuildTools +REM Install optional components required for supported Windows build targets. if exist "%VSINSTALLDIR%" ( echo Running Visual Studio installer.. - "%VSINSTALLER%" modify --installPath "%VSINSTALLDIR%" --config "%~dp0\.vsconfig.%VSVERSION%" --force --quiet --norestart + "%VSINSTALLER%" modify --installPath "%VSINSTALLDIR%" --config "%~dp0\.vsconfig.vs%VSVERSION%" --force --quiet --norestart ) where /Q vcpkg.exe diff --git a/tools/vcvars.cmd b/tools/vcvars.cmd index 18ae09e0a..ea0ec1c38 100644 --- a/tools/vcvars.cmd +++ b/tools/vcvars.cmd @@ -8,9 +8,34 @@ REM 2. Visual Studio 2017 BuildTools REM 3. Visual Studio 2019 Enterprise REM 4. Visual Studio 2019 Community REM 5. Visual Studio 2019 BuildTools +REM 6. Visual Studio 2022 Enterprise +REM 7. Visual Studio 2022 Professional +REM 8. Visual Studio 2022 Community +REM 9. Visual Studio 2022 BuildTools +REM 10. Visual Studio 2026 Enterprise +REM 11. Visual Studio 2026 Professional +REM 12. Visual Studio 2026 Community +REM 13. Visual Studio 2026 BuildTools REM REM 1st parameter - Visual Studio version + +REM Start from a clean detection state so values left in the shell by a previous +REM run (or the caller environment) can't be mistaken for the result of this run. +REM Only the matching detection path below sets these again; in particular this +REM prevents a stale VSINSTALLDIR/VSVERSION from making callers such as +REM tools\setup-buildtools.cmd act on the wrong install after a failed detection. +set "VSTOOLS_NOTFOUND=" +set "VSINSTALLDIR=" +set "VSDEVCMD=" +set "VSVERSION=" + +REM Remember an explicit version request so we can warn later if detection falls +REM back to a different Visual Studio install than the one that was asked for. +set "VSTOOLS_REQUESTED=" +if "%1" neq "" set "VSTOOLS_REQUESTED=%1" +if not defined VSTOOLS_REQUESTED if "%VSTOOLS_VERSION%" neq "" set "VSTOOLS_REQUESTED=%VSTOOLS_VERSION%" + if "%1" neq "" ( goto %1 ) @@ -84,6 +109,16 @@ if exist "%VSDEVCMD%" ( goto tools_configured ) +:vs2022_professional +SET VSVERSION=2022 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2022\Professional\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2022\Professional" + echo Building with vs2022 Professional... + call "%VSDEVCMD%" + goto tools_configured +) + :vs2022_community SET VSVERSION=2022 set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat" @@ -94,6 +129,89 @@ if exist "%VSDEVCMD%" ( goto tools_configured ) +:vs2022_buildtools +SET VSVERSION=2022 +set "VSDEVCMD=%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles(x86)%\Microsoft Visual Studio\2022\BuildTools" + echo Building with vs2022 BuildTools... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026 +:vs2026_enterprise +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2026\Enterprise\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2026\Enterprise" + echo Building with vs2026 Enterprise... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\18\Enterprise\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\18\Enterprise" + echo Building with vs2026 Enterprise... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026_professional +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2026\Professional\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2026\Professional" + echo Building with vs2026 Professional... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\18\Professional\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\18\Professional" + echo Building with vs2026 Professional... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026_community +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\2026\Community\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\2026\Community" + echo Building with vs2026 Community... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles%\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles%\Microsoft Visual Studio\18\Community" + echo Building with vs2026 Community... + call "%VSDEVCMD%" + goto tools_configured +) + +:vs2026_buildtools +SET VSVERSION=2026 +set "VSDEVCMD=%ProgramFiles(x86)%\Microsoft Visual Studio\2026\BuildTools\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles(x86)%\Microsoft Visual Studio\2026\BuildTools" + echo Building with vs2026 BuildTools... + call "%VSDEVCMD%" + goto tools_configured +) + +set "VSDEVCMD=%ProgramFiles(x86)%\Microsoft Visual Studio\18\BuildTools\Common7\Tools\VsDevCmd.bat" +if exist "%VSDEVCMD%" ( + set "VSINSTALLDIR=%ProgramFiles(x86)%\Microsoft Visual Studio\18\BuildTools" + echo Building with vs2026 BuildTools... + call "%VSDEVCMD%" + goto tools_configured +) + echo WARNING:********************************************* echo WARNING: cannot auto-detect Visual Studio version !!! echo WARNING:********************************************* @@ -102,3 +220,20 @@ set VSVERSION= exit /b 0 :tools_configured + +REM Warn if an explicit version was requested but detection fell back to a +REM different Visual Studio install (the label cascade silently moves forward, +REM e.g. a vs2022 request can end up on vs2026), which can lead to a confusing +REM toolset mismatch later when PlatformToolset is pinned to the requested version. +if not defined VSTOOLS_REQUESTED goto :tools_configured_done +if not defined VSVERSION goto :tools_configured_done +echo "%VSTOOLS_REQUESTED%" | findstr /I /C:"%VSVERSION%" >nul +if errorlevel 1 ( + echo WARNING: Requested Visual Studio "%VSTOOLS_REQUESTED%" was not found; using Visual Studio %VSVERSION% instead. + echo WARNING: If a specific toolset is required, install that Visual Studio version or set VSTOOLS_VERSION/PlatformToolset to match what is installed. +) + +:tools_configured_done +REM Visual Studio was configured; callers rely on VSTOOLS_NOTFOUND rather than +REM this script's exit code, so return success regardless of the version probe. +exit /b 0 From ecd8ccb4401a1a30631c994413a2d6debe4dd949 Mon Sep 17 00:00:00 2001 From: Matthew Koscumb <49213788+mkoscumb@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:48:43 -0700 Subject: [PATCH 08/30] Add noexcept to non-throwing methods (+ noexcept move ops and LogSessionData parse hardening) (#1179) * Add noexcept on non throwing methods identified by static analysis * LogSessionDataProvider: harden convertStrToLong (reset errno, reject negatives) Addresses the Copilot low-confidence (suppressed) review note on convertStrToLong: - strtoll's errno was checked without being reset first, so a stale errno from an earlier call could trip false "conversion failed" handling. errno is now cleared before the call. - a negative input wrapped silently into a large uint64_t. The value is now parsed into a signed temp and rejected (returns 0 + warns) if negative. - the imprecise res==LONG_MAX overflow heuristic is replaced by a direct errno==ERANGE check plus an explicit no-conversion / trailing-character check. noexcept is preserved (no throwing operations). Pre-existing logic, folded in here since this PR already annotates the same function. Verified: NDK aarch64-linux-android23 -fsyntax-only clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * LogSessionDataProvider: parse with strtoull to match uint64_t return type Follow-up to the Copilot review on convertStrToLong: the function returns uint64_t but parsed via std::strtoll (signed long long), which constrains the accepted range to LLONG_MAX and mixes signed/unsigned. Switch to std::strtoull so parsing matches the return type and the full uint64_t range is accepted. strtoull silently wraps a leading '-', so negatives are now rejected explicitly (first non-space char check) before parsing, preserving the earlier negative-rejection behavior. noexcept preserved. Verified: NDK aarch64-linux-android23 -fsyntax-only clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * LogSessionDataProvider: mark remove_eol noexcept remove_eol only inspects and shrinks the string in place (empty(), operator[], length(), and erase() at a validated position) -- none of which allocate or throw -- so it is genuinely non-throwing. Extends this PR's noexcept coverage to the one remaining sibling helper in the session-data classes that is safely non-throwing (parse()/writeFileContents()/the std::string ctor allocate, so they correctly stay un-annotated). Verified NDK aarch64-linux-android23 -fsyntax-only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * EventProperties: add noexcept move constructor and move assignment EventProperties is pimpl (single EventPropertiesStorage* m_storage) with a user-declared copy ctor/assign + virtual dtor, so the compiler generated no move operations -- every pass/return/vector-growth deep-copied the whole property map via `new EventPropertiesStorage(*copy.m_storage)`. Add O(1) noexcept move ctor + move assignment that transfer the storage pointer. The dtor is already null-safe (delete nullptr); copy-assignment is now also null-safe so a moved-from object can be reassigned. Backward-compatible API addition (the single-pointer layout is unchanged). Verified: NDK aarch64-linux-android23 -fsyntax-only clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * LogSessionDataProvider: fix %d format specifier for const char* names The three failure-path LOG_WARN calls passed the static const char* setting names (sessionFirstLaunchTimeName/sessionSdkUidName) to a %d conversion, which is undefined behavior and logs a garbage integer instead of the setting name exactly when a store/delete failed. Line 71 already uses %s with .c_str(); make these three consistent with %s. Folded into this PR since it already hardens this file. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Lalit Kumar Bhasin Co-authored-by: Bhagirath Mehta Co-authored-by: bmehta001 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/api/LogManagerImpl.cpp | 8 ++--- lib/api/LogSessionData.cpp | 2 +- lib/include/public/EventProperties.hpp | 12 ++++++++ lib/include/public/LogSessionData.hpp | 2 +- lib/offline/LogSessionDataProvider.cpp | 41 +++++++++++++++----------- lib/offline/LogSessionDataProvider.hpp | 8 ++--- lib/system/EventProperties.cpp | 27 ++++++++++++++++- lib/system/TelemetrySystemBase.hpp | 10 +++---- lib/utils/StringUtils.cpp | 10 +++---- lib/utils/StringUtils.hpp | 6 ++-- lib/utils/Utils.cpp | 2 +- lib/utils/Utils.hpp | 2 +- lib/utils/annex_k.hpp | 8 ++--- tests/common/Common.cpp | 4 +-- tests/common/Common.hpp | 4 +-- tests/common/MockIRuntimeConfig.hpp | 2 +- 16 files changed, 96 insertions(+), 52 deletions(-) diff --git a/lib/api/LogManagerImpl.cpp b/lib/api/LogManagerImpl.cpp index 2f0e8933d..24215c0cd 100644 --- a/lib/api/LogManagerImpl.cpp +++ b/lib/api/LogManagerImpl.cpp @@ -839,7 +839,7 @@ namespace MAT_NS_BEGIN return; } - auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&dataInspector](const std::shared_ptr& currentInspector) + auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&dataInspector](const std::shared_ptr& currentInspector) noexcept { return strcmp(dataInspector->GetName(), currentInspector->GetName()) == 0; }); @@ -862,7 +862,7 @@ namespace MAT_NS_BEGIN void LogManagerImpl::RemoveDataInspector(const std::string& name) { LOCKGUARD(m_dataInspectorGuard); - auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector){ + auto itDataInspector = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector) noexcept { return strcmp(inspector->GetName(), name.c_str()) == 0; }); @@ -875,7 +875,7 @@ namespace MAT_NS_BEGIN std::shared_ptr LogManagerImpl::GetDataInspector(const std::string& name) noexcept { LOCKGUARD(m_dataInspectorGuard); - auto it = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector){ + auto it = std::find_if(m_dataInspectors.begin(), m_dataInspectors.end(), [&name](const std::shared_ptr& inspector) noexcept{ return strcmp(inspector->GetName(), name.c_str()) == 0; }); @@ -944,7 +944,7 @@ namespace MAT_NS_BEGIN if (m_pause_state != PauseState::Pausing) { return; } - m_pause_cv.wait(lock, [this]() -> bool { + m_pause_cv.wait(lock, [this]() noexcept -> bool { return m_pause_state != PauseState::Pausing; }); } diff --git a/lib/api/LogSessionData.cpp b/lib/api/LogSessionData.cpp index 9ea280f6a..34f6538b2 100644 --- a/lib/api/LogSessionData.cpp +++ b/lib/api/LogSessionData.cpp @@ -10,7 +10,7 @@ using namespace std; namespace MAT_NS_BEGIN { - uint64_t LogSessionData::getSessionFirstTime() const + uint64_t LogSessionData::getSessionFirstTime() const noexcept { return m_sessionFirstTimeLaunch; } diff --git a/lib/include/public/EventProperties.hpp b/lib/include/public/EventProperties.hpp index a2aa3ecda..ef7f3b705 100644 --- a/lib/include/public/EventProperties.hpp +++ b/lib/include/public/EventProperties.hpp @@ -60,6 +60,18 @@ namespace MAT_NS_BEGIN /// EventProperties& operator=(EventProperties const& copy); + /// + /// The EventProperties move constructor. Transfers ownership of the + /// underlying storage (O(1)); the moved-from object is left empty and is + /// only valid to destroy or reassign. + /// + EventProperties(EventProperties&& move) noexcept; + + /// + /// The EventProperties move-assignment operator. + /// + EventProperties& operator=(EventProperties&& move) noexcept; + /// /// Constructs an EventProperties object from a map of string to EventProperty.
/// You must supply a non-empty name whenever you supply any custom properties for the event via EventProperties. diff --git a/lib/include/public/LogSessionData.hpp b/lib/include/public/LogSessionData.hpp index 024f2272f..0a6737f36 100644 --- a/lib/include/public/LogSessionData.hpp +++ b/lib/include/public/LogSessionData.hpp @@ -29,7 +29,7 @@ namespace MAT_NS_BEGIN /// Gets the time that this session began. /// /// A 64-bit integer that contains the time. - uint64_t getSessionFirstTime() const; + uint64_t getSessionFirstTime() const noexcept; /// /// Gets the SDK unique identifier. diff --git a/lib/offline/LogSessionDataProvider.cpp b/lib/offline/LogSessionDataProvider.cpp index 1615d84e7..68e152d0e 100644 --- a/lib/offline/LogSessionDataProvider.cpp +++ b/lib/offline/LogSessionDataProvider.cpp @@ -49,7 +49,7 @@ namespace MAT_NS_BEGIN } } - LogSessionData* LogSessionDataProvider::GetLogSessionData() + LogSessionData* LogSessionDataProvider::GetLogSessionData() noexcept { return m_logSessionData.get(); } @@ -69,7 +69,7 @@ namespace MAT_NS_BEGIN sessionSDKUid = PAL::generateUuidString(); if (!m_offlineStorage->StoreSetting(sessionFirstLaunchTimeName, std::to_string(sessionFirstTimeLaunch))) { - LOG_WARN("Unable to save session analytics to DB for %d", sessionFirstLaunchTimeName); + LOG_WARN("Unable to save session analytics to DB for %s", sessionFirstLaunchTimeName); } if (!m_offlineStorage->StoreSetting(sessionSdkUidName, sessionSDKUid)) { LOG_WARN("Unable to save session analytics to DB for %s", sessionSDKUid.c_str()); @@ -87,11 +87,11 @@ namespace MAT_NS_BEGIN } if (!m_offlineStorage->DeleteSetting(sessionFirstLaunchTimeName)) { - LOG_WARN("Unable to delete session analytics from DB for %d", sessionFirstLaunchTimeName); + LOG_WARN("Unable to delete session analytics from DB for %s", sessionFirstLaunchTimeName); } if (!m_offlineStorage->DeleteSetting(sessionSdkUidName)) { - LOG_WARN("Unable to delete session analytics from DB for %d", sessionSdkUidName); + LOG_WARN("Unable to delete session analytics from DB for %s", sessionSdkUidName); } } @@ -153,25 +153,32 @@ namespace MAT_NS_BEGIN return true; } - uint64_t LogSessionDataProvider::convertStrToLong(const std::string& s) + uint64_t LogSessionDataProvider::convertStrToLong(const std::string& s) noexcept { uint64_t res = 0ull; char *endptr = nullptr; - res = std::strtoll(s.c_str(), &endptr, 10); - if (errno == ERANGE && (res == LONG_MAX || res == 0 )) + // strtoull silently wraps a leading '-' into a large value, so reject + // negative input explicitly before parsing. + size_t firstNonSpace = s.find_first_not_of(" \t\n\r\f\v"); + if (firstNonSpace != std::string::npos && s[firstNonSpace] == '-') { - LOG_WARN ("Converted value falls out of uint64_t range."); - res = 0; - } - else if ( 0 != errno && 0 == res ) + LOG_WARN ("Converted value is negative; rejecting."); + return 0; + } + errno = 0; + unsigned long long parsed = std::strtoull(s.c_str(), &endptr, 10); + if (errno == ERANGE) { - LOG_WARN("Conversion cannot be performed."); + LOG_WARN ("Converted value falls out of range."); } - else if (std::strlen(endptr) > 0) + else if (endptr == s.c_str() || std::strlen(endptr) > 0) { - LOG_WARN ("Conversion cannot be performed. Alphanumeric characters present"); - res = 0; - } + LOG_WARN ("Conversion cannot be performed."); + } + else + { + res = static_cast(parsed); + } return res; } @@ -193,7 +200,7 @@ namespace MAT_NS_BEGIN } } - void LogSessionDataProvider::remove_eol(std::string& result) + void LogSessionDataProvider::remove_eol(std::string& result) noexcept { if (!result.empty() && result[result.length() - 1] == '\n') { diff --git a/lib/offline/LogSessionDataProvider.hpp b/lib/offline/LogSessionDataProvider.hpp index be782095e..3b45a9fc8 100644 --- a/lib/offline/LogSessionDataProvider.hpp +++ b/lib/offline/LogSessionDataProvider.hpp @@ -21,7 +21,7 @@ namespace MAT_NS_BEGIN { public: LogSessionDataProvider( - IOfflineStorage* offlineStorage) + IOfflineStorage* offlineStorage) noexcept : m_offlineStorage(offlineStorage), m_storageType(SessionStorageType::DatabaseStore), @@ -41,7 +41,7 @@ namespace MAT_NS_BEGIN void CreateLogSessionData(); void ResetLogSessionData(); void DeleteLogSessionData(); - LogSessionData *GetLogSessionData(); + LogSessionData *GetLogSessionData() noexcept; protected: void CreateLogSessionDataFromFile(); @@ -55,9 +55,9 @@ namespace MAT_NS_BEGIN std::string const m_cacheFilePath; SessionStorageType m_storageType; std::unique_ptr m_logSessionData; - static uint64_t convertStrToLong(const std::string&); + static uint64_t convertStrToLong(const std::string&) noexcept; static void writeFileContents(const std::string&, uint64_t, const std::string&); - void remove_eol(std::string& ); + void remove_eol(std::string& ) noexcept; }; } MAT_NS_END diff --git a/lib/system/EventProperties.cpp b/lib/system/EventProperties.cpp index ebe62ff58..2ade77741 100644 --- a/lib/system/EventProperties.cpp +++ b/lib/system/EventProperties.cpp @@ -90,8 +90,33 @@ namespace MAT_NS_BEGIN { EventProperties& EventProperties::operator=(EventProperties const& copy) { - *m_storage = *copy.m_storage; + // m_storage may be null if this object was moved-from; reallocate then. + if (m_storage == nullptr) + { + m_storage = new EventPropertiesStorage(*copy.m_storage); + } + else + { + *m_storage = *copy.m_storage; + } + + return *this; + } + + EventProperties::EventProperties(EventProperties&& move) noexcept + : m_storage(move.m_storage) + { + move.m_storage = nullptr; + } + EventProperties& EventProperties::operator=(EventProperties&& move) noexcept + { + if (this != &move) + { + delete m_storage; + m_storage = move.m_storage; + move.m_storage = nullptr; + } return *this; } diff --git a/lib/system/TelemetrySystemBase.hpp b/lib/system/TelemetrySystemBase.hpp index 30fd1d5af..fba193b1b 100644 --- a/lib/system/TelemetrySystemBase.hpp +++ b/lib/system/TelemetrySystemBase.hpp @@ -36,11 +36,11 @@ namespace MAT_NS_BEGIN { m_isPaused(false), stats(*this, taskDispatcher) { - onStart = []() { return true; }; - onStop = []() { return true; }; - onPause = []() { return true; }; - onResume = []() { return true; }; - onCleanup = []() { return true; }; + onStart = []() noexcept { return true; }; + onStop = []() noexcept { return true; }; + onPause = []() noexcept { return true; }; + onResume = []() noexcept { return true; }; + onCleanup = []() noexcept { return true; }; } /// diff --git a/lib/utils/StringUtils.cpp b/lib/utils/StringUtils.cpp index 7ee3318f3..d47d3c0b2 100644 --- a/lib/utils/StringUtils.cpp +++ b/lib/utils/StringUtils.cpp @@ -31,7 +31,7 @@ namespace MAT_NS_BEGIN } } - bool StringUtils::AreAllCharactersAllowlisted(const string& stringToTest, const string& allowlist) + bool StringUtils::AreAllCharactersAllowlisted(const string& stringToTest, const string& allowlist) noexcept { return (stringToTest.find_first_not_of(allowlist) == string::npos); } @@ -132,7 +132,7 @@ namespace MAT_NS_BEGIN { std::string result = str; std::transform(str.begin(), str.end(), result.begin(), - [](unsigned char c) { return (char)::tolower(c); }); + [](unsigned char c) noexcept { return (char)::tolower(c); }); return result; } @@ -140,7 +140,7 @@ namespace MAT_NS_BEGIN { std::string result = str; std::transform(str.begin(), str.end(), result.begin(), - [](unsigned char c) { return (char)::toupper(c); }); + [](unsigned char c) noexcept { return (char)::toupper(c); }); return result; } @@ -158,7 +158,7 @@ namespace MAT_NS_BEGIN return str; } - const char* priorityToStr(EventPriority priority) + const char* priorityToStr(EventPriority priority) noexcept { switch (priority) { @@ -185,7 +185,7 @@ namespace MAT_NS_BEGIN } } - const char* latencyToStr(EventLatency latency) + const char* latencyToStr(EventLatency latency) noexcept { switch (latency) { diff --git a/lib/utils/StringUtils.hpp b/lib/utils/StringUtils.hpp index 464f551f1..dc7c42bc8 100644 --- a/lib/utils/StringUtils.hpp +++ b/lib/utils/StringUtils.hpp @@ -17,7 +17,7 @@ namespace MAT_NS_BEGIN namespace StringUtils { void SplitString(const std::string& s, const char separator, std::vector& parts); - bool AreAllCharactersAllowlisted(const std::string& stringToTest, const std::string& allowlist); + bool AreAllCharactersAllowlisted(const std::string& stringToTest, const std::string& allowlist) noexcept; } std::string toString(char const* value); @@ -44,9 +44,9 @@ namespace MAT_NS_BEGIN std::string sanitizeIdentifier(const std::string& str); - const char* priorityToStr(EventPriority priority); + const char* priorityToStr(EventPriority priority) noexcept; - const char* latencyToStr(EventLatency latency); + const char* latencyToStr(EventLatency latency) noexcept; bool replace(std::string& str, const std::string& from, const std::string& to); diff --git a/lib/utils/Utils.cpp b/lib/utils/Utils.cpp index e2360ca18..22a48d87f 100644 --- a/lib/utils/Utils.cpp +++ b/lib/utils/Utils.cpp @@ -74,7 +74,7 @@ namespace MAT_NS_BEGIN { #endif } - bool IsRunningInApp() + bool IsRunningInApp() noexcept { #ifdef _WINRT_DLL // Win 10 UWP typedef LONG (*LPFN_GPFN)(UINT32*, PWSTR); diff --git a/lib/utils/Utils.hpp b/lib/utils/Utils.hpp index 78c47201a..ce249b9ab 100644 --- a/lib/utils/Utils.hpp +++ b/lib/utils/Utils.hpp @@ -67,7 +67,7 @@ namespace MAT_NS_BEGIN { long GetCurrentProcessId(); /* Detects if current process is running in a packaged app*/ - bool IsRunningInApp(); + bool IsRunningInApp() noexcept; std::string GetTempDirectory(); std::string GetAppLocalTempDirectory(); diff --git a/lib/utils/annex_k.hpp b/lib/utils/annex_k.hpp index 4e9fef9ed..5aa4b73af 100644 --- a/lib/utils/annex_k.hpp +++ b/lib/utils/annex_k.hpp @@ -45,7 +45,7 @@ namespace MAT_NS_BEGIN class BoundCheckFunctions { private: -static bool oneds_buffer_region_overlap(const char *buffer1, size_t buffer1_len, const char *buffer2, size_t buffer2_len) +static bool oneds_buffer_region_overlap(const char *buffer1, size_t buffer1_len, const char *buffer2, size_t buffer2_len) noexcept { if (buffer2 >= buffer1) { @@ -70,7 +70,7 @@ static bool oneds_buffer_region_overlap(const char *buffer1, size_t buffer1_len, // - returns zero if str is a null pointer // - returns strsz if the null character was not found in the first strsz bytes of str. -static size_t oneds_strnlen_s(const char *str, size_t strsz) +static size_t oneds_strnlen_s(const char *str, size_t strsz) noexcept { if ( str == NULL) { @@ -89,7 +89,7 @@ static size_t oneds_strnlen_s(const char *str, size_t strsz) // - count is greater than RSIZE_MAX // - count is greater or equal destsz, but destsz is less or equal strnlen_s(src, count), in other words, truncation would occur // - overlap would occur between the source and the destination strings -static errno_t oneds_strncpy_s(char * restrict dest, rsize_t destsz, const char *restrict src, rsize_t count) +static errno_t oneds_strncpy_s(char * restrict dest, rsize_t destsz, const char *restrict src, rsize_t count) noexcept { #if (defined __STDC_LIB_EXT1__) || ( defined _MSC_VER) return strncpy_s(dest, destsz, src, count); @@ -148,7 +148,7 @@ static errno_t oneds_strncpy_s(char * restrict dest, rsize_t destsz, const char // (if both dest and destsz are valid)) static errno_t oneds_memcpy_s( void *restrict dest, rsize_t destsz, - const void *restrict src, rsize_t count ) + const void *restrict src, rsize_t count ) noexcept { #if (defined __STDC_LIB_EXT1__) || ( defined _MSC_VER) return memcpy_s(dest, destsz, src, count); diff --git a/tests/common/Common.cpp b/tests/common/Common.cpp index f6023f656..0200402b9 100644 --- a/tests/common/Common.cpp +++ b/tests/common/Common.cpp @@ -268,7 +268,7 @@ namespace testing { return fname; } - void LogMemUsage(const char* label) + void LogMemUsage(const char* label) noexcept { #ifdef DEBUG_PERF #ifdef _WIN32 @@ -295,7 +295,7 @@ namespace testing { #endif } - void LogCpuUsage(const char* label) + void LogCpuUsage(const char* label) noexcept { #ifdef DEBUG_PERF static int64_t lastTime = GetUptimeMs(); diff --git a/tests/common/Common.hpp b/tests/common/Common.hpp index 754040ff8..da7f2903e 100644 --- a/tests/common/Common.hpp +++ b/tests/common/Common.hpp @@ -75,9 +75,9 @@ namespace testing { LogMemUsage(label); \ LogCpuUsage(label); - void LogMemUsage(const char* label); + void LogMemUsage(const char* label) noexcept; - void LogCpuUsage(const char* label); + void LogCpuUsage(const char* label) noexcept; void InflateVector(std::vector &in, std::vector &out, bool isGzip = false); } // namespace testing diff --git a/tests/common/MockIRuntimeConfig.hpp b/tests/common/MockIRuntimeConfig.hpp index 4a7509c84..a52ef8e8d 100644 --- a/tests/common/MockIRuntimeConfig.hpp +++ b/tests/common/MockIRuntimeConfig.hpp @@ -19,7 +19,7 @@ namespace testing { class MockIRuntimeConfig : public MAT::RuntimeConfig_Default /* MAT::IRuntimeConfig */ { protected: - std::unique_ptr& GetStaticConfig() + std::unique_ptr& GetStaticConfig() noexcept { static std::unique_ptr staticConfig; return staticConfig; From 9565cb4c253f29558c86acd42c13b32fc8ebd9fc Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 1 Jul 2026 01:30:57 -0500 Subject: [PATCH 09/30] Decorator: move CsProtocol::Value temporaries into ext maps instead of copying (#1498) * Decorator: move CsProtocol::Value temporaries into the ext maps EventPropertiesDecorator builds a throwaway CsProtocol::Value (or the Part B map) for every event property and copy-assigned it into the ext/extPartB maps. Each value is a local that is not used after insertion, so move it instead of copying. CsProtocol::Value is a heavy type (vectors of attributes/PII plus strings), and this runs on the per-event decorate path. Pure move-instead-of-copy of throwaway locals; no interface or behavior change. EventProperties decorator/serialization tests (54) pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot comment: include for std::move The decorator now uses std::move; add the explicit include instead of relying on transitive includes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/decorators/EventPropertiesDecorator.hpp | 53 +++++++++++---------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/lib/decorators/EventPropertiesDecorator.hpp b/lib/decorators/EventPropertiesDecorator.hpp index 800dc1635..91c5f74b4 100644 --- a/lib/decorators/EventPropertiesDecorator.hpp +++ b/lib/decorators/EventPropertiesDecorator.hpp @@ -15,6 +15,7 @@ #include #include #include +#include namespace MAT_NS_BEGIN { @@ -188,11 +189,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } } @@ -210,11 +211,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } #if 0 /* v2 code */ if (v.piiKind != PiiKind_None) @@ -252,11 +253,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -267,11 +268,11 @@ namespace MAT_NS_BEGIN { temp.longValue = v.as_int64; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -282,11 +283,11 @@ namespace MAT_NS_BEGIN { temp.doubleValue = v.as_double; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -297,11 +298,11 @@ namespace MAT_NS_BEGIN { temp.longValue = v.as_time_ticks.ticks; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -312,11 +313,11 @@ namespace MAT_NS_BEGIN { temp.longValue = v.as_bool; if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -346,11 +347,11 @@ namespace MAT_NS_BEGIN { temp.longArray.push_back(*v.as_longArray); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -361,11 +362,11 @@ namespace MAT_NS_BEGIN { temp.doubleArray.push_back(*v.as_doubleArray); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -376,11 +377,11 @@ namespace MAT_NS_BEGIN { temp.stringArray.push_back(*v.as_stringArray); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -399,11 +400,11 @@ namespace MAT_NS_BEGIN { temp.guidArray.push_back(values); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } break; } @@ -414,11 +415,11 @@ namespace MAT_NS_BEGIN { temp.stringValue = v.to_string(); if (v.dataCategory == DataCategory_PartB) { - extPartB[k] = temp; + extPartB[k] = std::move(temp); } else { - ext[k] = temp; + ext[k] = std::move(temp); } } } @@ -428,8 +429,8 @@ namespace MAT_NS_BEGIN { if (extPartB.size() > 0) { ::CsProtocol::Data partBdata; - partBdata.properties = extPartB; - record.baseData.push_back(partBdata); + partBdata.properties = std::move(extPartB); + record.baseData.push_back(std::move(partBdata)); } // special case of CorrelationVector value From e19b1cca82ee2e7e1699440ecc20aece9fd445de Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 1 Jul 2026 06:55:19 -0500 Subject: [PATCH 10/30] Clean up SQLite companion files when recreating a corrupt database (#1484) OfflineStorageTests_SQLite.InitializeDeletesFileAndCreatesNewIfFailed fails only on iOS: after a corrupt DB is detected, recreate() opens with deletePrevious=true, but the open then fails and OnStorageOpened reports "SQLite/None" instead of the expected "SQLite/Clean". Root cause hypothesis: deletePrevious only removed the main database file via the SQLite VFS xDelete. A stale -journal/-wal/-shm companion left behind by the failed first open can prevent the freshly created database from opening cleanly. This is benign on Windows/Linux (where the test passes) but trips the iOS VFS. Fix: in SQLiteWrapper::initialize's deletePrevious path, delete the main DB file plus its -journal/-wal/-shm companions. Only a failure to delete the main file is fatal; a leftover companion that cannot be removed no longer aborts the recreate, since the subsequent open may still succeed. No desktop regression: built UnitTests (Release x64, v145) and ran the full OfflineStorageTests_SQLite suite -- 43/43 pass on Windows. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/offline/SQLiteWrapper.hpp | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/lib/offline/SQLiteWrapper.hpp b/lib/offline/SQLiteWrapper.hpp index 3f4f998e3..982b4053b 100644 --- a/lib/offline/SQLiteWrapper.hpp +++ b/lib/offline/SQLiteWrapper.hpp @@ -249,15 +249,36 @@ namespace MAT_NS_BEGIN { // We cannot call plain ::remove() here, filename is in UTF-8. Rather // than adding a new set of functions to PAL, let's use SQLite VFS. sqlite3_vfs* vfs = g_sqlite3Proxy->sqlite3_vfs_find(NULL); - result = (vfs != NULL) ? vfs->xDelete(vfs, filename.c_str(), 0) : SQLITE_ERROR; - if (result == SQLITE_OK) { - LOG_INFO("Unusable existing database file was successfully deleted"); - } - else if (result != SQLITE_IOERR_DELETE_NOENT) { - LOG_WARN("Failed to delete unusable database file (%d)", result); + if (vfs == NULL) { + LOG_ERROR("Failed to delete unusable database file: no SQLite VFS"); shutdown_sqlite(); return false; } + // Delete the main database file plus any SQLite companion files + // (-journal/-wal/-shm) left behind by the failed open. A stale + // rollback journal or WAL can otherwise prevent the freshly created + // database below from opening cleanly (observed on iOS, where leaving + // the companions behind made the recreate() open fail). xDelete + // returns SQLITE_IOERR_DELETE_NOENT when a file is already absent, + // which is expected and not an error. + static const char* const companionSuffixes[] = { "", "-journal", "-wal", "-shm" }; + for (const char* suffix : companionSuffixes) { + const std::string companion = filename + suffix; + result = vfs->xDelete(vfs, companion.c_str(), 0); + if (result == SQLITE_OK) { + LOG_INFO("Deleted unusable database file \"%s\"", suffix); + } + else if (result != SQLITE_IOERR_DELETE_NOENT) { + LOG_WARN("Failed to delete database file \"%s\" (%d)", suffix, result); + // Only the main database file is fatal here; a leftover + // companion that cannot be removed must not by itself abort + // the recreate, since the open below may still succeed. + if (suffix[0] == '\0') { + shutdown_sqlite(); + return false; + } + } + } } // Take basename only, potential PII like profile name in the path must not be logged From 465106207bffdaefffc3b9371e832b3e43aa48f8 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 1 Jul 2026 08:09:18 -0500 Subject: [PATCH 11/30] Fix null EventProperties.getType() on Android (initialize eventType) (#1483) * android: initialize EventPropertiesStorage.eventType to fix null getType() EventPropertiesStorage's default constructor initialized every member except eventType, so EventProperties.getType() returned null on a freshly constructed object instead of its documented empty-string default (#1329). On Android/Java this surfaced as a NullPointerException for callers. - EventPropertiesStorage(): initialize eventType = "" (matching eventName and the other members). - EventsUnitTest: add newEventPropertiesGetTypeReturnsEmptyString as a regression test (getType() on a new EventProperties is non-null and ""). Resolves #1329. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Don't forward an empty event type to native SetType (Android) Code review of the #1329 fix found a native-path regression. Making Java getType() return "" (instead of null) fixes the Java-side NPE, but the JNI converter forwarded the type to native whenever the jstring was non-null: if (jstrEventType != NULL) eventProperties.SetType(JStringToStdString(env, jstrEventType)); Before #1329, getType() returned null for a default EventProperties, so SetType was skipped and native eventType stayed at its "" default. Now the non-null "" reaches native SetType(""), which fails validateEventName (length < 4) and, for EVERY typeless event, logs "Invalid event type!" and broadcasts an EVT_REJECTED DebugEvent to all registered listeners (EventProperties.cpp SetType -> ILManager::DispatchEventBroadcast) -- a false-positive rejection signal indistinguishable from a genuinely rejected event. Fix: in the JNI converter (the sole native SetType chokepoint), treat an empty type as "unset" and only call SetType for a non-empty string, restoring the pre-#1329 native behavior while keeping the corrected Java API contract. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot on #1483: test EventPropertiesStorage directly (no native) The regression test constructed EventProperties, whose constructor calls setName() -> native Utils.validateEventName(). These are JVM unit tests (@RunWith(MockitoJUnitRunner)) with no native library loaded, so it would throw UnsatisfiedLinkError instead of exercising the regression. Test the pure-Java EventPropertiesStorage (same package) directly -- the exact class the #1329 fix initialized (eventType = ""). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../applications/events/EventPropertiesStorage.java | 1 + .../applications/events/EventsUnitTest.java | 13 +++++++++++++ lib/jni/JniConvertors.cpp | 13 +++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java index b5b7929e8..9641caa67 100644 --- a/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java +++ b/lib/android_build/maesdk/src/main/java/com/microsoft/applications/events/EventPropertiesStorage.java @@ -21,6 +21,7 @@ class EventPropertiesStorage { EventPropertiesStorage() { eventName = ""; + eventType = ""; eventLatency = EventLatency.Normal; eventPersistence = EventPersistence.Normal; eventPopSample = 100; diff --git a/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java b/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java index 24bfa90ea..513354682 100644 --- a/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java +++ b/lib/android_build/maesdk/src/test/java/com/microsoft/applications/events/EventsUnitTest.java @@ -390,4 +390,17 @@ public void requestException() throws java.io.IOException, PackageManager.NameNo } } + @Test + public void newEventPropertiesStorageEventTypeDefaultsToEmptyString() { + // Regression test for #1329: EventPropertiesStorage previously left + // eventType uninitialized (null), so EventProperties.getType() (which + // returns mStorage.eventType) returned null instead of its documented + // "" default. Exercise the pure-Java storage directly: constructing an + // EventProperties here would call setName() -> native validateEventName(), + // which is not loaded in these JVM (MockitoJUnitRunner) unit tests. + EventPropertiesStorage storage = new EventPropertiesStorage(); + assertNotNull(storage.eventType); + assertEquals("", storage.eventType); + } + } diff --git a/lib/jni/JniConvertors.cpp b/lib/jni/JniConvertors.cpp index 87b7cb15b..d944ddff8 100644 --- a/lib/jni/JniConvertors.cpp +++ b/lib/jni/JniConvertors.cpp @@ -159,8 +159,17 @@ EventProperties GetEventProperties(JNIEnv* env, const jstring& jstrEventName, co const jobjectArray& jEventPropertyStringKeyArray, const jobjectArray& jEventPropertyValueArray) { EventProperties eventProperties; eventProperties.SetName(JStringToStdString(env, jstrEventName)); - if (jstrEventType != NULL) - eventProperties.SetType(JStringToStdString(env, jstrEventType)); + if (jstrEventType != NULL) { + // An empty type means "unset" (the native default). Before #1329 the + // Java getType() returned null for a default EventProperties, so this + // branch was skipped. getType() now returns "" to fix a Java-side NPE; + // forwarding SetType("") here would fail native event-name validation + // and broadcast a spurious EVT_REJECTED for every typeless event, so + // only set a non-empty type. + std::string eventType = JStringToStdString(env, jstrEventType); + if (!eventType.empty()) + eventProperties.SetType(eventType); + } eventProperties.SetLatency(static_cast(jEventLatency)); eventProperties.SetPersistence(static_cast(jEventPersistence)); eventProperties.SetPopsample(static_cast(jEventPopSample)); From 6157c826f8a444aa3f5d121d5efa477a6ce0df5a Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 1 Jul 2026 17:21:23 -0500 Subject: [PATCH 12/30] Build native desktop SDK in MSVC conformance mode (/permissive-) (#1480) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Build native desktop SDK in MSVC conformance mode (/permissive-) Enable ConformanceMode (/permissive-) for the native desktop SDK projects via the shared Solutions/build.props, so non-standard MSVC extensions are caught at build time (issue #255). Two project families are deliberately excluded, gated in the ItemDefinitionGroup condition: * UWP / Windows Store projects (AppContainerApplication=true) compile as C++/CX (/ZW), which MSVC rejects in combination with /permissive-. * C++/CLI managed projects (Keyword=ManagedCProj, e.g. net40) target the managed runtime and are a separate conformance domain. build.props is imported only by the SDK projects (net40, win10-*, win32-*), not by vendored sqlite/zlib or the test projects, so vendored third-party code is unaffected. Validated locally (VS 2026, v145, x64 Release): clean Rebuild of win32-lib (58 files) plus win32-dll / win32-mini-lib / win32-mini-dll all compile with 0 conformance errors and 0 warnings. Confirmed the gate excludes UWP (win10-lib does not receive /permissive-, so its /ZW build is unaffected). Files: Solutions/build.props Resolves #255. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Extend conformance mode to net40 (C++/CLI) and fix /permissive- errors Bring the net40 managed (C++/CLI) project under /permissive- by dropping the Keyword=ManagedCProj exclusion from the build.props gate, and fix the two conformance errors this surfaced in shared CX code: * lib/shared/EventPropertiesCX.cpp: add `typename` to the dependent name `map::iterator` in StoreEventProperties (C3878/C2065 under two-phase name lookup). * lib/shared/PlatformHelpers.h: forward-declare FromPlatformString before the FromPlatformMap templates that call it. Under strict two-phase lookup the dependent call fell back to ADL (which searches Platform::, not the SDK namespace) and failed with C3861. Now only UWP / Windows Store (C++/CX, /ZW) projects are excluded, since /permissive- is incompatible with /ZW. Validated (VS 2026, v145, x64 Release): clean Rebuild of the full non-UWP SDK set -- win32-lib, win32-dll, win32-mini-lib, win32-mini-dll and net40 -- with 0 conformance errors. (net40 is .NET Framework 4.0 C++/CLI; local builds need the v4.0 reference-assembly targeting pack, which the SDK CI provides.) Files: Solutions/build.props, lib/shared/EventPropertiesCX.cpp, lib/shared/PlatformHelpers.h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * build: address Copilot round-1 — scope /permissive- to SDK library projects only Solutions/build.props (Copilot): the ConformanceMode block lived on the shared build.props, which is ALSO imported by vendored sqlite/zlib, the test projects (gtest/gmock/UnitTests/FuncTests) and the samples -- so /permissive- was applied far more broadly than intended, and the PR description's claim was wrong. Verified with `git grep -l build.props -- '*.vcxproj'`: sqlite/sqlite.vcxproj, third_party/.../zlibvc.vcxproj and tests/{functests,unittests}/*.vcxproj all import it. Fix: move ConformanceMode into a dedicated Solutions/conformance.props, imported explicitly only by the five native SDK library projects (win32-lib, win32-dll, win32-mini-lib, win32-mini-dll, net40), right after their build.props import so the setting wins. build.props is reverted to its original content. Validated (VS 2026, v145, x64 Release): /permissive- present on win32-lib's cl invocations (0 conformance errors); a standalone sqlite:Rebuild now shows 0 /permissive- occurrences; clean Rebuild of all five SDK library projects. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Solutions/conformance.props | 23 +++++++++++++++++++ Solutions/net40/net40.vcxproj | 1 + Solutions/win32-dll/win32-dll.vcxproj | 1 + Solutions/win32-lib/win32-lib.vcxproj | 1 + .../win32-mini-dll/win32-mini-dll.vcxproj | 1 + .../win32-mini-lib/win32-mini-lib.vcxproj | 1 + lib/shared/EventPropertiesCX.cpp | 2 +- lib/shared/PlatformHelpers.h | 5 ++++ 8 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 Solutions/conformance.props diff --git a/Solutions/conformance.props b/Solutions/conformance.props new file mode 100644 index 000000000..79e1ae779 --- /dev/null +++ b/Solutions/conformance.props @@ -0,0 +1,23 @@ + + + + + true + + + diff --git a/Solutions/net40/net40.vcxproj b/Solutions/net40/net40.vcxproj index ec55747a0..d21aede17 100644 --- a/Solutions/net40/net40.vcxproj +++ b/Solutions/net40/net40.vcxproj @@ -290,6 +290,7 @@ + diff --git a/Solutions/win32-dll/win32-dll.vcxproj b/Solutions/win32-dll/win32-dll.vcxproj index 026356130..b01b9e690 100644 --- a/Solutions/win32-dll/win32-dll.vcxproj +++ b/Solutions/win32-dll/win32-dll.vcxproj @@ -337,6 +337,7 @@ + diff --git a/Solutions/win32-lib/win32-lib.vcxproj b/Solutions/win32-lib/win32-lib.vcxproj index 90602e232..1b9fb6a7c 100644 --- a/Solutions/win32-lib/win32-lib.vcxproj +++ b/Solutions/win32-lib/win32-lib.vcxproj @@ -537,6 +537,7 @@ + diff --git a/Solutions/win32-mini-dll/win32-mini-dll.vcxproj b/Solutions/win32-mini-dll/win32-mini-dll.vcxproj index ce42cf020..fe923aee2 100644 --- a/Solutions/win32-mini-dll/win32-mini-dll.vcxproj +++ b/Solutions/win32-mini-dll/win32-mini-dll.vcxproj @@ -390,6 +390,7 @@ + diff --git a/Solutions/win32-mini-lib/win32-mini-lib.vcxproj b/Solutions/win32-mini-lib/win32-mini-lib.vcxproj index e720328fd..700623d89 100644 --- a/Solutions/win32-mini-lib/win32-mini-lib.vcxproj +++ b/Solutions/win32-mini-lib/win32-mini-lib.vcxproj @@ -678,6 +678,7 @@ + diff --git a/lib/shared/EventPropertiesCX.cpp b/lib/shared/EventPropertiesCX.cpp index 9c212d919..889965f6d 100644 --- a/lib/shared/EventPropertiesCX.cpp +++ b/lib/shared/EventPropertiesCX.cpp @@ -26,7 +26,7 @@ namespace Microsoft { FromPlatformMap(propertiesMap, properties); FromPlatformMap(this->PIITags, piiTags); - for (map::iterator it = properties.begin(); it != properties.end(); ++it) + for (typename map::iterator it = properties.begin(); it != properties.end(); ++it) { MAT::PiiKind piiType = MAT::PiiKind_None; auto tag = piiTags.find(it->first); diff --git a/lib/shared/PlatformHelpers.h b/lib/shared/PlatformHelpers.h index 47d1e723d..222ee0024 100644 --- a/lib/shared/PlatformHelpers.h +++ b/lib/shared/PlatformHelpers.h @@ -105,6 +105,11 @@ namespace Microsoft { void ThrowPlatformInvalidArgumentException(String^ message); void ThrowPlatformException(String^ message); + // Forward declaration so the FromPlatformMap templates below can + // resolve this helper under /permissive- two-phase name lookup + // (the definition appears later in this header). + std::string FromPlatformString(String^ platformString); + // Defining the template function in the header file eliminates the need in additional linker definitions. // platformmaptype can be read-only or editable platform map. template class platformmaptype> From de3b23b8a9c1c8125780363c46beaf5681b360d9 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Thu, 9 Jul 2026 16:13:14 -0500 Subject: [PATCH 13/30] Devirtualize storage hot path: mark MemoryStorage/OfflineStorageHandler final + TU-local statics (#1488) * devirt: mark concrete leaf impl classes final Mark the leaf concrete implementations of IHttpClient and IOfflineStorage final so the compiler can devirtualize (and often inline) calls made through them: HttpClient_{Curl,WinInet,WinRt,Apple,Android,CAPI} and OfflineStorage_Room / MemoryStorage / OfflineStorageHandler. Each was verified to have no subclass anywhere in the tree (lib + tests + wrappers). Deliberately NOT marked: - OfflineStorage_SQLite -- tests/unittests/OfflineStorageTests_SQLite.cpp subclasses it (OfflineStorage_SQLiteNoAutoCommit). - TelemetrySystemBase -- base of TelemetrySystem / AITelemetrySystem. Validated: NDK aarch64 -fsyntax-only on OfflineStorage_Room, OfflineStorageHandler and MemoryStorage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Internal linkage: mark TU-local C-API helpers and DllMain global static Companion to the `final` devirtualization in this PR: give internal linkage to translation-unit-local symbols so the compiler can inline / drop them and keep them out of the (static-archive and shared-object) symbol table. This helps the static-lib consumption path that -fvisibility=hidden does not fully cover, since hidden visibility only trims the dynamic export table while these symbols keep external linkage across TUs. lib/api/capi.cpp: mark the 10 file-local C-API dispatch helpers static (remove_client, mat_open_core, mat_open, mat_open_with_params, mat_log, mat_close, mat_pause, mat_resume, mat_upload, mat_flushAndTeardown). Verified each is called only from the single exported entry point evt_api_call_default (and each other) within capi.cpp, and appears in no header and no other translation unit. capi_get_client stays external (it is MAT::capi_get_client, declared in a header and used by the CAPI HTTP client). Consistent with the file's existing static mtx/clients. lib/shared/dllmain.cpp: mark thread_count static -- a file-scope mutable global mutated only inside DllMain in this TU. get_platform_uuid (sysinfo_sources.cpp) was deliberately NOT touched: it has no caller in-tree, so marking it static would trip -Wunused-function under -Werror. Verified: NDK clang aarch64 -fsyntax-only on capi.cpp is clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix typo in capi.cpp comment: Marashal -> Marshal Addresses Copilot review comment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Narrow scope: keep 'final' only where it measurably devirtualizes Per review feedback (lalitb), measured the devirtualization effect of the 'final' markers (vendored Linux shared libmat.so, -O2 -ffunction-sections -fvisibility=hidden -Wl,--gc-sections, no LTO), base vs this PR: - Final binary size: NO change (.text byte-size identical). Devirtualization swaps an indirect call for a same-width direct call -- perf, not size. - 'final' devirtualizes only where the concrete type is known: object-level indirect-call counts dropped in OfflineStorageHandler.o (70->64) and MemoryStorage.o (8->3) -- a class calling its own virtual methods and OfflineStorageHandler's concrete MemoryStorage member. Call sites routed through IHttpClient/IOfflineStorage base references were unchanged (LogManagerImpl/TelemetrySystem/OfflineStorage_SQLite/HttpClient_Curl TUs byte-identical). So 'final' on the HttpClient_* clients and OfflineStorage_Room bought no measured devirtualization while still restricting subclassing. Drop it from those; keep it only on MemoryStorage and OfflineStorageHandler (internal lib/offline impl types, not extension points), where it does help. The internal-linkage (static) cleanups in capi.cpp and dllmain.cpp are orthogonal and retained. Files: lib/http/HttpClient_{Android,Apple,CAPI,Curl,WinInet,WinRt}.hpp, lib/offline/OfflineStorage_Room.hpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Drop redundant virtual on non-override methods in final storage classes Newer Clang (macOS-latest, LLVM 19+) enables -Wunnecessary-virtual-specifier, which errors under -Werror when a 'virtual' method that does not override a base method lives inside a 'final' class (it can never be overridden). Marking OfflineStorageHandler and MemoryStorage final left three such methods (DeleteRecordsByKeys, isKilled, GetReservedCount) still declared virtual, breaking the macOS debug build. Remove the now-redundant virtual specifier; these become non-virtual, consistent with the devirtualization goal. No subclasses or overrides of these methods exist. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/api/capi.cpp | 22 +++++++++++----------- lib/offline/MemoryStorage.hpp | 4 ++-- lib/offline/OfflineStorageHandler.hpp | 6 +++--- lib/shared/dllmain.cpp | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/api/capi.cpp b/lib/api/capi.cpp index 916c4ebda..cfe6e0a0e 100644 --- a/lib/api/capi.cpp +++ b/lib/api/capi.cpp @@ -49,7 +49,7 @@ capi_client * MAT::capi_get_client(evt_handle_t handle) /// /// Remove C API handle from active client tracking struct. /// -void remove_client(evt_handle_t handle) +static void remove_client(evt_handle_t handle) { LOCKGUARD(mtx); clients.erase(handle); @@ -66,7 +66,7 @@ void remove_client(evt_handle_t handle) return ENOENT; \ }; -evt_status_t mat_open_core( +static evt_status_t mat_open_core( evt_context_t *ctx, const char* config, http_send_fn_t httpSendFn, @@ -175,7 +175,7 @@ evt_status_t mat_open_core( return ctx->result; } -evt_status_t mat_open(evt_context_t *ctx) +static evt_status_t mat_open(evt_context_t *ctx) { if (ctx == nullptr) { @@ -186,7 +186,7 @@ evt_status_t mat_open(evt_context_t *ctx) return mat_open_core(ctx, config, nullptr, nullptr, nullptr, nullptr, nullptr); } -evt_status_t mat_open_with_params(evt_context_t *ctx) +static evt_status_t mat_open_with_params(evt_context_t *ctx) { if (ctx == nullptr) { @@ -231,9 +231,9 @@ evt_status_t mat_open_with_params(evt_context_t *ctx) } /** - * Marashal C struct to C++ API + * Marshal C struct to C++ API */ -evt_status_t mat_log(evt_context_t *ctx) +static evt_status_t mat_log(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); @@ -285,7 +285,7 @@ evt_status_t mat_log(evt_context_t *ctx) return ctx->result; } -evt_status_t mat_close(evt_context_t *ctx) +static evt_status_t mat_close(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(LogManagerProvider::Release(client->logmanager->GetLogConfiguration())); @@ -305,7 +305,7 @@ evt_status_t mat_close(evt_context_t *ctx) return result; } -evt_status_t mat_pause(evt_context_t *ctx) +static evt_status_t mat_pause(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(client->logmanager->PauseTransmission()); @@ -313,7 +313,7 @@ evt_status_t mat_pause(evt_context_t *ctx) return result; } -evt_status_t mat_resume(evt_context_t *ctx) +static evt_status_t mat_resume(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(client->logmanager->ResumeTransmission()); @@ -321,7 +321,7 @@ evt_status_t mat_resume(evt_context_t *ctx) return result; } -evt_status_t mat_upload(evt_context_t *ctx) +static evt_status_t mat_upload(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); const auto result = static_cast(client->logmanager->UploadNow()); @@ -329,7 +329,7 @@ evt_status_t mat_upload(evt_context_t *ctx) return result; } -evt_status_t mat_flushAndTeardown(evt_context_t *ctx) +static evt_status_t mat_flushAndTeardown(evt_context_t *ctx) { VERIFY_CLIENT_HANDLE(client, ctx); client->logmanager->FlushAndTeardown(); diff --git a/lib/offline/MemoryStorage.hpp b/lib/offline/MemoryStorage.hpp index 8a378dc5d..32dc82bdf 100644 --- a/lib/offline/MemoryStorage.hpp +++ b/lib/offline/MemoryStorage.hpp @@ -24,7 +24,7 @@ namespace MAT_NS_BEGIN { - class MemoryStorage : public IOfflineStorage + class MemoryStorage final : public IOfflineStorage { public: @@ -69,7 +69,7 @@ namespace MAT_NS_BEGIN { virtual size_t GetRemainingRecordCountForShutdown() const override; - virtual size_t GetReservedCount(); + size_t GetReservedCount(); virtual std::vector GetRecords(bool shutdown = false, EventLatency minLatency = EventLatency_Unspecified, unsigned maxCount = 0) override; diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index e7bdce4cb..1b07d5d1d 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -23,7 +23,7 @@ namespace MAT_NS_BEGIN { - class OfflineStorageHandler : public IOfflineStorage, public IOfflineStorageObserver + class OfflineStorageHandler final : public IOfflineStorage, public IOfflineStorageObserver { public: OfflineStorageHandler(ILogManager& logManager, IRuntimeConfig& runtimeConfig, ITaskDispatcher& taskDispatcher); @@ -64,7 +64,7 @@ namespace MAT_NS_BEGIN { virtual void OnStorageRecordsSaved(size_t numRecords) override; protected: - virtual void DeleteRecordsByKeys(const std::list & keys); + void DeleteRecordsByKeys(const std::list & keys); IOfflineStorageObserver * m_observer; ILogManager & m_logManager; @@ -75,7 +75,7 @@ namespace MAT_NS_BEGIN { KillSwitchManager m_killSwitchManager; ClockSkewManager m_clockSkewManager; - virtual bool isKilled(StorageRecord const& record); + bool isKilled(StorageRecord const& record); std::mutex m_flushLock; bool m_flushPending; diff --git a/lib/shared/dllmain.cpp b/lib/shared/dllmain.cpp index 4cfa868c6..d584c56c6 100644 --- a/lib/shared/dllmain.cpp +++ b/lib/shared/dllmain.cpp @@ -18,7 +18,7 @@ #ifdef _MANAGED #pragma unmanaged #endif -unsigned thread_count = 0; +static unsigned thread_count = 0; BOOL APIENTRY DllMain(HMODULE /* hModule */, DWORD ul_reason_for_call, LPVOID /* lpReserved */) { From fb500b0cebd18d8234b968927d8ceb7d24bf7986 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Thu, 9 Jul 2026 16:26:13 -0500 Subject: [PATCH 14/30] Skip MultipleLogManagersTests on iOS (fixes 60-min simulator CI hang) (#1500) * Skip MultipleLogManagersTests on iOS (hangs the simulator CI job) The iOS CI build (build-ios-mac.yml -> iOSFuncTests) consistently sat until its 60-minute timeout. The hang point is MultipleLogManagersTests.ThreeInstancesCoexist: these tests stand up an in-process HttpServer on a loopback port and run multiple concurrent LogManager instances uploading to it, which deadlocks inside the iOS simulator sandbox (the log shows the test starting, a loopback 'Connection reset by peer', then no further output until the job is canceled at 60 minutes). Skip the whole fixture on iOS via GTEST_SKIP() in SetUp(), guarded by TARGET_OS_IPHONE so macOS and desktop targets keep exercising it. The guard is a no-op on non-Apple platforms (TARGET_OS_IPHONE undefined). Files: tests/functests/MultipleLogManagersTests.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * iOS: exclude MultipleLogManagersTests at compile time, not via GTEST_SKIP The first attempt skipped the suite via GTEST_SKIP() in SetUp(), which stopped the 60-minute hang but the iOS xctest gtest wrapper does not honor a SetUp skip: the test bodies still ran (with the HttpServer never started) and failed at MultipleLogManagersTests.cpp:183 and :221. Exclude the whole suite from the iOS build with #if !defined(TARGET_OS_IPHONE) || !TARGET_OS_IPHONE so the tests don't exist in the iOS binary at all. macOS, Linux and Windows still build and run them (guard is true there; on non-Apple TARGET_OS_IPHONE is undefined -> included). Files: tests/functests/MultipleLogManagersTests.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/functests/MultipleLogManagersTests.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/functests/MultipleLogManagersTests.cpp b/tests/functests/MultipleLogManagersTests.cpp index eac2bfd00..7a9027b9b 100644 --- a/tests/functests/MultipleLogManagersTests.cpp +++ b/tests/functests/MultipleLogManagersTests.cpp @@ -22,6 +22,10 @@ #include "NullObjects.hpp" +#if defined(__APPLE__) +#include +#endif + #if defined __has_include && defined(HAVE_MAT_PRIVACYGUARD) #if __has_include("modules/privacyguard/PrivacyGuard.hpp") #include "modules/privacyguard/PrivacyGuard.hpp" @@ -36,6 +40,15 @@ using namespace testing; using namespace MAT; +// MultipleLogManagersTests stand up an in-process HttpServer on a loopback port +// and run multiple concurrent LogManager instances against it. That pattern +// hangs/fails inside the iOS simulator sandbox (the loopback uploads stall), +// which previously left the iOS CI job to sit until its 60-minute timeout. The +// behavior is still exercised on the desktop and macOS targets; exclude the +// whole suite from the iOS build. (GTEST_SKIP in SetUp is not honored by the +// iOS xctest gtest wrapper, so the exclusion must be at compile time.) +#if !defined(TARGET_OS_IPHONE) || !TARGET_OS_IPHONE + class RequestHandler : public HttpServer::Callback { public: @@ -292,5 +305,7 @@ TEST_F(MultipleLogManagersTests, PrivacyGuardSharedWithTwoInstancesCoexist) } #endif //END HAVE_MAT_PRIVACYGUARD +#endif // !TARGET_OS_IPHONE (suite excluded on iOS; see note above) + #endif // HAVE_MAT_DEFAULT_HTTP_CLIENT From 07fdf109e083fd2a59a6c332d5b95cfc75e47fbd Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Thu, 9 Jul 2026 17:47:44 -0500 Subject: [PATCH 15/30] Low-severity hardening: CSPRNG UUIDs (POSIX) and null-safe zlib error logs (#1493) * Low-severity hardening: CSPRNG UUIDs (POSIX) + null-safe zlib error logs Two low-severity issues found during a repo-wide review: 1) PAL::generateUuidString POSIX/Android fallback built the UUID entirely from std::rand(), seeded once with srand(time(0) ^ nanos). std::rand() is a weak, predictable PRNG with a guessable time-based seed, so the session / event / instance identifiers derived from it were predictable. Source the bytes from std::random_device instead (backed by /dev/urandom on Linux/Android), matching the existing CorrelationVector.cpp / PseudoRandomGenerator usage. Windows (CoCreateGuid) and Apple (CFUUIDCreate) paths are unchanged. Test: PalTests.UuidGeneration extended to assert 1000 generated UUIDs are all distinct (in addition to the existing format/entropy checks). 2) zlib error-path logs passed stream.msg / zs.msg straight to a %s conversion. zlib leaves msg == Z_NULL for several error codes (Z_MEM_ERROR, Z_BUF_ERROR, after a failed deflateInit2), and printf("%s", NULL) is undefined behavior -- benign "(null)" on glibc but not guaranteed across the MSVC/Android/Apple CRTs this SDK targets. Guard with `msg ? msg : "(null)"` in ZlibUtils.cpp and the two HttpDeflateCompression.cpp sites. Verified on Linux host: UnitTests builds; PalTests (incl. the UUID uniqueness check) and ZlibUtilsTests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address review comment: avoid reopening entropy source per UUID lib/pal/PAL.cpp (generateUuidString, POSIX/Android path): std::random_device was default-constructed on every call, which reopens the entropy source (/dev/urandom) per UUID on the event-logging hot path. Mark it thread_local so it is opened once per thread and reused; each operator() still draws fresh CSPRNG bytes, so the unpredictability property is unchanged and per-thread isolation keeps it lock-free. Verified the distinctness guard (PalTests.UuidGeneration, 1000 unique UUIDs) still passes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot round-1 comments zlib error logs (ZlibUtils.cpp:51, HttpDeflateCompression.cpp:47,83): zlib return codes are signed and frequently negative (Z_DATA_ERROR=-3, etc.). Logging them with %u misrepresented the value and was a format/type mismatch; use %d for both the step and the code. PAL.cpp generateUuidString (POSIX/Android): reduce std::random_device reads from 11 to 4 (random_device::max() spans the full unsigned int range, so 4x32 bits fills the 128-bit GUID), cutting per-event-ID entropy-source reads on the hot path. Also soften the comment: random_device is non-deterministic/CSPRNG-backed on our target platforms but the standard does not guarantee the backing source universally. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Name UUID test magic numbers as constexpr constants Address review feedback on PalTests UuidGeneration: replace the repeated literals 36 (canonical UUID string length) and 1000 (uniqueness-check batch size) with named constexpr constants UuidStringLength and UuidBatchSize so the test's intent is clear at each use site. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Unify PalTests local constants on constexpr Convert the remaining size_t const constants (NumQueries, NumBuckets) in the PseudoRandomGenerator test to constexpr so all local constants in PalTests.cpp use the same form. NumBuckets is used as an array bound, so constexpr also documents that it must be a compile-time constant. No behavior or codegen change (const-integral literals already fold to immediates). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/compression/HttpDeflateCompression.cpp | 4 +-- lib/pal/PAL.cpp | 35 +++++++++++++++------- lib/utils/ZlibUtils.cpp | 2 +- tests/unittests/PalTests.cpp | 26 ++++++++++++---- 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/lib/compression/HttpDeflateCompression.cpp b/lib/compression/HttpDeflateCompression.cpp index f8e2b1779..93605ea89 100644 --- a/lib/compression/HttpDeflateCompression.cpp +++ b/lib/compression/HttpDeflateCompression.cpp @@ -44,7 +44,7 @@ namespace MAT_NS_BEGIN { int result = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, m_windowBits, 8 /*DEF_MEM_LEVEL*/, Z_DEFAULT_STRATEGY); if (result != Z_OK) { - LOG_WARN("HTTP request compressing failed, error=%u/%u (%s)", 1, result, stream.msg); + LOG_WARN("HTTP request compressing failed, error=%d/%d (%s)", 1, result, (stream.msg ? stream.msg : "(null)")); compressionFailed(ctx); return false; } @@ -80,7 +80,7 @@ namespace MAT_NS_BEGIN { deflateEnd(&stream); if (result != Z_STREAM_END) { - LOG_WARN("HTTP request compressing failed, error=%u/%u (%s)", 2, result, stream.msg); + LOG_WARN("HTTP request compressing failed, error=%d/%d (%s)", 2, result, (stream.msg ? stream.msg : "(null)")); compressionFailed(ctx); return false; } diff --git a/lib/pal/PAL.cpp b/lib/pal/PAL.cpp index 01c6e6f75..3e667653f 100644 --- a/lib/pal/PAL.cpp +++ b/lib/pal/PAL.cpp @@ -369,19 +369,32 @@ namespace PAL_NS_BEGIN { std::transform(uuidStr.begin(), uuidStr.end(), uuidStr.begin(), ::tolower); return uuidStr; #else - static std::once_flag flag; - std::call_once(flag, [](){ - auto now = std::chrono::high_resolution_clock::now(); - auto nanos = std::chrono::duration_cast(now.time_since_epoch()).count(); - std::srand(static_cast(std::time(0) ^ nanos)); - }); + // Use std::random_device -- a non-deterministic, CSPRNG-backed source on + // the platforms we target (glibc/bionic/libc++ draw from getrandom or + // /dev/urandom) -- instead of std::rand()/srand(time(0)), so the session + // and event identifiers built from it are not predictable. It is + // thread_local so the backing source is opened once per thread rather than + // on every call (generateUuidString is on the event logging hot path), and + // the 128 bits are drawn with four operator() calls instead of eleven + // (random_device::max() is guaranteed to span the full unsigned int range). + thread_local std::random_device rd; GUID_t uuid; - uuid.Data1 = (static_cast(std::rand()) << 16) | static_cast(std::rand()); - uuid.Data2 = static_cast(std::rand()); - uuid.Data3 = static_cast(std::rand()); - for (size_t i = 0; i < sizeof(uuid.Data4); i++) - uuid.Data4[i] = static_cast(std::rand()); + const uint32_t r0 = rd(); + const uint32_t r1 = rd(); + const uint32_t r2 = rd(); + const uint32_t r3 = rd(); + uuid.Data1 = r0; + uuid.Data2 = static_cast(r1); + uuid.Data3 = static_cast(r1 >> 16); + uuid.Data4[0] = static_cast(r2); + uuid.Data4[1] = static_cast(r2 >> 8); + uuid.Data4[2] = static_cast(r2 >> 16); + uuid.Data4[3] = static_cast(r2 >> 24); + uuid.Data4[4] = static_cast(r3); + uuid.Data4[5] = static_cast(r3 >> 8); + uuid.Data4[6] = static_cast(r3 >> 16); + uuid.Data4[7] = static_cast(r3 >> 24); // TODO: [MG] - replace this sprintf by more robust GUID to string converter char buf[40] = { 0 }; diff --git a/lib/utils/ZlibUtils.cpp b/lib/utils/ZlibUtils.cpp index d091ab3fa..993aad2a8 100644 --- a/lib/utils/ZlibUtils.cpp +++ b/lib/utils/ZlibUtils.cpp @@ -48,7 +48,7 @@ namespace MAT_NS_BEGIN } while (ret == Z_OK); if (ret != Z_STREAM_END) { - LOG_WARN("Inflate failed, error=%u/%u (%s)", 2, ret, zs.msg); + LOG_WARN("Inflate failed, error=%d/%d (%s)", 2, ret, (zs.msg ? zs.msg : "(null)")); result = false; } inflateEnd(&zs); diff --git a/tests/unittests/PalTests.cpp b/tests/unittests/PalTests.cpp index 0bf8a06a3..9f20205fd 100644 --- a/tests/unittests/PalTests.cpp +++ b/tests/unittests/PalTests.cpp @@ -9,6 +9,7 @@ #include #include +#include #ifdef HAVE_MAT_LOGGING #include "pal/PAL.hpp" @@ -42,9 +43,14 @@ class PalTests : public Test {}; TEST_F(PalTests, UuidGeneration) { + // Canonical UUID string length ("8-4-4-4-12") and the number of UUIDs + // generated for the uniqueness check below. + constexpr size_t UuidStringLength = 36; + constexpr size_t UuidBatchSize = 1000; + std::string uuid0 = PAL::generateUuidString(); - EXPECT_THAT(uuid0.length(), 36u); + EXPECT_THAT(uuid0.length(), UuidStringLength); std::string mask = uuid0; for (char& ch : mask) { @@ -61,21 +67,31 @@ TEST_F(PalTests, UuidGeneration) std::string uuid1 = PAL::generateUuidString(); - EXPECT_THAT(uuid1.length(), 36u); + EXPECT_THAT(uuid1.length(), UuidStringLength); size_t diff = 0; - for (size_t i = 0; i < 36; i++) { + for (size_t i = 0; i < UuidStringLength; i++) { diff += (uuid0[i] != uuid1[i]); } EXPECT_THAT(diff, Gt(20u)); + + // A batch of generated UUIDs must all be distinct (guards against a stuck + // or low-entropy generator). + std::set uuids; + for (size_t i = 0; i < UuidBatchSize; i++) { + std::string u = PAL::generateUuidString(); + EXPECT_THAT(u.length(), UuidStringLength); + uuids.insert(u); + } + EXPECT_THAT(uuids.size(), UuidBatchSize); } TEST_F(PalTests, PseudoRandomGenerator) { PAL::PseudoRandomGenerator prg; - size_t const NumQueries = 1000; - size_t const NumBuckets = 11; + constexpr size_t NumQueries = 1000; + constexpr size_t NumBuckets = 11; size_t buckets[NumBuckets] = {}; for (size_t i = 0; i < NumQueries; i++) { From addba5666992e9846c1a3766b762ca4c230384ef Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Thu, 9 Jul 2026 18:14:29 -0500 Subject: [PATCH 16/30] Harden runtime task execution and offline-storage edge cases (#1495) * Harden runtime task execution and offline-storage edge cases These are latent correctness/robustness fixes found during a repo-wide review. WorkerThread.cpp: wrap the queued-task invocation `(*item)()` in try/catch. A task runs arbitrary work (storage I/O, HTTP encode, user DebugEventListener callbacks); an exception escaping the loop unwinds out of the thread entry function and calls std::terminate, killing the host process. Contain it and log. TaskDispatcher_CAPI.cpp: same exception barrier around `(*m_task)()` in Task_CAPI::OnCallback(), which runs on the host's external dispatcher thread. capi.cpp (mat_open_core): on the EALREADY path (a guest instance with the same config is already open) set ctx->handle to the existing instance's handle before returning, so the caller is not left with an uninitialized/stale handle. OfflineStorageHandler.cpp (Flush): null-guard m_offlineStorageMemory->GetSize() read; the subsequent block already null-checks the pointer, so reading it first was inconsistent and a potential null deref. OfflineStorageHandler (StoreRecord): the per-instance RAM cache size limit was held in a function-local `static`, so the first LogManager's CFG_INT_RAM_QUEUE_SIZE leaked to every other LogManager instance. Compute it once per instance in Initialize() into a member (preserving the original "compute once" intent). Tests: - PalTests.WorkerThreadContainsThrowingTask: a task that throws std/non-std exceptions does not tear down the worker thread; follow-up tasks still run. - TaskDispatcherCAPITests.ExecuteCallbackThatThrowsIsContained: a throwing CAPI task callback does not propagate back into the host dispatcher thread. Files changed: - lib/pal/WorkerThread.cpp - lib/pal/TaskDispatcher_CAPI.cpp - lib/api/capi.cpp - lib/offline/OfflineStorageHandler.cpp - lib/offline/OfflineStorageHandler.hpp - tests/unittests/PalTests.cpp - tests/unittests/TaskDispatcherCAPITests.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot round-1 comments capi.cpp (mat_open_core, EALREADY path): also set ctx->result to match the returned status. The success path sets ctx->result before returning; the EALREADY early-return set ctx->handle but left ctx->result stale, inconsistent with the other entrypoints. TaskDispatcher_CAPI.cpp (Task_CAPI::OnCallback): log the contained exception instead of swallowing it silently, mirroring WorkerThread, so host apps/SDK logs can diagnose why a queued task failed. Added include for std::exception. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot round-2 comments WorkerThread.cpp: add explicit #include ; the round-1 catch block uses std::exception but the TU only relied on transitive includes. capi.cpp (mat_open_core): the early error returns (invalid config, and the two HttpClient/TaskDispatcher creation catch blocks) left ctx->result and ctx->handle untouched. Since the public mat.h inline helpers return ctx.handle after calling in, callers could observe a stale/uninitialized handle on error. Set ctx->result = EFAULT and ctx->handle = 0 on all three paths (ctx is guaranteed non-null by mat_open / mat_open_with_params). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot round-3 comments capi.cpp (mat_open_core): if HttpClient_CAPI or TaskDispatcher_CAPI construction threw, clients[code] had already been populated (config + ctx_data assigned), but the catch returned EFAULT without removing it. That left an orphaned, half- initialized entry in the global clients map, so a later open with the same config would match ctx_data and wrongly return EALREADY. Call remove_client(code) on both creation-failure paths before returning EFAULT. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/api/capi.cpp | 23 +++++++++++- lib/offline/OfflineStorageHandler.cpp | 13 ++++--- lib/offline/OfflineStorageHandler.hpp | 1 + lib/pal/TaskDispatcher_CAPI.cpp | 14 ++++++- lib/pal/WorkerThread.cpp | 16 +++++++- tests/unittests/PalTests.cpp | 41 +++++++++++++++++++++ tests/unittests/TaskDispatcherCAPITests.cpp | 21 +++++++++++ 7 files changed, 120 insertions(+), 9 deletions(-) diff --git a/lib/api/capi.cpp b/lib/api/capi.cpp index cfe6e0a0e..531d9419c 100644 --- a/lib/api/capi.cpp +++ b/lib/api/capi.cpp @@ -77,7 +77,11 @@ static evt_status_t mat_open_core( { if ((config == nullptr) || (config[0] == 0)) { - // Invalid configuration + // Invalid configuration. ctx is guaranteed non-null by the callers + // (mat_open / mat_open_with_params); set result and a known-invalid + // handle so callers don't observe a stale ctx->handle on error. + ctx->result = static_cast(EFAULT); + ctx->handle = 0; return EFAULT; } @@ -91,7 +95,12 @@ static evt_status_t mat_open_core( { if (client->ctx_data == config) { - // Guest instance with the same config is already open + // Guest instance with the same config is already open. + // Return its handle so the caller still gets a usable handle + // (rather than leaving ctx->handle uninitialized), and set + // ctx->result to match the returned status like the other paths. + ctx->handle = code; + ctx->result = static_cast(EALREADY); return EALREADY; } // hash code is assigned to another client, increment and retry @@ -143,6 +152,11 @@ static evt_status_t mat_open_core( } catch (...) { + // Roll back the partially-populated client so a later open with the + // same config does not find stale half-initialized state. + remove_client(code); + ctx->result = static_cast(EFAULT); + ctx->handle = 0; return EFAULT; } } @@ -158,6 +172,11 @@ static evt_status_t mat_open_core( } catch (...) { + // Roll back the partially-populated client so a later open with the + // same config does not find stale half-initialized state. + remove_client(code); + ctx->result = static_cast(EFAULT); + ctx->handle = 0; return EFAULT; } } diff --git a/lib/offline/OfflineStorageHandler.cpp b/lib/offline/OfflineStorageHandler.cpp index 9049339c4..52ce15515 100644 --- a/lib/offline/OfflineStorageHandler.cpp +++ b/lib/offline/OfflineStorageHandler.cpp @@ -33,6 +33,7 @@ namespace MAT_NS_BEGIN { m_shutdownStarted(false), m_memoryDbSize(0), m_queryDbSize(0), + m_cacheMemorySizeLimitInBytes(0), m_isStorageFullNotificationSend(false) { // TODO: [MG] - OfflineStorage_SQLite.cpp is performing similar checks @@ -83,7 +84,7 @@ namespace MAT_NS_BEGIN { void OfflineStorageHandler::Initialize(IOfflineStorageObserver& observer) { m_observer = &observer; - uint32_t cacheMemorySizeLimitInBytes = m_config[CFG_INT_RAM_QUEUE_SIZE]; + m_cacheMemorySizeLimitInBytes = m_config[CFG_INT_RAM_QUEUE_SIZE]; m_offlineStorageDisk = OfflineStorageFactory::Create(m_logManager, m_config); if (m_offlineStorageDisk) @@ -94,7 +95,7 @@ namespace MAT_NS_BEGIN { // TODO: [MG] - consider passing m_offlineStorageDisk to m_offlineStorageMemory, // so that the Flush() op on memory storage leads to saving unflushed events to // disk. - if (cacheMemorySizeLimitInBytes > 0) + if (m_cacheMemorySizeLimitInBytes > 0) { m_offlineStorageMemory.reset(new MemoryStorage(m_logManager, m_config)); m_offlineStorageMemory->Initialize(*this); @@ -174,7 +175,7 @@ namespace MAT_NS_BEGIN { // than the handle gets replaced by nullptr in this DeferredCallbackHandle obj. m_flushHandle.Cancel(); - size_t dbSizeBeforeFlush = m_offlineStorageMemory->GetSize(); + size_t dbSizeBeforeFlush = (m_offlineStorageMemory != nullptr) ? m_offlineStorageMemory->GetSize() : 0; if ((m_offlineStorageMemory) && (dbSizeBeforeFlush > 0) && (m_offlineStorageDisk)) { // This will block on and then take a lock for the duration of this move, and @@ -233,8 +234,10 @@ namespace MAT_NS_BEGIN { return false; } - // Check cache size only once at start - static uint32_t cacheMemorySizeLimitInBytes = m_config[CFG_INT_RAM_QUEUE_SIZE]; + // Cache size limit is per-instance config computed once in Initialize(); + // it must NOT be a function-local static, which would share the first + // LogManager's value with every other LogManager instance. + uint32_t cacheMemorySizeLimitInBytes = m_cacheMemorySizeLimitInBytes; if (nullptr != m_offlineStorageMemory && !m_shutdownStarted) { diff --git a/lib/offline/OfflineStorageHandler.hpp b/lib/offline/OfflineStorageHandler.hpp index 1b07d5d1d..9a1131aff 100644 --- a/lib/offline/OfflineStorageHandler.hpp +++ b/lib/offline/OfflineStorageHandler.hpp @@ -92,6 +92,7 @@ namespace MAT_NS_BEGIN { unsigned m_memoryDbSize; unsigned m_memoryDbSizeNotificationLimit; unsigned m_queryDbSize; + uint32_t m_cacheMemorySizeLimitInBytes; bool m_isStorageFullNotificationSend; protected: diff --git a/lib/pal/TaskDispatcher_CAPI.cpp b/lib/pal/TaskDispatcher_CAPI.cpp index f8e432c74..e75ee1924 100644 --- a/lib/pal/TaskDispatcher_CAPI.cpp +++ b/lib/pal/TaskDispatcher_CAPI.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -37,7 +38,18 @@ namespace PAL_NS_BEGIN { void OnCallback() { if (m_task) { - (*m_task)(); + // The task is host/user code running on the external dispatcher's + // thread; an exception escaping here would terminate the process. + // Log it (mirroring WorkerThread) instead of swallowing silently. + try { + (*m_task)(); + } + catch (const std::exception& ex) { + LOG_ERROR("Unhandled exception in CAPI task: %s", ex.what()); + } + catch (...) { + LOG_ERROR("Unhandled non-standard exception in CAPI task"); + } } ReleaseItem(); } diff --git a/lib/pal/WorkerThread.cpp b/lib/pal/WorkerThread.cpp index 2bdbf6c67..3adfb9e61 100644 --- a/lib/pal/WorkerThread.cpp +++ b/lib/pal/WorkerThread.cpp @@ -6,6 +6,8 @@ #include "pal/WorkerThread.hpp" #include "pal/PAL.hpp" +#include + #if defined(MATSDK_PAL_CPP11) || defined(MATSDK_PAL_WIN32) /* Maximum scheduler interval for SDK is 1 hour required for clamping in case of monotonic clock drift */ @@ -238,7 +240,19 @@ namespace PAL_NS_BEGIN { // Item wasn't cancelled before it could be executed if (self->m_itemInProgress != nullptr) { LOG_TRACE("%10llu Execute item=%p type=%s\n", wakeupCount, item.get(), item.get()->TypeName.c_str() ); - (*item)(); + // A task can run arbitrary work (storage I/O, HTTP encode, and + // user DebugEventListener callbacks). An exception escaping here + // would unwind out of the thread entry function and call + // std::terminate, killing the host process. Contain it. + try { + (*item)(); + } + catch (const std::exception& ex) { + LOG_ERROR("Unhandled exception in worker task: %s", ex.what()); + } + catch (...) { + LOG_ERROR("Unhandled non-standard exception in worker task"); + } self->m_itemInProgress = nullptr; } diff --git a/tests/unittests/PalTests.cpp b/tests/unittests/PalTests.cpp index 9f20205fd..ddf1f6dd2 100644 --- a/tests/unittests/PalTests.cpp +++ b/tests/unittests/PalTests.cpp @@ -5,9 +5,13 @@ #include "common/Common.hpp" #include "pal/PseudoRandomGenerator.hpp" +#include "pal/TaskDispatcher.hpp" +#include "pal/WorkerThread.hpp" #include "Version.hpp" +#include #include +#include #include #include @@ -196,6 +200,43 @@ TEST_F(PalTests, SdkVersion) EXPECT_THAT(PAL::getSdkVersion(), Eq(v)); } +namespace +{ + class ThrowingTaskHelper + { + public: + void ThrowStdException() { throw std::runtime_error("worker task boom"); } + void ThrowNonStdException() { throw 123; } + void Signal(std::atomic* ran) { ran->store(true); } + }; +} + +// A task throwing an exception must be contained by the worker thread loop; +// otherwise the exception unwinds out of the thread entry function and calls +// std::terminate, killing the host process. +TEST_F(PalTests, WorkerThreadContainsThrowingTask) +{ + auto dispatcher = PAL::WorkerThreadFactory::Create(); + ThrowingTaskHelper helper; + std::atomic ranAfterStdThrow(false); + std::atomic ranAfterNonStdThrow(false); + + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::ThrowStdException); + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::Signal, &ranAfterStdThrow); + + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::ThrowNonStdException); + PAL::dispatchTask(dispatcher.get(), &helper, &ThrowingTaskHelper::Signal, &ranAfterNonStdThrow); + + // Wait for the follow-up tasks to run, proving the thread survived each throw. + for (int i = 0; i < 500 && !(ranAfterStdThrow.load() && ranAfterNonStdThrow.load()); ++i) + PAL::sleep(10); + + EXPECT_TRUE(ranAfterStdThrow.load()); + EXPECT_TRUE(ranAfterNonStdThrow.load()); + + dispatcher->Join(); +} + #ifdef HAVE_MAT_LOGGING class LogInitTest : public Test { diff --git a/tests/unittests/TaskDispatcherCAPITests.cpp b/tests/unittests/TaskDispatcherCAPITests.cpp index 0867ad046..b227deb13 100644 --- a/tests/unittests/TaskDispatcherCAPITests.cpp +++ b/tests/unittests/TaskDispatcherCAPITests.cpp @@ -9,6 +9,8 @@ #include "pal/typename.hpp" #include "mat.h" +#include + using namespace testing; using namespace MAT; using namespace PAL; @@ -227,3 +229,22 @@ TEST(TaskDispatcherCAPITests, Join) EXPECT_EQ(wasJoined, true); } +TEST(TaskDispatcherCAPITests, ExecuteCallbackThatThrowsIsContained) +{ + TaskDispatcher_CAPI taskDispatcher(&OnTaskDispatcherQueue, &OnTaskDispatcherCancel, &OnTaskDispatcherJoin); + + AutoTestHelper testHelper; + testHelper->SetShouldExecute(true); + + bool wasExecuted = false; + testHelper->SetCallbackValidation([&wasExecuted](int /*param1*/, int /*param2*/) { + wasExecuted = true; + throw std::runtime_error("task threw"); + }); + + // The dispatcher must contain the exception so it never escapes back into + // the host's dispatcher thread (which would terminate the process). + EXPECT_NO_THROW(dispatchTask(&taskDispatcher, testHelper.get(), &TestHelper::Callback, 10 /*param1*/, 20 /*param2*/)); + EXPECT_EQ(wasExecuted, true); +} + From fa2734cffda026ff9c5557252c9ce4306b711ebc Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Thu, 9 Jul 2026 18:40:17 -0500 Subject: [PATCH 17/30] Make public headers safe for consumers building with -Werror (#1501) * Make public headers safe for consumers building with -Werror Consumers that embed the SDK and compile with -Wall -Wextra -Werror (e.g. ONNX Runtime / Foundry Local via add_subdirectory) were broken by warnings emitted from inside the SDK's public headers. Two complementary changes fix this: Primary (covers every consumer + every warning flag): mark mat's exported public include directory as SYSTEM. find_package consumers already treat an imported target's includes as system; SYSTEM extends that to add_subdirectory/FetchContent consumers, so their -Werror no longer promotes SDK-header warnings to errors. Verified: an -isystem consumer compiles clean under -Wall -Wextra -Wpedantic -Wshadow -Wconversion -Werror on gcc and clang. Defense in depth (also helps non-CMake consumers and NO_SYSTEM_FROM_IMPORTED): - UNREFERENCED_PARAMETER(...) expanded to nothing on gcc/clang, leaving the parameter unused; it now casts to void, eliminating 5 -Wunused-parameter warnings across NullObjects.hpp and LogManagerProvider.hpp. - assert(!"msg") triggered -Wstring-conversion on clang; switched to the canonical assert(false && "msg") in ISemanticContext.hpp (2 sites). Result: including the main public headers under -Wall -Wextra now yields 0 warnings (was 5). SDK's own top-level build verified unaffected (75 UNREFERENCED_PARAMETER call sites compile clean). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Scope SYSTEM to the public include dir only Address Copilot review comment on lib/CMakeLists.txt:358. The SYSTEM keyword is a per-call flag, so a single target_include_directories(mat SYSTEM PUBLIC ... PRIVATE ...) also marked the SDK-internal PRIVATE include dirs as system for mat's own build, which would suppress warnings in the SDK's own internal headers under -Werror. Split into two calls: SYSTEM PUBLIC for the exported public dir, and a separate non-SYSTEM PRIVATE call for internal dirs. Verified via CMake property query: INTERFACE_SYSTEM_INCLUDE_DIRECTORIES contains only the public dir; the private internal dirs return to their original (non-system) treatment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/CMakeLists.txt | 13 ++++++++++++- lib/include/public/ISemanticContext.hpp | 4 ++-- lib/include/public/ctmacros.hpp | 8 ++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 0d4161811..9307c05ea 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -351,10 +351,21 @@ endif() # PUBLIC propagates to consumers; PRIVATE is SDK-internal only. # BUILD_INTERFACE is used during the SDK build; INSTALL_INTERFACE is used # by consumers after cmake --install. +# +# The public headers are added in a separate SYSTEM call: SYSTEM marks them as +# system includes for consumers, so a consumer building with -Wall -Wextra +# -Werror is not broken by warnings originating inside the SDK's headers (e.g. +# -Wpedantic variadic-macro or -Wconversion diagnostics). find_package consumers +# already treat an imported target's includes as system; SYSTEM extends the same +# courtesy to add_subdirectory/FetchContent consumers. The PRIVATE internal +# include dirs are deliberately kept out of this SYSTEM call so the SDK's own +# -Werror build still diagnoses warnings in its internal headers. target_include_directories(mat - PUBLIC + SYSTEM PUBLIC $ $ +) +target_include_directories(mat PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include diff --git a/lib/include/public/ISemanticContext.hpp b/lib/include/public/ISemanticContext.hpp index 76d9f991b..28d9d97b4 100644 --- a/lib/include/public/ISemanticContext.hpp +++ b/lib/include/public/ISemanticContext.hpp @@ -141,7 +141,7 @@ namespace MAT_NS_BEGIN break; default: - assert(!"Unknown NetworkCost enum value"); + assert(false && "Unknown NetworkCost enum value"); value = ""; break; } @@ -180,7 +180,7 @@ namespace MAT_NS_BEGIN break; default: - assert(!"Unknown NetworkType enum value"); + assert(false && "Unknown NetworkType enum value"); value = ""; break; } diff --git a/lib/include/public/ctmacros.hpp b/lib/include/public/ctmacros.hpp index cabd36f5f..026176a04 100644 --- a/lib/include/public/ctmacros.hpp +++ b/lib/include/public/ctmacros.hpp @@ -67,9 +67,13 @@ # endif #endif -// TODO: [MG] - ideally we'd like to use __attribute__((unused)) with gcc/clang +// Cast the argument(s) to void so the parameter is genuinely referenced. An empty +// expansion left the parameter unused under -Wunused-parameter, which broke +// consumers compiling the SDK headers with -Wextra -Werror. On Windows the Win32 +// SDK provides its own UNREFERENCED_PARAMETER, so this definition only applies +// where that macro is not already defined. #ifndef UNREFERENCED_PARAMETER -#define UNREFERENCED_PARAMETER(...) +#define UNREFERENCED_PARAMETER(...) (void)(__VA_ARGS__) #endif #define OACR_USE_PTR(...) From 5152cb4067c3c0f46ffd79672702ffcffcade9c8 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Tue, 14 Jul 2026 13:53:17 -0500 Subject: [PATCH 18/30] Reduce binary footprint and support embedding the SDK as a CMake subproject (#1499) * Add BUILD_CURL_HTTP_CLIENT option to build Linux without curl/TLS On the CPP11/curl path (non-Apple, non-Windows), the built-in libcurl HTTP client was always compiled and curl was a hard find_package(CURL REQUIRED) dependency -- pulling in curl and a TLS backend (OpenSSL/mbedTLS) even for hosts that already have their own HTTP stack. Add option(BUILD_CURL_HTTP_CLIENT ON). When OFF, the curl block is skipped (no find_package(CURL), no link, no -DHAVE_MAT_CURL_HTTP_CLIENT) and the build instead defines -DMATSDK_NO_DEFAULT_HTTP_CLIENT. mat/config.h then undefines HAVE_MAT_DEFAULT_HTTP_CLIENT centrally (regardless of the config preset), which the SDK already handles end-to-end: HttpClientFactory and HttpClient_Curl.cpp compile out, and LogManagerImpl's existing !HAVE_MAT_DEFAULT_HTTP_CLIENT branch requires the host to supply an IHttpClient via CFG_MODULE_HTTP_CLIENT. Default ON keeps existing behavior unchanged. Apple/Windows are unaffected (they use native HTTP stacks and never enter the curl block). Validated on WSL x64-linux: with OFF, libmat has no curl symbols and a consumer links with no -lcurl/-lTLS (1.43 MB stripped, vs 4.39 MB with curl+mbedTLS and 10.65 MB with curl+OpenSSL). Files changed: - CMakeLists.txt: BUILD_CURL_HTTP_CLIENT option + gating - lib/include/mat/config.h: central HAVE_MAT_DEFAULT_HTTP_CLIENT opt-out Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Add MATSDK_MINIMAL_SQLITE: private feature-stripped SQLite to cut footprint The SDK uses SQLite only for its offline event-storage cache (plain tables, transactions, WAL, autovacuum/VACUUM, a few PRAGMAs, and one custom UTF-8 function), so most SQLite subsystems are dead weight. Add an option to compile a private SQLite from the vendored amalgamation with a set of amalgamation-safe strip flags (single source of truth: MATSDK_SQLITE_MINIMAL_DEFS), removing the external sqlite3 dependency and shrinking SQLite ~10.2% (.text) / ~12.5% (object). - Root CMakeLists.txt: add option(MATSDK_MINIMAL_SQLITE) (default OFF). In vcpkg mode, skip find_package(unofficial-sqlite3) when bundling, and emit a clear FATAL_ERROR pointing at the system-sqlite/minimal-sqlite features when neither provides SQLite (e.g. a bare [core] install). - lib/CMakeLists.txt: define MATSDK_SQLITE_MINIMAL_DEFS, compute MATSDK_BUNDLE_SQLITE (minimal OR vendored-Android), and build a single sqlite3_bundled. The strip flags are applied ONLY when MATSDK_MINIMAL_SQLITE is ON, so the default Android legacy build keeps its existing unstripped bundled SQLite. Warnings are disabled on the vendored target (/w on MSVC, -w on GCC/Clang for the stripped build) so the SDK's -Werror/-WX does not fire on amalgamation code. A static mat propagates the PRIVATE sqlite3_bundled through its link interface, so export+install it. - MSTelemetryConfig.cmake.in: skip find_dependency(unofficial-sqlite3) when bundled. - vcpkg port: add a minimal-sqlite feature (-DMATSDK_MINIMAL_SQLITE=ON) and move sqlite3 into a default system-sqlite feature so [core,minimal-sqlite] drops it. - docs/building-with-vcpkg.md: document the feature, the size win, and the static-absorption symbol-visibility caveat. SQLITE_OMIT_AUTOINIT and SQLITE_DEFAULT_MEMSTATUS=0 are deliberately NOT stripped: the former because skipSqliteInitAndShutdown lets the host skip the SDK's explicit sqlite3_initialize() (which the host cannot do against a private SQLite), the latter because the SDK arms a soft heap limit via sqlite3_soft_heap_limit64() that is only enforced while memory statistics are enabled. Validated: vendored Linux Debug (77 offline-storage/SQLite unit tests pass on the debug amalgamation), vcpkg [core,minimal-sqlite] consumer (links MSTelemetry::sqlite3_bundled, runs 10/10, external sqlite3 dropped), default vcpkg path regression (system-sqlite intact), and MSVC compile/link of sqlite3_bundled+mat. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot review: scope bundled-SQLite export to static mat - lib/CMakeLists.txt: only add sqlite3_bundled to the install/export set when mat is a STATIC_LIBRARY. A shared mat absorbs the private SQLite into libmat and does not propagate the PRIVATE dependency, so exporting the separate archive there was unnecessary and could let a consumer link a second SQLite copy. For a static mat the archive must stay exported because the static library propagates its PRIVATE dependency through its link interface (\$). - CMakeLists.txt: make the vcpkg dependency-mode status message reflect whether the external sqlite3 package or the private minimal SQLite is used. Verified with an isolated CMake export test: static mat exports m+sq (consumer linking only the namespaced lib resolves sq); shared mat exports only m and install(EXPORT) succeeds with sq excluded. Re-ran the vcpkg [core,minimal-sqlite] consumer (static x64-linux): 10/10. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Clarify sqlite3_bundled PRIVATE-link comment (Copilot review) Correct the inline comment: a PRIVATE link of the bundled SQLite suppresses propagation of its include dirs / compile definitions, but a static mat still propagates the archive for linking via \$ (hence it is exported for static builds); a shared mat absorbs it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg port: selectable TLS backend (curl-openssl/curl-mbedtls) + no-default-http-client feature Make the HTTP-client footprint a consumer choice instead of hardcoding curl[openssl]: - vcpkg.json: replace the base curl[openssl] dependency with three features -- curl-openssl (default; libcurl + OpenSSL), curl-mbedtls (libcurl + mbedTLS), and no-default-http-client (omit the built-in client). curl-openssl is a default feature so a plain install keeps current behavior; [core,no-default-http-client] drops curl entirely. - portfile.cmake: map the no-default-http-client feature to -DBUILD_CURL_HTTP_CLIENT=OFF via INVERTED_FEATURES. - CMakeLists.txt: when the built-in client is enabled in vcpkg mode but libcurl is not found, emit a clear FATAL_ERROR pointing at the curl-openssl/curl-mbedtls/ no-default-http-client features (instead of a bare find_package failure). - docs: document the size ladder (OpenSSL ~10.6MB / mbedTLS ~4.4MB / no-curl ~1.4MB) and the exact mbedTLS recipe -- crucially, the consumer must ALSO list curl with default-features:false at the top level, because vcpkg only honors curl's default-features:false for top-level dependencies (otherwise curl's ssl default pulls OpenSSL in transitively alongside mbedTLS). Validated on WSL with vcpkg: default resolves curl[openssl]+sqlite3; the documented mbedTLS recipe builds with mbedTLS only (no libssl/libcrypto, libcurl carries no OpenSSL symbols) and the consumer runs; [core,no-default-http-client] drops curl from the graph; [core,minimal-sqlite,no-default-http-client] drops curl and the external sqlite3. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Harden HTTP-client features: CURL CONFIG mode + mutual-exclusivity guard (Copilot review) - CMakeLists.txt: in vcpkg mode use find_package(CURL CONFIG QUIET) and gate on TARGET CURL::libcurl. Forcing CONFIG selects the vcpkg-provided CURLConfig (which defines the imported target) rather than the module FindCURL, which on some CMake versions does not define CURL::libcurl and would fail at link. - portfile.cmake: fail fast when more than one of curl-openssl/curl-mbedtls/ no-default-http-client is selected. vcpkg cannot express mutual exclusivity, so a consumer requesting e.g. curl-mbedtls without [core] keeps the default curl-openssl and would union both TLS backends; the guard now errors with guidance to use the [core,...] form. Validated: the guard passes single selections and fires on curl-openssl+curl-mbedtls and curl-openssl+no-default-http-client; the default (curl-openssl) vcpkg consumer still configures via CURL CONFIG, links, and runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Remove the no-curl (no-default-http-client) option Drop the ability to build without the built-in libcurl HTTP client. That option only benefited consumers that already ship their own IHttpClient; the SDK's named consumers (and Apple/Windows, which use NSURLSession/WinInet) never needed it, and it added a fragile feature plus a config-flow opt-out. The TLS-backend selection (curl-openssl default / curl-mbedtls) and minimal-SQLite remain. - CMakeLists.txt: remove option(BUILD_CURL_HTTP_CLIENT) and the no-curl else branch; the curl HTTP client is always built on the CPP11/curl path again (keeping the find_package(CURL CONFIG) + TARGET CURL::libcurl hardening). - lib/include/mat/config.h: remove the MATSDK_NO_DEFAULT_HTTP_CLIENT -> HAVE_MAT_DEFAULT_HTTP_CLIENT opt-out. - vcpkg.json: remove the no-default-http-client feature. - portfile.cmake: remove the INVERTED_FEATURES mapping; the mutual-exclusivity guard now covers just curl-openssl vs curl-mbedtls. - docs: drop the no-curl row/section; note the size figures are worst-case (without consumer-side --gc-sections). Validated: vcpkg.json parses, CMake configures cleanly, and the mat target builds and links with the curl client compiled in. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg: use Apple's system libsqlite3/libz on macOS/iOS instead of vcpkg packages macOS/iOS ship libsqlite3 and libz as system libraries, so pulling and statically linking the vcpkg sqlite3 + zlib packages added ~1 MB of redundant code to Apple binaries. Link the system libraries instead -- consistent with the SDK's own Swift Package (which links .linkedLibrary("sqlite3"/"z")) and with how analogous telemetry SDKs (e.g. sentry-native) gate these deps off Apple platforms. - vcpkg.json: gate the zlib dependency and the system-sqlite feature's sqlite3 dependency to "!osx & !ios" so they are not installed on Apple. - CMakeLists.txt: on APPLE in vcpkg mode, find_package(SQLite3)/find_package(ZLIB) (CMake's modules resolve to the OS libraries) and set MATSDK_APPLE_SYSTEM_DEPS. - lib/CMakeLists.txt: link SQLite::SQLite3 + ZLIB::ZLIB on Apple; never bundle a private SQLite on Apple (MATSDK_MINIMAL_SQLITE is a no-op there since the system lib is already smaller). - MSTelemetryConfig.cmake.in: re-find system SQLite3 on Apple, the vcpkg unofficial-sqlite3 elsewhere. - docs: note the Apple system-lib behavior. Validated: non-Apple paths unchanged -- Linux vendored mat builds, and the Linux vcpkg consumer's generated config resolves unofficial-sqlite3 (if(OFF)) and runs. The Apple build itself needs validation on macOS/iOS CI (no Mac available here); the risk is whether find_package(SQLite3) resolves the system lib under the vcpkg Apple triplets' find-root settings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Clarify mbedTLS guidance: [core,...] drops system-sqlite too The curl-openssl/curl-mbedtls guidance in the port's fatal-error messages and docs recommended cpp-client-telemetry[core,curl-mbedtls], but the [core,...] form (default-features:false) drops ALL default features -- including system-sqlite -- not just curl-openssl. That example yields a config-time failure with no SQLite backend selected. Update both FATAL_ERROR messages (portfile.cmake mutual-exclusivity guard, CMakeLists.txt libcurl-not-found) and the docs prose to show a complete, working feature set ([core,curl-mbedtls,system-sqlite]) and to note that [core,...] also drops system-sqlite, so a SQLite backend must be re-selected. Files: tools/ports/cpp-client-telemetry/portfile.cmake, CMakeLists.txt, docs/building-with-vcpkg.md Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix misleading curl 'optional' label and mbedTLS feature description Two doc/manifest accuracy fixes from Copilot review: - The dependency table labeled libcurl 'optional' for non-Windows/non-Apple, but since the no-curl option was removed, Linux/Android vcpkg builds always require curl (only the TLS backend is selectable). Relabel as required. - The curl-mbedtls feature description recommended [core,curl-mbedtls], which drops all defaults (incl. system-sqlite); note that a SQLite backend must be re-selected to avoid a configure-time failure. Files: docs/building-with-vcpkg.md, tools/ports/cpp-client-telemetry/vcpkg.json Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Harden [core,...] guidance and fix exported CURL find_dependency mode Five fixes from the Copilot review on the curl/SQLite feature interactions (all stem from vcpkg's [core,...] form dropping ALL default features, not just one): - portfile.cmake: fail fast on Linux/Android when no curl TLS backend is selected (verified: a real vcpkg install of [core,minimal-sqlite] now stops at the portfile with a complete [core,curl-openssl,system-sqlite] example, instead of a later, opaque libcurl-not-found error). - CMakeLists.txt libcurl message: show how to re-select a curl backend (not only mbedTLS) under [core,...], alongside a SQLite backend. - CMakeLists.txt SQLite message: include the valid [core,system-sqlite] path, not only [core,minimal-sqlite]. - MSTelemetryConfig.cmake.in: find_dependency(CURL CONFIG) so the exported package config uses the vcpkg CURLConfig that defines CURL::libcurl (the target MSTelemetryTargets references), matching the unofficial-sqlite3/ nlohmann_json CONFIG siblings and the root CMakeLists CURL CONFIG lookup. - docs: minimal-sqlite manifest example re-selects curl-openssl so the Linux/Android manifest actually configures. Files: CMakeLists.txt, cmake/MSTelemetryConfig.cmake.in, docs/building-with-vcpkg.md, tools/ports/cpp-client-telemetry/portfile.cmake Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Scope curl TLS-backend guards to Linux/Android only The mutual-exclusivity check (curl-openssl vs curl-mbedtls) previously ran on all platforms. Since curl-openssl is a default feature and the curl dependency is platform-filtered to linux|android, a cross-platform manifest that enables curl-mbedtls without [core] would falsely fail the port on Windows/macOS/iOS -- where curl is not used (WinInet / Apple HTTP) and neither feature pulls curl. Wrap both the mutual-exclusivity (count>1) and no-curl (count==0) checks in a single VCPKG_TARGET_IS_LINUX/ANDROID block so they only fire where the curl backend selection is actually meaningful. Verified on x64-linux: [core,minimal- sqlite] still fails with the no-curl message, and [curl-mbedtls] (no core) still fails with the mutual-exclusivity message. Files: tools/ports/cpp-client-telemetry/portfile.cmake Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg port tests: build the working tree, not a pinned release The port tests built the SDK from the portfile's pinned vcpkg_from_github REF (v3.10.161.1), so they never exercised the PR's own source -- and the macOS/iOS jobs failed because this PR's manifest drops the Apple sqlite3/zlib packages while the old pinned source still calls find_package(unofficial-sqlite3) unconditionally (the Apple system-libs branch only exists in the PR source). Add an opt-in MATSDK_VCPKG_SOURCE_DIR hook to portfile.cmake: when set, the port builds that local source; when unset (production installs), the pinned release is downloaded as before, so the published port behavior is unchanged. The five tests/vcpkg/* scripts set it to the repo root so the port tests validate the actual source + manifest together. Verified on Linux (x64-linux): the port now builds the working-tree SDK and the consumer passes 10/10; the macOS/iOS jobs will exercise the Apple system-libs branch (find_package(SQLite3)/ZLIB) instead of the dropped vcpkg packages. Files: tools/ports/cpp-client-telemetry/portfile.cmake, tests/vcpkg/test-vcpkg-{linux,macos,ios,android}.sh, tests/vcpkg/test-vcpkg-windows.ps1 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix Windows vcpkg test to actually build the working tree On Windows, vcpkg runs portfiles in a sanitized environment and strips custom variables unless allow-listed via VCPKG_KEEP_ENV_VARS. Without it the portfile never saw MATSDK_VCPKG_SOURCE_DIR and silently fell back to the pinned release (v3.10.161.1), so the Windows port test validated the old release instead of the PR source while still reporting PASS. Allow-list the variable so the test builds the working tree, matching the Linux/macOS scripts (POSIX vcpkg passes the variable through, so they need no change). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Gate test suites on top-level project (default OFF for consumers) BUILD_UNIT_TESTS/BUILD_FUNC_TESTS defaulted to ON unconditionally, so a downstream project consuming this repo via add_subdirectory()/FetchContent built the whole test suite and required the third_party/googletest submodule. Default them ON only when this repo is the top-level project (PROJECT_IS_TOP_LEVEL on CMake >= 3.21, source-dir comparison on older CMake) and OFF when consumed as a subproject. Direct/CI builds are unchanged (top-level => ON) since build scripts rely on the default; verified BUILD_UNIT_TESTS/BUILD_FUNC_TESTS=ON for a top-level configure and OFF via add_subdirectory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot review: validate source dir, append VCPKG_KEEP_ENV_VARS - portfile.cmake: validate MATSDK_VCPKG_SOURCE_DIR points at a real checkout (CMakeLists.txt present) and fail early with a clear message instead of a confusing downstream CMake error. - test-vcpkg-windows.ps1: append MATSDK_VCPKG_SOURCE_DIR to VCPKG_KEEP_ENV_VARS instead of overwriting it, preserving any entries the caller/CI already set. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Support consuming SDK as a CMake subproject in legacy mode Applies the two changes the ONNX Runtime consumer patch carried so they can be dropped from the downstream patch set. Change 1 (CMakeLists.txt): use CMAKE_CURRENT_SOURCE_DIR instead of CMAKE_SOURCE_DIR for the vendored sqlite/zlib/nlohmann include path, so the headers still resolve when the SDK is added via add_subdirectory/FetchContent (where CMAKE_SOURCE_DIR points at the consumer's root, not this repo). Change 2 (lib/CMakeLists.txt): extend the Android bundled-deps legacy path to also cover iOS. A cross-compile cannot reliably find a system libsqlite3, and the vendored zlib renames its exports to act_z_* (zlib/names.h) so a system libz cannot satisfy those symbols. iOS now builds the vendored sqlite amalgamation + bundled zlib, matching Android. Only affects legacy mode (MATSDK_USE_VCPKG_DEPS=OFF); the vcpkg Apple path still links system libs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Explicitly disable warning-as-error for the vendored SQLite TU on MSVC Address Copilot review comment on lib/CMakeLists.txt:470. The comment claimed the build drops /WX for the vendored SQLite translation unit, but the code only added /w. /w disables all warnings, but MSVC can still promote a non-suppressible warning to an error under an inherited /WX. Add /WX- so the code literally matches the comment's stated intent and cannot be broken by such a warning. Verified cl.exe accepts /w /WX- together. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Clarify MATSDK_MINIMAL_SQLITE-on-Apple comment for the iOS legacy path Address Copilot review comment on lib/CMakeLists.txt:446. Adding iOS to the legacy bundled-SQLite path means MATSDK_MINIMAL_SQLITE is no longer a strict no-op on all Apple builds: iOS in legacy mode (MATSDK_USE_VCPKG_DEPS=OFF) bundles the amalgamation and applies the strip definitions to it, matching Android legacy. Clarify the comment so it no longer reads as a blanket 'no effect on Apple' statement. No behavior change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Link system sqlite3 + zlib on Apple legacy builds (match repo convention) Take further inspiration from the ONNX Runtime consumer patch, verified against what the repo already does on Apple. The SDK's own iOS Xcode projects link libsqlite3.tbd + libz.tbd from the SDKROOT, Package.swift links .linkedLibrary("sqlite3")/("z"), and #1499 already links the system libsqlite3/libz on the vcpkg Apple path. Bundling is an Android-only convention (the NDK ships no system zlib). So the earlier change that made iOS legacy bundle sqlite+zlib was the inconsistent one; this aligns iOS with the rest of the repo. - Apple legacy (macOS + iOS) now links system `sqlite3 z` by portable names in a single elseif(APPLE) branch. macOS moves off find_package(ZLIB) + hardcoded Homebrew .a paths (non-relocatable) onto the same portable link names, so exported static packages stay relocatable. iOS no longer bundles. - iOS dropped from the MATSDK_BUNDLE_SQLITE gating and the bundled-zlib branch, which are now Android-only. - Exclude iOS from include_directories(/usr/local/include): that host (macOS) path must not be injected into an iOS cross-compile's search path where it can shadow the iOS SDK's own headers. - Linux legacy simplified to find_package(SQLite3) (the Homebrew .a fallbacks were macOS-only and are now handled by the Apple branch). Verified: Linux top-level and add_subdirectory legacy builds both produce libmat.so. The Apple legacy path is exercised by the macOS-latest CI leg (build-posix-latest, legacy mode); iOS cannot be built on this host. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Drop redundant ZLIB include dir on the Linux legacy path target_include_directories(mat PRIVATE ${ZLIB_INCLUDE_DIRS}) was redundant: mat already links ZLIB::ZLIB (and SQLite::SQLite3), imported targets that propagate their own include directories. Verified: Linux legacy mat build still resolves and produces libmat.so. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * vcpkg port: bump pinned REF to v3.10.173.1 to match the SDK version The port pinned v3.10.161.1 while the SDK source on this branch is at v3.10.173.1 (Version.hpp), leaving production installs two releases behind. Bump the portfile REF + SHA512 and the vcpkg.json version to v3.10.173.1; the SHA512 is computed from the release source tarball. Note: the port's minimal-sqlite and Apple system-sqlite features depend on CMake changes introduced by this PR that are not yet in any release tag. The in-repo port tests exercise them against local source via MATSDK_VCPKG_SOURCE_DIR, and the pinned REF must be advanced again to the release that includes these changes once it is cut. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Validate vcpkg release bump production port path After updating the vcpkg port REF/SHA512/version for a new SDK release, exercise the real production port path with MATSDK_VCPKG_SOURCE_DIR unset. This catches mismatches where the port manifest assumes source changes that are not present in the release tag the port downloads. When the new footprint features are present, validate the opt-in minimal-sqlite + curl-openssl feature set so release automation covers both release pinning and feature wiring before opening the vcpkg PR. Validation: - Parsed .github/workflows/vcpkg-release-bump.yml with PyYAML. - Verified the feature-selection expression resolves to cpp-client-telemetry[core,minimal-sqlite,curl-openssl] for the current port. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Remove vcpkg release-bump production validation Remove the release-bump production-port validation added in 93dd36e9. The vcpkg port update will instead rely on the explicit release sequencing: merge the SDK source changes, cut a new SDK tag, then bump the vcpkg REF, SHA512, and version to that tag. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Revert "Remove vcpkg release-bump production validation" This reverts commit dd6007a693363b457d1ab12c576db606086891aa. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/vcpkg-release-bump.yml | 25 +++ CMakeLists.txt | 88 +++++++- cmake/MSTelemetryConfig.cmake.in | 16 +- docs/building-with-vcpkg.md | 127 ++++++++++- lib/CMakeLists.txt | 197 +++++++++++++++--- tests/vcpkg/test-vcpkg-android.sh | 4 + tests/vcpkg/test-vcpkg-ios.sh | 4 + tests/vcpkg/test-vcpkg-linux.sh | 4 + tests/vcpkg/test-vcpkg-macos.sh | 4 + tests/vcpkg/test-vcpkg-windows.ps1 | 14 ++ .../ports/cpp-client-telemetry/portfile.cmake | 73 ++++++- tools/ports/cpp-client-telemetry/vcpkg.json | 66 ++++-- 12 files changed, 552 insertions(+), 70 deletions(-) diff --git a/.github/workflows/vcpkg-release-bump.yml b/.github/workflows/vcpkg-release-bump.yml index d09706242..77ed47444 100644 --- a/.github/workflows/vcpkg-release-bump.yml +++ b/.github/workflows/vcpkg-release-bump.yml @@ -147,6 +147,31 @@ jobs: mv "${MANIFEST}.tmp" "${MANIFEST}" ./vcpkg format-manifest "${MANIFEST}" + - name: Validate updated production port + if: ${{ steps.ver.outputs.skip != 'true' }} + run: | + set -euo pipefail + cd vcpkg + MANIFEST="ports/${PORT}/vcpkg.json" + + # Exercise the real production path: MATSDK_VCPKG_SOURCE_DIR must be + # unset so the port downloads the just-updated REF/SHA512 instead of + # accidentally validating this workflow's working tree. This catches + # manifest/portfile changes that require source changes not present in + # the release tag. + unset MATSDK_VCPKG_SOURCE_DIR + + PORT_SPEC="${PORT}" + if jq -e '(.features["minimal-sqlite"] != null) and (.features["curl-openssl"] != null)' "${MANIFEST}" >/dev/null; then + # Use an opt-in feature set when available so release validation covers + # feature wiring as well as the default graph. The default graph is + # still covered by regular vcpkg CI and by consumers. + PORT_SPEC="${PORT}[core,minimal-sqlite,curl-openssl]" + fi + + echo "Validating production port: ${PORT_SPEC}" + ./vcpkg install "${PORT_SPEC}" --triplet x64-linux --clean-after-build + - name: Detect change id: diff if: ${{ steps.ver.outputs.skip != 'true' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 360bf7436..8b2321e3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,16 @@ else() endif() message(STATUS "MATSDK_USE_VCPKG_DEPS: ${MATSDK_USE_VCPKG_DEPS}") +# Build a private, feature-stripped copy of the vendored SQLite amalgamation +# instead of linking an external SQLite. The SDK uses SQLite only for its offline +# event-storage cache, so the minimal build (see lib/CMakeLists.txt +# MATSDK_SQLITE_MINIMAL_DEFS) omits every optional SQLite subsystem the SDK does +# not use, shrinking the SQLite code ~10% and removing the external sqlite3 +# dependency. Off by default to preserve the existing external/system-SQLite +# behavior; the Android NDK path always bundles SQLite regardless. +option(MATSDK_MINIMAL_SQLITE "Build a feature-stripped vendored SQLite instead of an external one" OFF) +message(STATUS "MATSDK_MINIMAL_SQLITE: ${MATSDK_MINIMAL_SQLITE}") + # Begin Uncomment for i386 build #set(CMAKE_SYSTEM_PROCESSOR i386) #set(CMAKE_C_FLAGS -m32) @@ -327,8 +337,20 @@ message(STATUS "SDK version: ${SDK_VERSION_PREFIX}-${MATSDK_BUILD_VERSION}") option(BUILD_HEADERS "Build API headers" YES) option(BUILD_LIBRARY "Build library" YES) option(BUILD_TEST_TOOL "Build console test tool" YES) -option(BUILD_UNIT_TESTS "Build unit tests" YES) -option(BUILD_FUNC_TESTS "Build functional tests" YES) +# Default the test suites ON only when this repository is the top-level project +# (developer/CI build), and OFF when it is consumed via add_subdirectory()/ +# FetchContent, so downstream projects don't build the tests or require the +# third_party/googletest submodule. PROJECT_IS_TOP_LEVEL exists on CMake >= 3.21; +# fall back to comparing the source dirs on older CMake (floor is 3.15). +if(DEFINED PROJECT_IS_TOP_LEVEL) + set(MATSDK_TESTS_DEFAULT ${PROJECT_IS_TOP_LEVEL}) +elseif(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(MATSDK_TESTS_DEFAULT ON) +else() + set(MATSDK_TESTS_DEFAULT OFF) +endif() +option(BUILD_UNIT_TESTS "Build unit tests" ${MATSDK_TESTS_DEFAULT}) +option(BUILD_FUNC_TESTS "Build functional tests" ${MATSDK_TESTS_DEFAULT}) option(BUILD_JNI_WRAPPER "Build JNI wrapper" NO) option(BUILD_OBJC_WRAPPER "Build Obj-C wrapper" YES) option(BUILD_SWIFT_WRAPPER "Build Swift Wrappers" YES) @@ -363,10 +385,25 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11" AND NOT BUILD_APPLE_HTTP) set(MATSDK_NEEDS_CURL ON) add_definitions(-DHAVE_MAT_CURL_HTTP_CLIENT) - find_package(CURL REQUIRED) if(MATSDK_USE_VCPKG_DEPS) + # The TLS backend (OpenSSL/mbedTLS) is selected by the vcpkg port's + # curl-openssl (default) / curl-mbedtls features; the SDK just links libcurl. + # Force CONFIG mode so the vcpkg-provided CURLConfig (which defines the + # CURL::libcurl imported target) is used rather than the module FindCURL, + # which on some CMake versions does not define that target. + find_package(CURL CONFIG QUIET) + if(NOT TARGET CURL::libcurl) + message(FATAL_ERROR + "libcurl was not found. The vcpkg port provides the curl HTTP client " + "through the curl-openssl (default) or curl-mbedtls feature. Install " + "cpp-client-telemetry with its default features, or, under the [core,...] " + "form (which drops the default curl-openssl and system-sqlite features), " + "re-select a curl backend and a SQLite backend together, e.g. " + "[core,curl-openssl,system-sqlite] or [core,curl-mbedtls,minimal-sqlite].") + endif() list(APPEND LIBS CURL::libcurl) else() + find_package(CURL REQUIRED) # Prefer the imported target, which carries curl's include dirs and link # flags. Fall back to the find-module variables on CMake < 3.12, where # find_package(CURL) does not define CURL::libcurl. @@ -383,13 +420,46 @@ endif() # Dependency resolution (vcpkg mode vs vendored) ################################################################################################ if(MATSDK_USE_VCPKG_DEPS) - find_package(unofficial-sqlite3 CONFIG REQUIRED) - find_package(ZLIB REQUIRED) - find_package(nlohmann_json CONFIG REQUIRED) - message(STATUS "Using vcpkg-provided sqlite3, zlib, nlohmann-json") + if(APPLE) + # macOS/iOS ship libsqlite3 and libz as system libraries (the SDK's SPM + # distribution links them the same way), so the vcpkg sqlite3/zlib packages are + # not pulled there -- find the system ones via CMake's standard find modules. + find_package(SQLite3 REQUIRED) + find_package(ZLIB REQUIRED) + find_package(nlohmann_json CONFIG REQUIRED) + set(MATSDK_APPLE_SYSTEM_DEPS ON) + message(STATUS "Apple: using system SQLite3 + zlib; vcpkg-provided nlohmann-json") + else() + set(MATSDK_APPLE_SYSTEM_DEPS OFF) + # SQLite is provided by the private minimal build when MATSDK_MINIMAL_SQLITE is + # ON, so only require the external vcpkg sqlite3 package otherwise. + if(NOT MATSDK_MINIMAL_SQLITE) + find_package(unofficial-sqlite3 CONFIG QUIET) + if(NOT unofficial-sqlite3_FOUND) + message(FATAL_ERROR + "SQLite was not found and the minimal SQLite is not enabled. The vcpkg " + "port provides SQLite through one of two features: 'system-sqlite' " + "(default, links the external sqlite3 package) or 'minimal-sqlite' " + "(builds a private feature-stripped SQLite). Install " + "cpp-client-telemetry with its default features, or with " + "[core,system-sqlite] or [core,minimal-sqlite]. For a direct CMake build, pass " + "-DMATSDK_MINIMAL_SQLITE=ON or ensure unofficial-sqlite3 is discoverable.") + endif() + endif() + find_package(ZLIB REQUIRED) + find_package(nlohmann_json CONFIG REQUIRED) + if(MATSDK_MINIMAL_SQLITE) + message(STATUS "Using vcpkg-provided zlib, nlohmann-json; private minimal SQLite") + else() + message(STATUS "Using vcpkg-provided sqlite3, zlib, nlohmann-json") + endif() + endif() else() - # Include repo root to allow includes of vendored sqlite, zlib, and nlohmann - include_directories(${CMAKE_SOURCE_DIR}) + # Include repo root to allow includes of vendored sqlite, zlib, and nlohmann. + # Use CMAKE_CURRENT_SOURCE_DIR (this repo's root) rather than CMAKE_SOURCE_DIR + # so the vendored headers still resolve when the SDK is consumed as a subproject + # (add_subdirectory/FetchContent), where CMAKE_SOURCE_DIR is the consumer's root. + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) message(STATUS "Using vendored sqlite3, zlib, nlohmann-json") endif() diff --git a/cmake/MSTelemetryConfig.cmake.in b/cmake/MSTelemetryConfig.cmake.in index 5cf00c560..af3838d65 100644 --- a/cmake/MSTelemetryConfig.cmake.in +++ b/cmake/MSTelemetryConfig.cmake.in @@ -2,8 +2,14 @@ include(CMakeFindDependencyMacro) -# Re-find dependencies that consumers need -find_dependency(unofficial-sqlite3 CONFIG) +# Re-find dependencies that consumers need. +# On Apple the SDK links the system libsqlite3 (SQLite::SQLite3); elsewhere it uses +# the vcpkg sqlite3 package unless a private minimal SQLite is bundled. +if(@MATSDK_APPLE_SYSTEM_DEPS@) + find_dependency(SQLite3) +elseif(NOT @MATSDK_BUNDLE_SQLITE@) + find_dependency(unofficial-sqlite3 CONFIG) +endif() find_dependency(ZLIB) find_dependency(nlohmann_json CONFIG) @@ -14,7 +20,11 @@ find_dependency(nlohmann_json CONFIG) # because the macOS BUILD_APPLE_HTTP choice can't be inferred from # CMAKE_SYSTEM_NAME alone. if(@MATSDK_NEEDS_CURL@) - find_dependency(CURL) + # Force CONFIG mode so the vcpkg-provided CURLConfig (which defines the + # CURL::libcurl imported target referenced by MSTelemetryTargets.cmake) is + # used, rather than module-mode FindCURL, which on some CMake versions does + # not define that target. + find_dependency(CURL CONFIG) endif() # Pthreads are needed on Linux and Android (POSIX threading) diff --git a/docs/building-with-vcpkg.md b/docs/building-with-vcpkg.md index 7e5cff7d0..315fa8662 100644 --- a/docs/building-with-vcpkg.md +++ b/docs/building-with-vcpkg.md @@ -145,10 +145,24 @@ The vcpkg port automatically resolves the following dependencies: | Dependency | vcpkg Package | CMake Target | Platforms | | -------------- | --------------- | --------------------------------- | ------------------ | -| SQLite3 | `sqlite3` | `unofficial::sqlite3::sqlite3` | All | -| zlib | `zlib` | `ZLIB::ZLIB` | All | +| SQLite3 | `sqlite3` | `unofficial::sqlite3::sqlite3` | Non-Apple (default; see `minimal-sqlite`). **macOS/iOS link the system `libsqlite3`** (`SQLite::SQLite3`) | +| zlib | `zlib` | `ZLIB::ZLIB` | Non-Apple. **macOS/iOS link the system `libz`** | | nlohmann JSON | `nlohmann-json` | `nlohmann_json::nlohmann_json` | All | -| libcurl | `curl[openssl]` | `CURL::libcurl` | Non-Windows, non-Apple | +| libcurl | `curl[openssl]` or `curl[mbedtls]` | `CURL::libcurl` | Non-Windows, non-Apple (required; TLS backend selectable: OpenSSL default or mbedTLS) | + +On **macOS/iOS** the SDK links the OS-provided `libsqlite3` and `libz` (the same +system libraries the SDK's Swift Package links), so the vcpkg `sqlite3` and `zlib` +packages are not pulled there — those binaries carry no bundled SQLite/zlib. +(`minimal-sqlite` therefore has no effect on Apple.) + +The external `sqlite3` package is provided by the default `system-sqlite` +feature. The `minimal-sqlite` feature replaces it with a private, feature-stripped +SQLite built from the SDK's vendored amalgamation — see +[Build a private minimal SQLite](#build-a-private-minimal-sqlite-minimal-sqlite-feature). + +libcurl is provided by the default `curl-openssl` feature; `curl-mbedtls` swaps in +the mbedTLS backend — see +[Choose the HTTP client / TLS backend](#choose-the-http-client--tls-backend-largest-lever-on-linux). Windows and macOS/iOS use platform-native HTTP clients (WinInet and NSURLSession respectively). Android vcpkg consumers use native libcurl because @@ -230,6 +244,54 @@ the stripping happens at your link. Keep the SDK a static dependency linked *into* your binary: if you re-export its API across your own DLL boundary, the export table pins its symbols and defeats `/OPT:REF`. +### Choose the HTTP client / TLS backend (largest lever on Linux) + +On Linux/Android the built-in HTTP client is libcurl, and curl's TLS backend +dominates the SDK's footprint. (Windows uses WinInet and Apple uses NSURLSession, +so this section does not apply there.) The port exposes the TLS backend as two +mutually-exclusive features; pick the one that matches what your application +already has: + +| Feature | Transport | Approx. stripped size¹ | Use when | +| ------- | --------- | ---------------------- | -------- | +| `curl-openssl` (default) | libcurl + OpenSSL | ~10.6 MB | your app already links OpenSSL (share it) | +| `curl-mbedtls` | libcurl + mbedTLS | ~4.4 MB | your app has no HTTP/TLS stack of its own | + +¹ Rough sizes of a minimal Linux consumer **without** consumer-side dead-stripping +(worst case); enabling `-Wl,--gc-sections` at your link reduces them. Your numbers +depend on triplet, dead-stripping, and what else shares those libraries. + +To select **mbedTLS**, two things are required in *your top-level* manifest: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": [ "minimal-sqlite", "curl-mbedtls" ] + }, + { "name": "curl", "default-features": false, "features": [ "mbedtls" ] } + ] +} +``` + +1. `"default-features": false` (the `[core,...]` form) drops **all** of the SDK's + default features -- both `curl-openssl` *and* `system-sqlite` -- so the SDK no + longer *requests* OpenSSL. Because it also drops `system-sqlite`, you must + re-select a SQLite backend (`minimal-sqlite` above, or `system-sqlite`); + otherwise the SDK configure step fails with no SQLite feature selected. +2. The explicit top-level `curl` entry is also needed because vcpkg honors curl's + own `"default-features": false` **only for top-level dependencies** — curl's + default `ssl` feature (which pulls OpenSSL on Linux) and `non-http` are + installed transitively otherwise. With both, curl resolves to `curl[core,mbedtls]` + and OpenSSL is not built; with only the feature, you get + `curl[mbedtls,ssl,openssl,non-http]` (mbedTLS *and* OpenSSL). This recipe is + verified with `vcpkg install --dry-run`. + +The default install (no features specified) keeps `curl-openssl` and works out of +the box. + ### Drop unused SQLite features (json1) The SDK uses SQLite only for offline event storage — plain tables and indexes, @@ -256,6 +318,65 @@ If any package in your build (or your own code) needs SQLite's JSON functions, request `sqlite3[json1]` instead and the extension is restored for the whole graph. +### Build a private minimal SQLite (`minimal-sqlite` feature) + +For a larger, self-contained reduction, the port can compile a private, +feature-stripped SQLite directly from the SDK's vendored amalgamation instead of +linking the external `sqlite3` package at all. The SDK uses SQLite only for its +offline event-storage cache (plain tables and indexes, transactions, WAL, +autovacuum/`VACUUM`, a few PRAGMAs, and one custom UTF-8 SQL function), so this +build omits the unused SQLite subsystems — `SQLITE_OMIT_JSON` plus load-extension, +shared-cache, deprecated APIs, authorization, EXPLAIN, introspection pragmas, +deserialize, and more. The result is **~10% smaller SQLite code** (`.text`) and +**~13% smaller** as a stripped object, and it drops the external `sqlite3` +dependency from your graph entirely. + +Enable it through the vcpkg feature: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": [ "minimal-sqlite", "curl-openssl" ] + } + ] +} +``` + +Use the `[core,minimal-sqlite]` form (here, `"default-features": false` is the +`[core]` part) so the default `system-sqlite` feature — and its `sqlite3` +dependency — is dropped. Because `[core]` drops **all** defaults, the example +also re-selects `curl-openssl`: on Linux/Android the built-in curl client +requires a TLS backend, so omitting it would fail to configure (swap in +`curl-mbedtls` for the smaller mbedTLS backend). Requesting `minimal-sqlite` +*without* `[core]` still pulls in the default `system-sqlite`; that is harmless +(the external `sqlite3` is installed but unused) but does not save the dependency. + +For a plain (non-vcpkg) CMake build, pass the option directly: + +```bash +cmake -DMATSDK_MINIMAL_SQLITE=ON .. +``` + +The strip is **amalgamation-safe**: it changes no SQLite grammar/parser, so no +code generation is required. All offline storage features the SDK relies on (WAL, +autovacuum, `VACUUM`, PRAGMAs, the custom UTF-8 function, blobs, 64-bit integers, +transactions) are retained, and the SDK's offline-storage unit tests pass +unchanged against the minimal build. + +> **Caveat — symbol visibility when linking statically.** The private SQLite keeps +> SQLite's default `sqlite3_*` symbol names. For a **shared** `mat` +> (`mat.dll` / `libmat.so` / `libmat.dylib`), those symbols are hidden by the +> SDK's `-fvisibility=hidden`, so there is no conflict. For a **static** `mat`, +> the minimal SQLite is installed and exported as a separate +> `MSTelemetry::sqlite3_bundled` archive that links into your binary; if **any** +> part of the final static link — your own code *or another dependency* — also +> pulls in SQLite, the duplicate `sqlite3_*` symbols will collide at link time. In +> that case, prefer the default `system-sqlite` feature so the whole graph shares a +> single SQLite. + ## How It Works: MATSDK_USE_VCPKG_DEPS When the SDK detects it is being built via vcpkg (by checking for diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 9307c05ea..97428a101 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -6,8 +6,10 @@ cmake_policy(SET CMP0063 NEW) # to downstream consumers via find_package() (see target_include_directories below). include_directories( . ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include/public ${CMAKE_CURRENT_SOURCE_DIR}/include/mat ${CMAKE_CURRENT_SOURCE_DIR}/pal ${CMAKE_CURRENT_SOURCE_DIR}/utils ${CMAKE_CURRENT_SOURCE_DIR}/modules/exp ${CMAKE_CURRENT_SOURCE_DIR}/modules/dataviewer ${CMAKE_CURRENT_SOURCE_DIR}/modules/privacyguard ${CMAKE_CURRENT_SOURCE_DIR}/modules/liveeventinspector ${CMAKE_CURRENT_SOURCE_DIR}/modules/cds ${CMAKE_CURRENT_SOURCE_DIR}/modules/signals ${CMAKE_CURRENT_SOURCE_DIR}/modules/sanitizer ) -# Legacy builds may need system-installed deps from /usr/local/include -if(NOT MATSDK_USE_VCPKG_DEPS) +# Legacy builds may need system-installed deps from /usr/local/include. Excluded on +# iOS: /usr/local/include is a host (macOS) path, and injecting it into an iOS +# cross-compile's search path can shadow the iOS SDK's own headers. +if(NOT MATSDK_USE_VCPKG_DEPS AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS") include_directories(/usr/local/include) endif() @@ -401,6 +403,97 @@ endif() ################################################################################################ # Link dependencies ################################################################################################ +# --- Minimal SQLite ----------------------------------------------------------- +# The SDK uses SQLite only for its offline event-storage cache: plain tables, +# indexes, transactions, WAL, autovacuum/VACUUM, a handful of PRAGMAs, and one +# custom UTF-8 SQL function. None of SQLite's optional subsystems are needed, so +# when MATSDK_MINIMAL_SQLITE is set the bundled SQLite is compiled with these +# options to strip out everything the SDK does not use (~10% smaller SQLite code). +# They are all amalgamation-safe (no grammar/parser regeneration) and validated +# against the offline-storage unit tests. +# +# Two options are deliberately NOT stripped because the SDK depends on them: +# * SQLITE_OMIT_AUTOINIT: the skipSqliteInitAndShutdown runtime config lets the +# host own SQLite's lifecycle and skip the SDK's explicit sqlite3_initialize() +# (lib/offline/SQLiteWrapper.hpp). With a private bundled SQLite the host cannot +# initialize the SDK's copy, so auto-init must remain on. +# * SQLITE_DEFAULT_MEMSTATUS=0: the SDK arms a soft heap limit via +# sqlite3_soft_heap_limit64(cacheMemorySizeLimitInBytes) on every open, and that +# limit is only enforced while memory statistics are enabled. Disabling them +# would silently turn the configured memory cap into a no-op. +set(MATSDK_SQLITE_MINIMAL_DEFS + SQLITE_DQS=0 + SQLITE_THREADSAFE=1 + SQLITE_DEFAULT_WAL_SYNCHRONOUS=1 + SQLITE_DEFAULT_FOREIGN_KEYS=0 + SQLITE_LIKE_DOESNT_MATCH_BLOBS + SQLITE_MAX_EXPR_DEPTH=0 + SQLITE_MAX_MMAP_SIZE=0 + SQLITE_USE_ALLOCA + SQLITE_OMIT_DEPRECATED + SQLITE_OMIT_PROGRESS_CALLBACK + SQLITE_OMIT_SHARED_CACHE + SQLITE_OMIT_LOAD_EXTENSION + SQLITE_OMIT_DECLTYPE + SQLITE_OMIT_JSON + SQLITE_OMIT_TRACE + SQLITE_OMIT_COMPLETE + SQLITE_OMIT_GET_TABLE + SQLITE_OMIT_TCL_VARIABLE + SQLITE_OMIT_EXPLAIN + SQLITE_OMIT_AUTHORIZATION + SQLITE_OMIT_DESERIALIZE + SQLITE_OMIT_INTROSPECTION_PRAGMAS + SQLITE_UNTESTABLE +) + +# Bundle a vendored SQLite (built from sqlite/sqlite3.c) when MATSDK_MINIMAL_SQLITE +# is requested, or on the Android NDK legacy path (which has no system SQLite and +# has always built the vendored amalgamation). Otherwise an external/system SQLite +# is used. The feature-strip definitions above are applied ONLY when +# MATSDK_MINIMAL_SQLITE is ON, so the default Android legacy build keeps its +# existing (unstripped) bundled SQLite behavior. +set(MATSDK_BUNDLE_SQLITE OFF) +if(MATSDK_MINIMAL_SQLITE AND NOT APPLE) + # On Apple the SDK links the system libsqlite3/libz (see the Apple branch below), + # so MATSDK_MINIMAL_SQLITE has no effect there. + set(MATSDK_BUNDLE_SQLITE ON) +elseif(NOT MATSDK_USE_VCPKG_DEPS AND CMAKE_SYSTEM_NAME STREQUAL "Android") + # Android NDK ships no system SQLite, so the vendored amalgamation is always bundled. + set(MATSDK_BUNDLE_SQLITE ON) +endif() + +if(MATSDK_BUNDLE_SQLITE AND NOT TARGET sqlite3_bundled) + add_library(sqlite3_bundled STATIC "${CMAKE_CURRENT_SOURCE_DIR}/../sqlite/sqlite3.c") + # Consumers of MSTelemetry::mat never include sqlite3.h (it is an internal + # implementation detail), so the header path is only needed while building the + # SDK itself -- wrap it in BUILD_INTERFACE so install(EXPORT) stays valid. + target_include_directories(sqlite3_bundled PUBLIC + "$") + set_target_properties(sqlite3_bundled PROPERTIES POSITION_INDEPENDENT_CODE ON) + if(MATSDK_MINIMAL_SQLITE) + # Feature-stripped build: apply the minimal definitions. + target_compile_definitions(sqlite3_bundled PRIVATE ${MATSDK_SQLITE_MINIMAL_DEFS}) + endif() + if(MSVC) + # Silence the vendored amalgamation's warnings (/w) and turn off + # warning-as-error (/WX-) for this third-party translation unit, so the SDK's + # /WX does not promote any amalgamation warning that survives /w to an error. + target_compile_options(sqlite3_bundled PRIVATE /w /WX-) + elseif(MATSDK_MINIMAL_SQLITE) + # -w disables all warnings for this vendored translation unit so the SDK's + # -Werror does not fire on amalgamation code (the OMIT_* options leave some + # debug-build macros expanding to empty/unused statements). -fno-finite-math-only: + # the amalgamation relies on the INFINITY macro, which -ffast-math / + # -ffinite-math-only would break. + target_compile_options(sqlite3_bundled PRIVATE -w -fno-finite-math-only) + else() + # Unstripped vendored build (Android legacy): keep the existing narrower + # warning suppression. -fno-finite-math-only guards the INFINITY macro. + target_compile_options(sqlite3_bundled PRIVATE -fno-finite-math-only -Wno-unused-function) + endif() +endif() + # TODO: allow adding "${Tcmalloc_LIBRARIES}" to target_link_libraries for memory leak debugging # (USE_TCMALLOC / FindTcmalloc.cmake are configured for Debug builds in the root CMakeLists.txt, # but the library is not yet linked here). @@ -408,25 +501,42 @@ if(MATSDK_USE_VCPKG_DEPS) # vcpkg mode: all deps resolved via find_package() in root CMakeLists.txt # These are PUBLIC so static-library consumers get the transitive link set # through the exported MSTelemetry::mat target. - target_link_libraries(mat - PUBLIC - unofficial::sqlite3::sqlite3 - ZLIB::ZLIB - nlohmann_json::nlohmann_json - ${LIBS} - ) + if(APPLE) + # macOS/iOS link the system libsqlite3 + libz (SQLite::SQLite3 / ZLIB::ZLIB + # resolve to the OS libraries via CMake's find modules), so the vcpkg + # sqlite3/zlib packages are neither pulled nor linked here. + target_link_libraries(mat + PUBLIC + SQLite::SQLite3 + ZLIB::ZLIB + nlohmann_json::nlohmann_json + ${LIBS} + ) + else() + if(MATSDK_BUNDLE_SQLITE) + # Private minimal SQLite instead of the vcpkg sqlite3 package. PRIVATE so its + # include dirs / compile definitions are not propagated as a public usage + # requirement. A static mat still propagates the archive itself for linking + # (via $), so it is added to the export set for static builds + # below; a shared mat absorbs it and propagates nothing. + target_link_libraries(mat PRIVATE sqlite3_bundled) + else() + target_link_libraries(mat PUBLIC unofficial::sqlite3::sqlite3) + endif() + target_link_libraries(mat + PUBLIC + ZLIB::ZLIB + nlohmann_json::nlohmann_json + ${LIBS} + ) + endif() else() # Legacy mode: use vendored or system-installed deps if(CMAKE_SYSTEM_NAME STREQUAL "Android") - # Android NDK has no system sqlite3 or zlib — build from bundled source. - add_library(sqlite3_bundled STATIC "${CMAKE_CURRENT_SOURCE_DIR}/../sqlite/sqlite3.c") - target_include_directories(sqlite3_bundled PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../sqlite") - set_target_properties(sqlite3_bundled PROPERTIES POSITION_INDEPENDENT_CODE ON) - # Guard bundled sqlite3 against toolchain or environment flags that imply finite-math-only (uses INFINITY macro). - # Also suppress warnings treated as errors in vendored code. - target_compile_options(sqlite3_bundled PRIVATE -fno-finite-math-only -Wno-unused-function) - - # Build zlib from bundled source. + # Build zlib from bundled source: the Android NDK ships no system zlib, and the + # vendored zlib renames its exports to act_z_* (via zlib/names.h). SQLite is + # provided by sqlite3_bundled, created above (MATSDK_BUNDLE_SQLITE is ON for + # the Android NDK path). add_library(zlib_bundled STATIC "${CMAKE_CURRENT_SOURCE_DIR}/../zlib/adler32.c" "${CMAKE_CURRENT_SOURCE_DIR}/../zlib/compress.c" @@ -456,26 +566,33 @@ else() elseif(PAL_IMPLEMENTATION STREQUAL "WIN32") # Windows legacy: vendored sqlite/zlib headers are included via # include_directories in the PAL section above; link only ${LIBS} - # (e.g. CURL if needed — sqlite/zlib come from .vcxproj references). - target_link_libraries(mat PRIVATE ${LIBS}) + # (e.g. CURL if needed — sqlite/zlib come from .vcxproj references), plus the + # private minimal SQLite when MATSDK_MINIMAL_SQLITE is enabled. + if(MATSDK_BUNDLE_SQLITE) + target_link_libraries(mat PRIVATE sqlite3_bundled ${LIBS}) + else() + target_link_libraries(mat PRIVATE ${LIBS}) + endif() + elseif(APPLE) + # macOS and iOS both ship system libsqlite3 and libz. Link them by portable + # names -- matching the SDK's own iOS Xcode projects (libsqlite3.tbd + libz.tbd + # from the SDKROOT), Package.swift (.linkedLibrary sqlite3/z), and the vcpkg + # Apple path -- so nothing is bundled and exported static packages stay + # relocatable. On Apple, #include / resolve from the SDK + # sysroot, so no explicit include dir or find_package is needed. + target_link_libraries(mat PRIVATE sqlite3 z ${LIBS}) else() - # Linux/macOS legacy: link system-installed sqlite3 and zlib - if(EXISTS "/usr/local/lib/libsqlite3.a") - set(MATSDK_SQLITE3_LIB "/usr/local/lib/libsqlite3.a") - elseif(EXISTS "/usr/local/opt/sqlite/lib/libsqlite3.a") - set(MATSDK_SQLITE3_LIB "/usr/local/opt/sqlite/lib/libsqlite3.a") - elseif(EXISTS "/opt/homebrew/opt/sqlite/lib/libsqlite3.a") - set(MATSDK_SQLITE3_LIB "/opt/homebrew/opt/sqlite/lib/libsqlite3.a") + # Linux legacy: system zlib + system (or private minimal) sqlite3. ZLIB::ZLIB + # and SQLite::SQLite3 are imported targets that carry their own include dirs. + find_package(ZLIB REQUIRED) + if(MATSDK_BUNDLE_SQLITE) + target_link_libraries(mat PRIVATE sqlite3_bundled ZLIB::ZLIB ${LIBS}) else() # find_package(SQLite3) needs CMake >= 3.14, guaranteed by the project floor; # SQLite::SQLite3 is an imported target carrying its own include dirs. find_package(SQLite3 REQUIRED) - set(MATSDK_SQLITE3_LIB SQLite::SQLite3) + target_link_libraries(mat PRIVATE SQLite::SQLite3 ZLIB::ZLIB ${LIBS}) endif() - - find_package(ZLIB REQUIRED) - target_include_directories(mat PRIVATE ${ZLIB_INCLUDE_DIRS}) - target_link_libraries(mat PRIVATE ${MATSDK_SQLITE3_LIB} ZLIB::ZLIB ${LIBS}) endif() endif() @@ -512,7 +629,21 @@ endif() # consumer that does find_package(MSTelemetry). Legacy (non-vcpkg) builds install # via install.sh or MSBuild output directories and don't need this. if(MATSDK_USE_VCPKG_DEPS) - install(TARGETS mat + # A static libmat propagates its PRIVATE static dependencies through its link + # interface (as $), so the bundled SQLite must be part of the same + # export set and installed alongside mat for downstream find_package() consumers + # to link. A shared libmat absorbs sqlite3_bundled into the .so/.dylib/.dll and + # does not propagate the PRIVATE dep, so exporting the archive there is + # unnecessary (and risks a consumer linking a second SQLite copy) -- only export + # it for a static mat. + set(MATSDK_INSTALL_TARGETS mat) + if(MATSDK_BUNDLE_SQLITE AND TARGET sqlite3_bundled) + get_target_property(_mat_type mat TYPE) + if(_mat_type STREQUAL "STATIC_LIBRARY") + list(APPEND MATSDK_INSTALL_TARGETS sqlite3_bundled) + endif() + endif() + install(TARGETS ${MATSDK_INSTALL_TARGETS} EXPORT MSTelemetryTargets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/tests/vcpkg/test-vcpkg-android.sh b/tests/vcpkg/test-vcpkg-android.sh index c43c73967..f49a24195 100755 --- a/tests/vcpkg/test-vcpkg-android.sh +++ b/tests/vcpkg/test-vcpkg-android.sh @@ -10,6 +10,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" + # Android ABI/API (defaults match the repo's Android minSdk) ANDROID_ABI="${1:-arm64-v8a}" ANDROID_API="${2:-23}" diff --git a/tests/vcpkg/test-vcpkg-ios.sh b/tests/vcpkg/test-vcpkg-ios.sh index f564e4615..c1097c3bd 100755 --- a/tests/vcpkg/test-vcpkg-ios.sh +++ b/tests/vcpkg/test-vcpkg-ios.sh @@ -10,6 +10,10 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" + +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" USE_SIMULATOR=false for arg in "$@"; do diff --git a/tests/vcpkg/test-vcpkg-linux.sh b/tests/vcpkg/test-vcpkg-linux.sh index 3482abe53..d98757db8 100755 --- a/tests/vcpkg/test-vcpkg-linux.sh +++ b/tests/vcpkg/test-vcpkg-linux.sh @@ -9,6 +9,10 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" BUILD_DIR="${SCRIPT_DIR}/build-linux" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" + echo "=== MSTelemetry vcpkg port test (Linux) ===" echo "Repository root: ${REPO_ROOT}" diff --git a/tests/vcpkg/test-vcpkg-macos.sh b/tests/vcpkg/test-vcpkg-macos.sh index d864928c8..9a7d1bfd3 100755 --- a/tests/vcpkg/test-vcpkg-macos.sh +++ b/tests/vcpkg/test-vcpkg-macos.sh @@ -9,6 +9,10 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" BUILD_DIR="${SCRIPT_DIR}/build-macos" OVERLAY_PORTS="${REPO_ROOT}/tools/ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +export MATSDK_VCPKG_SOURCE_DIR="${REPO_ROOT}" + echo "=== MSTelemetry vcpkg port test (macOS) ===" echo "Repository root: ${REPO_ROOT}" diff --git a/tests/vcpkg/test-vcpkg-windows.ps1 b/tests/vcpkg/test-vcpkg-windows.ps1 index 0536f2b8f..b1390425a 100644 --- a/tests/vcpkg/test-vcpkg-windows.ps1 +++ b/tests/vcpkg/test-vcpkg-windows.ps1 @@ -14,6 +14,20 @@ $RepoRoot = (Resolve-Path (Join-Path $ScriptDir "..\..")).Path $BuildDir = Join-Path $ScriptDir "build-windows" $OverlayPorts = Join-Path $RepoRoot "tools\ports" +# Build the working tree under review (not a pinned release) so this test +# validates the actual SDK source together with the port manifest/portfile. +$env:MATSDK_VCPKG_SOURCE_DIR = $RepoRoot +# On Windows, vcpkg runs portfiles in a sanitized environment and strips custom +# variables unless they are allow-listed here. Without this, the portfile does +# not see MATSDK_VCPKG_SOURCE_DIR and silently builds the pinned release instead +# of the working tree (POSIX vcpkg passes the variable through, so the Linux/ +# macOS scripts do not need this). +if ($env:VCPKG_KEEP_ENV_VARS) { + $env:VCPKG_KEEP_ENV_VARS = "$($env:VCPKG_KEEP_ENV_VARS);MATSDK_VCPKG_SOURCE_DIR" +} else { + $env:VCPKG_KEEP_ENV_VARS = "MATSDK_VCPKG_SOURCE_DIR" +} + Write-Host "=== MSTelemetry vcpkg port test (Windows) ===" -ForegroundColor Cyan # Resolve vcpkg root: parameter > VCPKG_ROOT env var > error diff --git a/tools/ports/cpp-client-telemetry/portfile.cmake b/tools/ports/cpp-client-telemetry/portfile.cmake index cfdab2368..c9ce3168c 100644 --- a/tools/ports/cpp-client-telemetry/portfile.cmake +++ b/tools/ports/cpp-client-telemetry/portfile.cmake @@ -1,10 +1,25 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO microsoft/cpp_client_telemetry - REF v3.10.161.1 - SHA512 4664b34ddce601d6a95669df4a59d11a6cc67de1f23de132192f791a275edc6a10b8498d340e6cf7d120d9e7a22c494d7517b24fc0954bf9e236e84a8800589a - HEAD_REF main -) +# In-repo port validation (tests/vcpkg/*) sets MATSDK_VCPKG_SOURCE_DIR so the port +# builds the working tree under review instead of a pinned release -- this is what +# lets the port tests actually exercise the SDK source + manifest together. When +# the variable is unset (production installs), the pinned release is downloaded as +# usual, so the published port behavior is unchanged. +if(DEFINED ENV{MATSDK_VCPKG_SOURCE_DIR}) + set(SOURCE_PATH "$ENV{MATSDK_VCPKG_SOURCE_DIR}") + if(NOT EXISTS "${SOURCE_PATH}/CMakeLists.txt") + message(FATAL_ERROR + "MATSDK_VCPKG_SOURCE_DIR is set to '${SOURCE_PATH}', but no CMakeLists.txt " + "was found there. It must point to a cpp_client_telemetry source checkout.") + endif() + message(STATUS "cpp-client-telemetry: building local source $ENV{MATSDK_VCPKG_SOURCE_DIR} (MATSDK_VCPKG_SOURCE_DIR is set)") +else() + vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO microsoft/cpp_client_telemetry + REF v3.10.173.1 + SHA512 e55bc35274236f57757660073c4dccccab3462342c8566212f1df4bf8824295a2bb3d3d79a11f3950e7c9252641827e9dd3d7c28c421dea3bdaee277e4f2ce32 + HEAD_REF main + ) +endif() # Determine if Apple HTTP should be used (no curl needed). # Note: BUILD_APPLE_HTTP must remain ON for macOS/iOS because the vcpkg.json @@ -20,9 +35,53 @@ if(VCPKG_TARGET_IS_IOS) set(MATSDK_BUILD_IOS ON) endif() +# curl-openssl (default) and curl-mbedtls choose the TLS backend for the built-in +# HTTP client and are mutually exclusive. They only matter on Linux/Android: the +# curl dependency is platform-filtered to those triplets, so on Windows/macOS/iOS +# both features may be present (curl-openssl is a default) yet pull no curl, and +# the SDK uses WinInet / Apple HTTP there. vcpkg cannot express mutual exclusivity +# or "exactly one of", so validate it here -- but only where curl is actually used, +# to avoid failing legitimate cross-platform manifests on Windows/Apple. +set(_matsdk_http_features "") +foreach(_matsdk_http_feature curl-openssl curl-mbedtls) + if(_matsdk_http_feature IN_LIST FEATURES) + list(APPEND _matsdk_http_features ${_matsdk_http_feature}) + endif() +endforeach() +list(LENGTH _matsdk_http_features _matsdk_http_feature_count) +if(VCPKG_TARGET_IS_LINUX OR VCPKG_TARGET_IS_ANDROID) + if(_matsdk_http_feature_count GREATER 1) + message(FATAL_ERROR + "curl-openssl (default) and curl-mbedtls are mutually exclusive but both were " + "selected. To use mbedTLS, drop the defaults with the [core,...] form and " + "re-select a SQLite backend (the [core,...] form also drops the default " + "system-sqlite feature), e.g. " + "cpp-client-telemetry[core,curl-mbedtls,system-sqlite] " + "(or minimal-sqlite in place of system-sqlite).") + elseif(_matsdk_http_feature_count EQUAL 0) + # The built-in curl HTTP client requires exactly one TLS backend. The [core,...] + # form drops the default curl-openssl, so fail fast (with a complete example) + # rather than letting the SDK CMake fail later on a missing libcurl. + message(FATAL_ERROR + "On Linux/Android the built-in curl HTTP client requires exactly one TLS " + "backend feature, but none was selected. The [core,...] form drops the " + "default curl-openssl feature, so re-add a curl backend together with a " + "SQLite backend, e.g. cpp-client-telemetry[core,curl-openssl,system-sqlite] " + "(or curl-mbedtls / minimal-sqlite in place of those).") + endif() +endif() + +# minimal-sqlite -> -DMATSDK_MINIMAL_SQLITE=ON (private feature-stripped SQLite). +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + minimal-sqlite MATSDK_MINIMAL_SQLITE +) + vcpkg_cmake_configure( SOURCE_PATH "${SOURCE_PATH}" OPTIONS + ${FEATURE_OPTIONS} -DMATSDK_USE_VCPKG_DEPS=ON -DBUILD_HEADERS=ON -DBUILD_LIBRARY=ON diff --git a/tools/ports/cpp-client-telemetry/vcpkg.json b/tools/ports/cpp-client-telemetry/vcpkg.json index 2ee6d9bc3..e8a9eb5a5 100644 --- a/tools/ports/cpp-client-telemetry/vcpkg.json +++ b/tools/ports/cpp-client-telemetry/vcpkg.json @@ -1,24 +1,12 @@ { "name": "cpp-client-telemetry", - "version": "3.10.161.1", + "version": "3.10.173.1", "description": "Microsoft 1DS C/C++ Client Telemetry Library", "homepage": "https://github.com/microsoft/cpp_client_telemetry", "license": "Apache-2.0", "supports": "((windows & !mingw) | linux | osx | ios | android) & !uwp", "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "openssl" - ], - "platform": "linux | android" - }, "nlohmann-json", - { - "name": "sqlite3", - "default-features": false - }, { "name": "vcpkg-cmake", "host": true @@ -27,6 +15,54 @@ "name": "vcpkg-cmake-config", "host": true }, - "zlib" - ] + { + "name": "zlib", + "platform": "!osx & !ios" + } + ], + "default-features": [ + "system-sqlite", + "curl-openssl" + ], + "features": { + "system-sqlite": { + "description": "Link the external vcpkg sqlite3 package for the offline storage cache (default). On macOS/iOS the SDK links the system libsqlite3 instead, so this dependency is not pulled there.", + "dependencies": [ + { + "name": "sqlite3", + "default-features": false, + "platform": "!osx & !ios" + } + ] + }, + "minimal-sqlite": { + "description": "Build a private, feature-stripped SQLite compiled from the SDK's vendored amalgamation instead of linking the external sqlite3 package. Smaller footprint; combine with [core,minimal-sqlite] to also drop the sqlite3 dependency." + }, + "curl-openssl": { + "description": "Built-in libcurl HTTP client with the OpenSSL TLS backend (default). Affects Linux/Android only; Windows uses WinInet and Apple uses NSURLSession regardless.", + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "openssl" + ], + "platform": "linux | android" + } + ] + }, + "curl-mbedtls": { + "description": "Built-in libcurl HTTP client with the mbedTLS backend instead of OpenSSL (smaller footprint). Affects Linux/Android only. Use [core,curl-mbedtls,system-sqlite] to drop the default OpenSSL curl; the [core,...] form drops all defaults (including system-sqlite), so also re-select system-sqlite or minimal-sqlite.", + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "mbedtls" + ], + "platform": "linux | android" + } + ] + } + } } From 84f028e6dfaca819e370e648b0fb390b77646ab7 Mon Sep 17 00:00:00 2001 From: Lakshmi Manaswi Karra <31495364+lkarra2@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:41:12 -0700 Subject: [PATCH 19/30] Fix crash: replace malformed UTF-8 instead of throwing in PayloadDecoder (#1512) * Replace malformed UTF-8 instead of throwing in PayloadDecoder * Address review: reword comment and add PayloadDecoder regression tests - Reword the DecodeRequest comment to state the local rationale for error_handler_t::replace instead of referencing the HAVE_MAT_AI-gated AIJsonSerializer, which is not part of the default OSS build. - Add PayloadDecoderTests covering the invalid-UTF-8 regression: a record whose string field carries a non-UTF-8 byte must be decoded without throwing type_error.316, and the bad byte must surface as U+FFFD. Also verifies valid UTF-8 is left untouched. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6ab3a16b-98f7-4fa7-a933-684b34196436 * Harden PayloadDecoder tests against -Werror char conversion Address review feedback on the regression tests: - Inject the invalid UTF-8 byte via push_back(static_cast(0xFF)) and search for it with static_cast(0xFF) instead of a '\xFF' string/char literal, which relies on implementation-defined char conversion and can trip -Werror constant-conversion on some toolchains. - Drop the product-specific phrasing from the test comment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6ab3a16b-98f7-4fa7-a933-684b34196436 --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6ab3a16b-98f7-4fa7-a933-684b34196436 --- lib/decoder/PayloadDecoder.cpp | 11 +++- tests/unittests/CMakeLists.txt | 1 + tests/unittests/PayloadDecoderTests.cpp | 86 +++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 tests/unittests/PayloadDecoderTests.cpp diff --git a/lib/decoder/PayloadDecoder.cpp b/lib/decoder/PayloadDecoder.cpp index 5789d5939..6d86a7057 100644 --- a/lib/decoder/PayloadDecoder.cpp +++ b/lib/decoder/PayloadDecoder.cpp @@ -563,7 +563,12 @@ namespace MAT_NS_BEGIN { if (result) { - out = j.dump(2); + // Use error_handler_t::replace so that malformed UTF-8 in the + // decoded telemetry payload is replaced with U+FFFD instead of + // throwing nlohmann::json::type_error (id 316). Telemetry event + // strings can legitimately contain non-UTF-8 bytes; without this + // the unhandled C++ exception terminates the hosting process. + out = j.dump(2, ' ', false, json::error_handler_t::replace); } return result; @@ -580,7 +585,9 @@ namespace MAT_NS_BEGIN { nlohmann::json j; to_json(j, in); - std::string s = j.dump(4); + // See DecodeRequest above: replace malformed UTF-8 rather than + // throwing so a bad record cannot terminate the process. + std::string s = j.dump(4, ' ', false, json::error_handler_t::replace); out.assign(s.begin(), s.end()); return true; diff --git a/tests/unittests/CMakeLists.txt b/tests/unittests/CMakeLists.txt index 945fc23df..7233d2920 100644 --- a/tests/unittests/CMakeLists.txt +++ b/tests/unittests/CMakeLists.txt @@ -40,6 +40,7 @@ set(SRCS OfflineStorageTests_Room.cpp OfflineStorageTests_SQLite.cpp PackagerTests.cpp + PayloadDecoderTests.cpp PalTests.cpp RouteTests.cpp StringUtilsTests.cpp diff --git a/tests/unittests/PayloadDecoderTests.cpp b/tests/unittests/PayloadDecoderTests.cpp new file mode 100644 index 000000000..8e8051cee --- /dev/null +++ b/tests/unittests/PayloadDecoderTests.cpp @@ -0,0 +1,86 @@ +// +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +#include "common/Common.hpp" +#include "PayloadDecoder.hpp" + +using namespace testing; +using namespace MAT; + +namespace +{ + // Builds a minimally-populated Common Schema record. to_json() in the + // PayloadDecoder unconditionally dereferences element [0] of every ext + // vector, so all seven must contain at least one element for serialization + // to succeed. + CsProtocol::Record MakeMinimalRecord() + { + CsProtocol::Record record; + record.ver = "3.0"; + record.name = "Test.Event"; + record.time = 0; + record.iKey = "o:0000"; + record.baseType = "custom"; + record.extProtocol.push_back(CsProtocol::Protocol{}); + record.extUser.push_back(CsProtocol::User{}); + record.extDevice.push_back(CsProtocol::Device{}); + record.extOs.push_back(CsProtocol::Os{}); + record.extApp.push_back(CsProtocol::App{}); + record.extNet.push_back(CsProtocol::Net{}); + record.extSdk.push_back(CsProtocol::Sdk{}); + return record; + } +} + +// A telemetry event field can legitimately contain bytes that are not valid +// UTF-8. nlohmann::json::dump() defaults to error_handler_t::strict, which +// throws type_error.316 on such input. Because DecodeRecord/DecodeRequest run +// on the decode path inside the hosting process, an unhandled throw terminates +// that process. These tests lock in the error_handler_t::replace behavior: no +// throw, and the malformed byte is emitted as the U+FFFD replacement character +// (EF BF BD). +TEST(PayloadDecoderTests, DecodeRecord_InvalidUtf8_DoesNotThrow) +{ + CsProtocol::Record record = MakeMinimalRecord(); + // Build the field with an explicit 0xFF byte (never valid UTF-8). A string + // literal escape ("...\xFF...") would rely on implementation-defined char + // conversion and can trip -Werror constant-conversion on some toolchains. + std::string name = "Bad"; + name.push_back(static_cast(0xFF)); + name += "Name"; + record.name = name; + + std::string out; + bool decoded = false; + EXPECT_NO_THROW({ decoded = exporters::DecodeRecord(record, out); }); + + // When the SDK is built with JSON + Zlib support the real decoder runs and + // must have replaced the bad byte. In a stubbed build DecodeRecord returns + // false with an empty string, in which case the no-throw guarantee above is + // what this test protects. + if (decoded) + { + EXPECT_NE(out.find("\xEF\xBF\xBD"), std::string::npos) + << "Malformed UTF-8 should be replaced with U+FFFD"; + EXPECT_EQ(out.find(static_cast(0xFF)), std::string::npos) + << "Raw invalid byte must not survive in the output"; + } +} + +TEST(PayloadDecoderTests, DecodeRecord_ValidUtf8_IsPreserved) +{ + CsProtocol::Record record = MakeMinimalRecord(); + record.name = "Valid.Event"; + + std::string out; + bool decoded = false; + EXPECT_NO_THROW({ decoded = exporters::DecodeRecord(record, out); }); + + if (decoded) + { + EXPECT_NE(out.find("Valid.Event"), std::string::npos); + EXPECT_EQ(out.find("\xEF\xBF\xBD"), std::string::npos) + << "Valid UTF-8 must not be altered"; + } +} From 1654b44abc78fe052aae3a335801fdfa1766b7e5 Mon Sep 17 00:00:00 2001 From: Derek Morris Date: Wed, 29 Jul 2026 10:57:21 -0700 Subject: [PATCH 20/30] Roll forward lib/modules to get past state with broken normalization (#1509) Broken normalization in the lib/modules subrepo is causing enlistment issues in the Edge project; rolling forward the submodule to the commit 7bd8b516e2d93d1704834e0895733ae7bc2d1f43 picks up fixes made in that repo that amend this. Co-authored-by: bmehta001 --- lib/modules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules b/lib/modules index c637015fb..7bd8b516e 160000 --- a/lib/modules +++ b/lib/modules @@ -1 +1 @@ -Subproject commit c637015fbbe904ed556d27e3b9072f6f2a5ee401 +Subproject commit 7bd8b516e2d93d1704834e0895733ae7bc2d1f43 From 52e4545b85daec9d876a1978621ff57f4ea0c5a9 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 29 Jul 2026 18:26:14 -0500 Subject: [PATCH 21/30] Default Android vcpkg builds to Java HTTP (#1510) * Add Android HTTP transport selection for vcpkg Decouple Android HTTP transport choice from MATSDK_USE_VCPKG_DEPS so consumers can use vcpkg-provided native dependencies while still packaging the Java/JNI transport. Files changed: - CMakeLists.txt, lib/CMakeLists.txt, cmake/MSTelemetryConfig.cmake.in: add MATSDK_ANDROID_HTTP_CLIENT and export Android Java source metadata. - tools/ports/cpp-client-telemetry/*, docs/building-with-vcpkg.md: add android-java-http feature, source guard, and consumer documentation. - lib/pal/posix/*_Android.cpp: align initializer order for current NDK Clang builds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b12c5862-01e3-45e4-bf91-6389c20cae41 * Default Android vcpkg builds to Java HTTP Make Android AUTO select the Java/JNI HTTP transport regardless of dependency sourcing, and require explicit android-curl-* features for native curl. Files changed: - CMakeLists.txt: resolve Android AUTO to JAVA. - tools/ports/cpp-client-telemetry/portfile.cmake: use the in-repo checkout for overlay builds and add explicit Android curl feature handling. - tools/ports/cpp-client-telemetry/vcpkg.json: move Android curl to explicit android-curl-openssl/android-curl-mbedtls features. - docs/building-with-vcpkg.md: document Java default and curl escape hatch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b12c5862-01e3-45e4-bf91-6389c20cae41 * Address Copilot review for Android transport PR NetworkInformationImpl_Android.cpp: initialize m_registeredCount so callback registration bookkeeping does not read an indeterminate value. Verified m_registeredCount is incremented/decremented in lib/pal/NetworkInformationImpl.hpp. DeviceInformationImpl_Android.cpp: initialize m_registeredCount for the same callback bookkeeping path. Verified m_registeredCount is incremented/decremented in lib/pal/DeviceInformationImpl.hpp. MSTelemetryConfig.cmake.in: update the curl dependency comment to reflect Linux, explicit Android curl builds, and macOS-without-Apple-HTTP only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b12c5862-01e3-45e4-bf91-6389c20cae41 * Address Copilot follow-up review for Android transport portfile.cmake: make in-repo overlay source detection require SDK-specific paths so registry/vcpkg checkouts cannot be mistaken for cpp_client_telemetry. MSTelemetryConfig.cmake.in: set MSTelemetry_ANDROID_JAVA_SOURCE_DIR only for Java-transport packages; set it empty otherwise. Android PAL: initialize m_type and m_os_architecture alongside the callback counters so inline getters never read indeterminate values. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b12c5862-01e3-45e4-bf91-6389c20cae41 * Address final Copilot comments on Android transport MSTelemetryConfig.cmake.in: export NONE instead of an empty Android transport sentinel for non-Android packages, while keeping the Java source directory empty outside JAVA transport. portfile.cmake: add the [core,...]/default-features=false remediation hint to the curl backend mutual-exclusivity error. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: b12c5862-01e3-45e4-bf91-6389c20cae41 --------- Copilot-Session: b12c5862-01e3-45e4-bf91-6389c20cae41 --- CMakeLists.txt | 31 ++++- cmake/MSTelemetryConfig.cmake.in | 17 ++- docs/building-with-vcpkg.md | 109 ++++++++++++++---- lib/CMakeLists.txt | 10 +- .../posix/DeviceInformationImpl_Android.cpp | 5 +- .../posix/NetworkInformationImpl_Android.cpp | 5 +- .../posix/SystemInformationImpl_Android.cpp | 5 +- .../ports/cpp-client-telemetry/portfile.cmake | 88 +++++++++----- tools/ports/cpp-client-telemetry/vcpkg.json | 56 ++++++--- 9 files changed, 252 insertions(+), 74 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b2321e3c..cc36e9da3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -362,6 +362,35 @@ option(BUILD_SIGNALS "Build Signals" YES) option(BUILD_SANITIZER "Build Sanitizer" YES) option(LINK_STATIC_DEPENDS "Link dependencies for static build" YES) +set(MATSDK_ANDROID_HTTP_CLIENT "AUTO" CACHE STRING "Android HTTP client: AUTO, JAVA, or CURL") +set_property(CACHE MATSDK_ANDROID_HTTP_CLIENT PROPERTY STRINGS AUTO JAVA CURL) +string(TOUPPER "${MATSDK_ANDROID_HTTP_CLIENT}" MATSDK_ANDROID_HTTP_CLIENT_UPPER) +if(NOT MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "AUTO" + AND NOT MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "JAVA" + AND NOT MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "CURL") + message(FATAL_ERROR + "MATSDK_ANDROID_HTTP_CLIENT must be AUTO, JAVA, or CURL; got " + "'${MATSDK_ANDROID_HTTP_CLIENT}'.") +endif() + +set(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED "") +set(MATSDK_ANDROID_USES_CURL OFF) +set(MATSDK_ANDROID_USES_JAVA_HTTP OFF) +if(CMAKE_SYSTEM_NAME STREQUAL "Android") + if(MATSDK_ANDROID_HTTP_CLIENT_UPPER STREQUAL "AUTO") + set(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED "JAVA") + else() + set(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED "${MATSDK_ANDROID_HTTP_CLIENT_UPPER}") + endif() + + if(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED STREQUAL "CURL") + set(MATSDK_ANDROID_USES_CURL ON) + elseif(MATSDK_ANDROID_HTTP_CLIENT_RESOLVED STREQUAL "JAVA") + set(MATSDK_ANDROID_USES_JAVA_HTTP ON) + endif() + message(STATUS "MATSDK_ANDROID_HTTP_CLIENT: ${MATSDK_ANDROID_HTTP_CLIENT} -> ${MATSDK_ANDROID_HTTP_CLIENT_RESOLVED}") +endif() + # Enable Azure Monitor / Application Insights end-point support option(BUILD_AZMON "Build for Azure Monitor" YES) @@ -381,7 +410,7 @@ endif() set(MATSDK_NEEDS_CURL OFF) if(PAL_IMPLEMENTATION STREQUAL "CPP11" AND NOT BUILD_IOS - AND (NOT CMAKE_SYSTEM_NAME STREQUAL "Android" OR MATSDK_USE_VCPKG_DEPS) + AND (NOT CMAKE_SYSTEM_NAME STREQUAL "Android" OR MATSDK_ANDROID_USES_CURL) AND NOT BUILD_APPLE_HTTP) set(MATSDK_NEEDS_CURL ON) add_definitions(-DHAVE_MAT_CURL_HTTP_CLIENT) diff --git a/cmake/MSTelemetryConfig.cmake.in b/cmake/MSTelemetryConfig.cmake.in index af3838d65..8d63ac1f0 100644 --- a/cmake/MSTelemetryConfig.cmake.in +++ b/cmake/MSTelemetryConfig.cmake.in @@ -14,8 +14,9 @@ find_dependency(ZLIB) find_dependency(nlohmann_json CONFIG) # Curl is re-found only when the SDK was built with the curl HTTP client -# (Linux, Android via vcpkg, and macOS built without Apple HTTP). -# Windows (WinInet), iOS, and macOS-with-Apple-HTTP do not link curl. +# (Linux, explicit Android curl builds, and macOS built without Apple HTTP). +# Windows (WinInet), default Android Java/JNI HTTP, iOS, and +# macOS-with-Apple-HTTP do not link curl. # We bake the build-time decision into a boolean rather than re-deriving it, # because the macOS BUILD_APPLE_HTTP choice can't be inferred from # CMAKE_SYSTEM_NAME alone. @@ -27,6 +28,18 @@ if(@MATSDK_NEEDS_CURL@) find_dependency(CURL CONFIG) endif() +if("@MATSDK_ANDROID_HTTP_CLIENT_RESOLVED@" STREQUAL "") + set(MSTelemetry_ANDROID_HTTP_CLIENT "NONE") +else() + set(MSTelemetry_ANDROID_HTTP_CLIENT "@MATSDK_ANDROID_HTTP_CLIENT_RESOLVED@") +endif() +if(MSTelemetry_ANDROID_HTTP_CLIENT STREQUAL "JAVA") + set(MSTelemetry_ANDROID_JAVA_SOURCE_DIR + "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_DATADIR@/cpp-client-telemetry/android/java") +else() + set(MSTelemetry_ANDROID_JAVA_SOURCE_DIR "") +endif() + # Pthreads are needed on Linux and Android (POSIX threading) if(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") find_dependency(Threads) diff --git a/docs/building-with-vcpkg.md b/docs/building-with-vcpkg.md index 315fa8662..fed2dbdcf 100644 --- a/docs/building-with-vcpkg.md +++ b/docs/building-with-vcpkg.md @@ -139,6 +139,62 @@ scaffolding and not part of the published package.) Supported triplets: `arm64-android`, `arm-neon-android`, `x64-android`, `x86-android`. +#### Android HTTP transport + +The CMake option `MATSDK_ANDROID_HTTP_CLIENT` selects the Android HTTP transport: + +| Value | Behavior | +| ----- | -------- | +| `AUTO` | Default. Uses the Android Java/JNI transport. | +| `CURL` | Builds the native libcurl transport. This is an explicit escape hatch and requires one Android curl backend feature. | +| `JAVA` | Builds `HttpClient_Android`, which calls the Android Java bridge via JNI. | + +For vcpkg, Android uses Java transport by default: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": ["system-sqlite"] + } + ] +} +``` + +To opt into native curl on Android, select exactly one Android curl backend: + +```json +{ + "dependencies": [ + { + "name": "cpp-client-telemetry", + "default-features": false, + "features": ["android-curl-openssl", "system-sqlite"] + } + ] +} +``` + +Use `android-curl-mbedtls` in place of `android-curl-openssl` for the mbedTLS +backend. + +When Java transport is selected, the package installs the bridge sources under: + +```text +share/cpp-client-telemetry/android/java/com/microsoft/applications/events/ +``` + +The installed bridge contains `HttpClient.java` and `HttpClientRequest.java`. +Consumers are responsible for compiling those Java sources into their Android +application/AAR and constructing `com.microsoft.applications.events.HttpClient` +so it initializes the native `HttpClient_Android` singleton before telemetry is +uploaded. The bridge imports AndroidX annotations (`@Keep`, `@NonNull`, +`@Nullable`, `@RequiresApi`), so ensure `androidx.annotation:annotation` is on +the Java compile classpath, for example as a Gradle `compileOnly` or +`implementation` dependency. + ## Dependencies The vcpkg port automatically resolves the following dependencies: @@ -148,7 +204,7 @@ The vcpkg port automatically resolves the following dependencies: | SQLite3 | `sqlite3` | `unofficial::sqlite3::sqlite3` | Non-Apple (default; see `minimal-sqlite`). **macOS/iOS link the system `libsqlite3`** (`SQLite::SQLite3`) | | zlib | `zlib` | `ZLIB::ZLIB` | Non-Apple. **macOS/iOS link the system `libz`** | | nlohmann JSON | `nlohmann-json` | `nlohmann_json::nlohmann_json` | All | -| libcurl | `curl[openssl]` or `curl[mbedtls]` | `CURL::libcurl` | Non-Windows, non-Apple (required; TLS backend selectable: OpenSSL default or mbedTLS) | +| libcurl | `curl[openssl]` or `curl[mbedtls]` | `CURL::libcurl` | Linux by default; Android only when `android-curl-openssl` or `android-curl-mbedtls` is selected | On **macOS/iOS** the SDK links the OS-provided `libsqlite3` and `libz` (the same system libraries the SDK's Swift Package links), so the vcpkg `sqlite3` and `zlib` @@ -160,17 +216,17 @@ feature. The `minimal-sqlite` feature replaces it with a private, feature-stripp SQLite built from the SDK's vendored amalgamation — see [Build a private minimal SQLite](#build-a-private-minimal-sqlite-minimal-sqlite-feature). -libcurl is provided by the default `curl-openssl` feature; `curl-mbedtls` swaps in -the mbedTLS backend — see -[Choose the HTTP client / TLS backend](#choose-the-http-client--tls-backend-largest-lever-on-linux). +On Linux, libcurl is provided by the default `curl-openssl` feature; +`curl-mbedtls` swaps in the mbedTLS backend — see +[Choose the Linux HTTP client / TLS backend](#choose-the-linux-http-client--tls-backend-largest-lever-on-linux). Windows and macOS/iOS use platform-native HTTP clients (WinInet and -NSURLSession respectively). Android vcpkg consumers use native libcurl because -the Java-backed `HttpClient_Android` singleton is initialized by the repo's -Android Gradle/AAR flow, not by standalone native vcpkg consumers. +NSURLSession respectively). Android defaults to the platform Java/JNI HTTP +bridge; native curl is available only through explicit `android-curl-*` features. > **Note (Windows):** The port targets the MSVC/`WIN32` PAL on Windows, which -> uses WinInet, so `curl` is declared for `linux | android` only. A MinGW / +> uses WinInet, so the default `curl` dependency is declared for Linux only +> (Android has separate explicit `android-curl-*` features). A MinGW / > non-MSVC Windows triplet — or forcing `-DPAL_IMPLEMENTATION=CPP11` on Windows — > selects the curl HTTP client, which the port does not provision on Windows > (broadening `curl` to `windows` would pull an unused curl into every MSVC @@ -244,13 +300,13 @@ the stripping happens at your link. Keep the SDK a static dependency linked *into* your binary: if you re-export its API across your own DLL boundary, the export table pins its symbols and defeats `/OPT:REF`. -### Choose the HTTP client / TLS backend (largest lever on Linux) +### Choose the Linux HTTP client / TLS backend (largest lever on Linux) -On Linux/Android the built-in HTTP client is libcurl, and curl's TLS backend -dominates the SDK's footprint. (Windows uses WinInet and Apple uses NSURLSession, -so this section does not apply there.) The port exposes the TLS backend as two -mutually-exclusive features; pick the one that matches what your application -already has: +On Linux the built-in HTTP client is libcurl, and curl's TLS backend dominates +the SDK's footprint. (Windows uses WinInet, Apple uses NSURLSession, and Android +uses the Java/JNI bridge by default, so this section does not apply there.) The +port exposes the Linux TLS backend as two mutually-exclusive features; pick the +one that matches what your application already has: | Feature | Transport | Approx. stripped size¹ | Use when | | ------- | --------- | ---------------------- | -------- | @@ -261,7 +317,8 @@ already has: (worst case); enabling `-Wl,--gc-sections` at your link reduces them. Your numbers depend on triplet, dead-stripping, and what else shares those libraries. -To select **mbedTLS**, two things are required in *your top-level* manifest: +To select **mbedTLS on Linux**, two things are required in *your top-level* +manifest: ```json { @@ -290,7 +347,9 @@ To select **mbedTLS**, two things are required in *your top-level* manifest: verified with `vcpkg install --dry-run`. The default install (no features specified) keeps `curl-openssl` and works out of -the box. +the box on Linux. Android uses the Java/JNI HTTP bridge by default; use +`android-curl-openssl` or `android-curl-mbedtls` only when you explicitly want +the native curl Android escape hatch. ### Drop unused SQLite features (json1) @@ -347,12 +406,14 @@ Enable it through the vcpkg feature: Use the `[core,minimal-sqlite]` form (here, `"default-features": false` is the `[core]` part) so the default `system-sqlite` feature — and its `sqlite3` -dependency — is dropped. Because `[core]` drops **all** defaults, the example -also re-selects `curl-openssl`: on Linux/Android the built-in curl client -requires a TLS backend, so omitting it would fail to configure (swap in -`curl-mbedtls` for the smaller mbedTLS backend). Requesting `minimal-sqlite` -*without* `[core]` still pulls in the default `system-sqlite`; that is harmless -(the external `sqlite3` is installed but unused) but does not save the dependency. +dependency — is dropped. Because `[core]` drops **all** defaults, Linux examples +also re-select `curl-openssl`; on Linux the built-in curl client requires a TLS +backend, so omitting it would fail to configure (swap in `curl-mbedtls` for the +smaller mbedTLS backend). Android does not need a curl feature unless you +explicitly opt into `android-curl-openssl` or `android-curl-mbedtls`. +Requesting `minimal-sqlite` *without* `[core]` still pulls in the default +`system-sqlite`; that is harmless (the external `sqlite3` is installed but +unused) but does not save the dependency. For a plain (non-vcpkg) CMake build, pass the option directly: @@ -382,7 +443,9 @@ unchanged against the minimal build. When the SDK detects it is being built via vcpkg (by checking for `VCPKG_TOOLCHAIN` or `VCPKG_TARGET_TRIPLET`), it automatically sets `MATSDK_USE_VCPKG_DEPS=ON`. This switches dependency resolution from -vendored sources to vcpkg-provided packages via `find_package()`. +vendored sources to vcpkg-provided packages via `find_package()`. Android HTTP +transport selection is controlled separately by `MATSDK_ANDROID_HTTP_CLIENT`, +which defaults to `JAVA` on Android. You can also set this explicitly for custom CMake workflows: diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 97428a101..13b4d46d4 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -197,7 +197,7 @@ if(PAL_IMPLEMENTATION STREQUAL "CPP11") list(APPEND SRCS pal/posix/NetworkInformationImpl_Android.cpp ) - if(MATSDK_USE_VCPKG_DEPS) + if(MATSDK_ANDROID_USES_CURL) list(APPEND SRCS http/HttpClient_Curl.cpp http/HttpClient_Curl.hpp @@ -680,6 +680,14 @@ if(MATSDK_USE_VCPKG_DEPS) "${CMAKE_CURRENT_BINARY_DIR}/MSTelemetryConfigVersion.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/MSTelemetry ) + + if(CMAKE_SYSTEM_NAME STREQUAL "Android" AND MATSDK_ANDROID_USES_JAVA_HTTP) + install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/android_build/maesdk/src/main/java/com/microsoft/applications/events/HttpClient.java" + "${CMAKE_CURRENT_SOURCE_DIR}/android_build/maesdk/src/main/java/com/microsoft/applications/events/HttpClientRequest.java" + DESTINATION "${CMAKE_INSTALL_DATADIR}/cpp-client-telemetry/android/java/com/microsoft/applications/events" + ) + endif() else() # Legacy install: just put the library and headers in standard locations install(TARGETS mat diff --git a/lib/pal/posix/DeviceInformationImpl_Android.cpp b/lib/pal/posix/DeviceInformationImpl_Android.cpp index 61746d664..fc132629d 100644 --- a/lib/pal/posix/DeviceInformationImpl_Android.cpp +++ b/lib/pal/posix/DeviceInformationImpl_Android.cpp @@ -52,8 +52,10 @@ namespace PAL_NS_BEGIN { ///// IDeviceInformation API DeviceInformationImpl::DeviceInformationImpl(IRuntimeConfig& configuration) : + m_os_architecture(OsArchitectureType_Unknown), + m_powerSource(PowerSource_Battery), m_info_helper(), - m_powerSource(PowerSource_Battery) + m_registeredCount(0) {} std::string DeviceInformationImpl::GetDeviceTicket() const @@ -260,4 +262,3 @@ Java_com_microsoft_applications_events_HttpClient_onPowerChange(JNIEnv* env, PAL::AndroidDeviceInformationConnector::setModel(std::string(start, end)); env->ReleaseStringUTFChars(model, start); } - diff --git a/lib/pal/posix/NetworkInformationImpl_Android.cpp b/lib/pal/posix/NetworkInformationImpl_Android.cpp index 0c12464b4..04f1960f5 100644 --- a/lib/pal/posix/NetworkInformationImpl_Android.cpp +++ b/lib/pal/posix/NetworkInformationImpl_Android.cpp @@ -42,8 +42,10 @@ namespace PAL_NS_BEGIN { NetworkCost AndroidNetcostConnector::s_cost = NetworkCost_Unknown; NetworkInformationImpl::NetworkInformationImpl(IRuntimeConfig& configuration) : - m_info_helper(), + m_type(NetworkType_Unknown), m_cost(NetworkCost_Unknown), + m_info_helper(), + m_registeredCount(0), m_isNetDetectEnabled(configuration[CFG_BOOL_ENABLE_NET_DETECT]){}; NetworkInformationImpl::~NetworkInformationImpl() {}; @@ -156,4 +158,3 @@ Java_com_microsoft_applications_events_HttpClient_onCostChange(JNIEnv* env, { PAL::AndroidNetcostConnector::UpdateCost(isMetered ? NetworkCost_Metered : NetworkCost_Unmetered); } - diff --git a/lib/pal/posix/SystemInformationImpl_Android.cpp b/lib/pal/posix/SystemInformationImpl_Android.cpp index 15e0bb9b1..b1911f8ae 100644 --- a/lib/pal/posix/SystemInformationImpl_Android.cpp +++ b/lib/pal/posix/SystemInformationImpl_Android.cpp @@ -163,8 +163,8 @@ namespace PAL_NS_BEGIN { std::string AndroidSystemInformationConnector::s_device_class; SystemInformationImpl::SystemInformationImpl(IRuntimeConfig& configuration) : - m_info_helper(), - m_os_name("Android") + m_os_name("Android"), + m_info_helper() { if (configuration.HasConfig(CFG_PTR_ANDROID_JVM) && configuration.HasConfig(CFG_JOBJECT_ANDROID_ACTIVITY)) { @@ -245,4 +245,3 @@ extern "C" JNIEXPORT void JNICALL Java_com_microsoft_applications_events_HttpCli PAL::AndroidSystemInformationConnector::s_device_class, deviceClass); } - diff --git a/tools/ports/cpp-client-telemetry/portfile.cmake b/tools/ports/cpp-client-telemetry/portfile.cmake index c9ce3168c..b2fdab830 100644 --- a/tools/ports/cpp-client-telemetry/portfile.cmake +++ b/tools/ports/cpp-client-telemetry/portfile.cmake @@ -1,8 +1,7 @@ -# In-repo port validation (tests/vcpkg/*) sets MATSDK_VCPKG_SOURCE_DIR so the port -# builds the working tree under review instead of a pinned release -- this is what -# lets the port tests actually exercise the SDK source + manifest together. When -# the variable is unset (production installs), the pinned release is downloaded as -# usual, so the published port behavior is unchanged. +# In-repo overlay-port use should build the working tree under review instead of +# a pinned release -- this is what lets local port installs and tests exercise +# the SDK source + manifest together. The registry copy of this port is not under +# the SDK checkout, so it falls back to the pinned release below. if(DEFINED ENV{MATSDK_VCPKG_SOURCE_DIR}) set(SOURCE_PATH "$ENV{MATSDK_VCPKG_SOURCE_DIR}") if(NOT EXISTS "${SOURCE_PATH}/CMakeLists.txt") @@ -12,6 +11,18 @@ if(DEFINED ENV{MATSDK_VCPKG_SOURCE_DIR}) endif() message(STATUS "cpp-client-telemetry: building local source $ENV{MATSDK_VCPKG_SOURCE_DIR} (MATSDK_VCPKG_SOURCE_DIR is set)") else() + get_filename_component(_matsdk_overlay_source "${CURRENT_PORT_DIR}/../../.." ABSOLUTE) +endif() + +if(NOT DEFINED SOURCE_PATH + AND EXISTS "${_matsdk_overlay_source}/CMakeLists.txt" + AND EXISTS "${_matsdk_overlay_source}/lib/CMakeLists.txt" + AND EXISTS "${_matsdk_overlay_source}/tools/ports/cpp-client-telemetry/portfile.cmake") + set(SOURCE_PATH "${_matsdk_overlay_source}") + message(STATUS "cpp-client-telemetry: building in-repo overlay source ${SOURCE_PATH}") +endif() + +if(NOT DEFINED SOURCE_PATH) vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO microsoft/cpp_client_telemetry @@ -35,39 +46,63 @@ if(VCPKG_TARGET_IS_IOS) set(MATSDK_BUILD_IOS ON) endif() -# curl-openssl (default) and curl-mbedtls choose the TLS backend for the built-in -# HTTP client and are mutually exclusive. They only matter on Linux/Android: the -# curl dependency is platform-filtered to those triplets, so on Windows/macOS/iOS -# both features may be present (curl-openssl is a default) yet pull no curl, and -# the SDK uses WinInet / Apple HTTP there. vcpkg cannot express mutual exclusivity -# or "exactly one of", so validate it here -- but only where curl is actually used, -# to avoid failing legitimate cross-platform manifests on Windows/Apple. +set(MATSDK_ANDROID_HTTP_CLIENT AUTO) +if(VCPKG_TARGET_IS_ANDROID) + file(READ "${SOURCE_PATH}/CMakeLists.txt" _matsdk_root_cmake) + if(NOT _matsdk_root_cmake MATCHES "MATSDK_ANDROID_HTTP_CLIENT") + message(FATAL_ERROR + "Android vcpkg builds require a cpp-client-telemetry source revision that " + "supports MATSDK_ANDROID_HTTP_CLIENT. Update this port's REF/SHA512 to a " + "newer SDK release, or set MATSDK_VCPKG_SOURCE_DIR to a local checkout " + "that contains the Android Java transport selector.") + endif() + if("android-curl-openssl" IN_LIST FEATURES OR "android-curl-mbedtls" IN_LIST FEATURES) + set(MATSDK_ANDROID_HTTP_CLIENT CURL) + endif() +endif() + +# curl-openssl/curl-mbedtls choose the Linux TLS backend. Android defaults to +# Java/JNI HTTP and uses separate explicit android-curl-* features for its curl +# escape hatch. vcpkg cannot express mutual exclusivity or "exactly one of", so +# validate it here -- but only where curl is actually used, to avoid failing +# legitimate cross-platform manifests on Windows/Apple. set(_matsdk_http_features "") -foreach(_matsdk_http_feature curl-openssl curl-mbedtls) +if(VCPKG_TARGET_IS_ANDROID) + set(_matsdk_http_feature_candidates android-curl-openssl android-curl-mbedtls) +else() + set(_matsdk_http_feature_candidates curl-openssl curl-mbedtls) +endif() +foreach(_matsdk_http_feature ${_matsdk_http_feature_candidates}) if(_matsdk_http_feature IN_LIST FEATURES) list(APPEND _matsdk_http_features ${_matsdk_http_feature}) endif() endforeach() list(LENGTH _matsdk_http_features _matsdk_http_feature_count) -if(VCPKG_TARGET_IS_LINUX OR VCPKG_TARGET_IS_ANDROID) +if(VCPKG_TARGET_IS_LINUX OR MATSDK_ANDROID_HTTP_CLIENT STREQUAL "CURL") if(_matsdk_http_feature_count GREATER 1) message(FATAL_ERROR - "curl-openssl (default) and curl-mbedtls are mutually exclusive but both were " - "selected. To use mbedTLS, drop the defaults with the [core,...] form and " - "re-select a SQLite backend (the [core,...] form also drops the default " - "system-sqlite feature), e.g. " - "cpp-client-telemetry[core,curl-mbedtls,system-sqlite] " - "(or minimal-sqlite in place of system-sqlite).") - elseif(_matsdk_http_feature_count EQUAL 0) + "The curl HTTP backend features are mutually exclusive but multiple were " + "selected. On Linux, use exactly one of curl-openssl/curl-mbedtls. On " + "Android, use exactly one of android-curl-openssl/android-curl-mbedtls. " + "If you added a non-default backend, use the [core,...] form " + "(default-features=false) so the default curl-openssl feature is dropped.") + elseif(_matsdk_http_feature_count EQUAL 0 AND VCPKG_TARGET_IS_LINUX) # The built-in curl HTTP client requires exactly one TLS backend. The [core,...] # form drops the default curl-openssl, so fail fast (with a complete example) # rather than letting the SDK CMake fail later on a missing libcurl. message(FATAL_ERROR - "On Linux/Android the built-in curl HTTP client requires exactly one TLS " - "backend feature, but none was selected. The [core,...] form drops the " - "default curl-openssl feature, so re-add a curl backend together with a " - "SQLite backend, e.g. cpp-client-telemetry[core,curl-openssl,system-sqlite] " - "(or curl-mbedtls / minimal-sqlite in place of those).") + "On Linux the built-in curl HTTP client requires exactly one TLS backend " + "feature, but none was selected. The [core,...] form drops the default " + "curl-openssl feature, so re-add a curl backend together with a SQLite " + "backend, e.g. " + "cpp-client-telemetry[core,curl-mbedtls,system-sqlite] " + "(or minimal-sqlite in place of system-sqlite).") + elseif(_matsdk_http_feature_count EQUAL 0) + message(FATAL_ERROR + "On Android, MATSDK_ANDROID_HTTP_CLIENT=CURL requires exactly one explicit " + "Android curl backend feature. Use android-curl-openssl or " + "android-curl-mbedtls together with a SQLite backend, e.g. " + "cpp-client-telemetry[core,android-curl-openssl,system-sqlite].") endif() endif() @@ -83,6 +118,7 @@ vcpkg_cmake_configure( OPTIONS ${FEATURE_OPTIONS} -DMATSDK_USE_VCPKG_DEPS=ON + -DMATSDK_ANDROID_HTTP_CLIENT=${MATSDK_ANDROID_HTTP_CLIENT} -DBUILD_HEADERS=ON -DBUILD_LIBRARY=ON -DBUILD_TEST_TOOL=OFF diff --git a/tools/ports/cpp-client-telemetry/vcpkg.json b/tools/ports/cpp-client-telemetry/vcpkg.json index e8a9eb5a5..d183bf6ca 100644 --- a/tools/ports/cpp-client-telemetry/vcpkg.json +++ b/tools/ports/cpp-client-telemetry/vcpkg.json @@ -21,25 +21,27 @@ } ], "default-features": [ - "system-sqlite", - "curl-openssl" + "curl-openssl", + "system-sqlite" ], "features": { - "system-sqlite": { - "description": "Link the external vcpkg sqlite3 package for the offline storage cache (default). On macOS/iOS the SDK links the system libsqlite3 instead, so this dependency is not pulled there.", + "android-curl-mbedtls": { + "description": "On Android, explicitly build the native libcurl HTTP client with the mbedTLS backend instead of the default Java/JNI HttpClient_Android bridge.", + "supports": "android", "dependencies": [ { - "name": "sqlite3", + "name": "curl", "default-features": false, - "platform": "!osx & !ios" + "features": [ + "mbedtls" + ], + "platform": "android" } ] }, - "minimal-sqlite": { - "description": "Build a private, feature-stripped SQLite compiled from the SDK's vendored amalgamation instead of linking the external sqlite3 package. Smaller footprint; combine with [core,minimal-sqlite] to also drop the sqlite3 dependency." - }, - "curl-openssl": { - "description": "Built-in libcurl HTTP client with the OpenSSL TLS backend (default). Affects Linux/Android only; Windows uses WinInet and Apple uses NSURLSession regardless.", + "android-curl-openssl": { + "description": "On Android, explicitly build the native libcurl HTTP client with the OpenSSL backend instead of the default Java/JNI HttpClient_Android bridge.", + "supports": "android", "dependencies": [ { "name": "curl", @@ -47,12 +49,12 @@ "features": [ "openssl" ], - "platform": "linux | android" + "platform": "android" } ] }, "curl-mbedtls": { - "description": "Built-in libcurl HTTP client with the mbedTLS backend instead of OpenSSL (smaller footprint). Affects Linux/Android only. Use [core,curl-mbedtls,system-sqlite] to drop the default OpenSSL curl; the [core,...] form drops all defaults (including system-sqlite), so also re-select system-sqlite or minimal-sqlite.", + "description": "Built-in libcurl HTTP client with the mbedTLS backend instead of OpenSSL (smaller footprint). Affects Linux only. Use [core,curl-mbedtls,system-sqlite] to drop the default OpenSSL curl; the [core,...] form drops all defaults (including system-sqlite), so also re-select system-sqlite or minimal-sqlite.", "dependencies": [ { "name": "curl", @@ -60,7 +62,33 @@ "features": [ "mbedtls" ], - "platform": "linux | android" + "platform": "linux" + } + ] + }, + "curl-openssl": { + "description": "Built-in libcurl HTTP client with the OpenSSL TLS backend (default). Affects Linux only; Android uses the Java/JNI bridge unless an android-curl-* feature is selected, Windows uses WinInet, and Apple uses NSURLSession.", + "dependencies": [ + { + "name": "curl", + "default-features": false, + "features": [ + "openssl" + ], + "platform": "linux" + } + ] + }, + "minimal-sqlite": { + "description": "Build a private, feature-stripped SQLite compiled from the SDK's vendored amalgamation instead of linking the external sqlite3 package. Smaller footprint; combine with [core,minimal-sqlite] to also drop the sqlite3 dependency." + }, + "system-sqlite": { + "description": "Link the external vcpkg sqlite3 package for the offline storage cache (default). On macOS/iOS the SDK links the system libsqlite3 instead, so this dependency is not pulled there.", + "dependencies": [ + { + "name": "sqlite3", + "default-features": false, + "platform": "!osx & !ios" } ] } From f89b0021902dd925674e7e1748dd255a9512e7a7 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 29 Jul 2026 19:26:48 -0500 Subject: [PATCH 22/30] Create the offline event cache with owner-only (0600) permissions (#1506) * Create the offline event cache with owner-only (0600) permissions The SQLite offline cache buffers pending telemetry events (tenant ids, user identifiers, serialized event payloads) but was created with SQLite's default file permissions (0644 -- world-readable), letting any co-located user on a POSIX system read the buffered event stream or tamper with pending events. Restrict the database file to 0600 immediately after opening it in SQLiteWrapper::open. This runs before WAL is enabled, so the -wal/-journal companion files inherit 0600 from the main database file (SQLite's findCreateFileMode derives their mode from the main db). The chmod is best-effort (a failure, e.g. an in-memory ":memory:" database, does not fail the open) and POSIX-only -- on Windows the Unix mode bits are meaningless (NTFS ACLs govern access). Adds a POSIX unit test asserting the cache and its companions are not group/world accessible. Verified on Linux (umask 022): a bare SQLite db is created 0644; with this change the cache and its -wal/-journal are 0600. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Also tighten pre-existing companion files on open (round 1) Address Copilot review: only the main database was chmod'd, so a cache created by an older SDK (or a companion left behind after a crash) could retain the old world-readable 0644 mode on its -wal/-shm/-journal files. Best-effort chmod any pre-existing companions to 0600 on open, ignoring ENOENT. Add ExistingFilesAreTightenedOnOpen covering the migration path (loosen db + plant a leftover -wal, then verify reopen re-tightens both). Files: - lib/offline/SQLiteWrapper.hpp - tests/unittests/OfflineStorageTests_SQLite.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Silence in-memory chmod warning and clean up test companions (round 2) Address Copilot review round 2: - The main-database chmod now ignores ENOENT, so opening an in-memory (":memory:") database -- which LogManagerImpl uses when no tenant token is configured -- no longer logs a spurious permission-tightening warning on every open (there is no file to secure). - ExistingFilesAreTightenedOnOpen now removes the database and its companion files (including the planted -wal) at the end, so they don't leak into other tests that reuse the same storage filename. Files: - lib/offline/SQLiteWrapper.hpp - tests/unittests/OfflineStorageTests_SQLite.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Remove WAL companion files in shared test teardown (round 3) Address Copilot review round 3: CacheFileCreatedOwnerReadWriteOnly (and any other WAL-mode test) relied on shutdownAndRemoveFile(), which only removed the main database and could leave -wal/-shm/-journal companions behind to pollute the temp dir. Remove the companions in the shared teardown helper so every test is covered, and drop the now-redundant explicit cleanup from ExistingFilesAreTightenedOnOpen. Files: - tests/unittests/OfflineStorageTests_SQLite.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/offline/SQLiteWrapper.hpp | 30 ++++++++ .../unittests/OfflineStorageTests_SQLite.cpp | 72 +++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/lib/offline/SQLiteWrapper.hpp b/lib/offline/SQLiteWrapper.hpp index 982b4053b..2a5f0d108 100644 --- a/lib/offline/SQLiteWrapper.hpp +++ b/lib/offline/SQLiteWrapper.hpp @@ -16,6 +16,11 @@ #include #include +#if !defined(_WIN32) +#include +#include +#endif + namespace MAT_NS_BEGIN { using SQLRecord = std::vector; @@ -300,6 +305,31 @@ namespace MAT_NS_BEGIN { g_sqlite3Proxy->sqlite3_extended_result_codes(m_db, 1); + // SECURITY: the offline cache buffers pending telemetry/audit events + // (tenant ids, user identifiers, serialized event payloads). SQLite creates + // the database file with SQLITE_DEFAULT_FILE_PERMISSIONS -- 0644, i.e. + // world-readable -- so restrict it to owner read/write only (0600). This runs + // before WAL is enabled: SQLite derives the -wal/-journal permissions from the + // main database file (findCreateFileMode), so companions it creates inherit + // 0600. A cache created by an older SDK (before this fix) may already have + // companion files on disk with the old 0644 mode, so tighten any pre-existing + // ones too. POSIX only -- on Windows the Unix mode bits are meaningless (access + // is governed by NTFS ACLs). Best-effort: a failure (e.g. a filesystem that + // ignores chmod) must not fail the open, and a missing file -- ENOENT, e.g. an + // in-memory ":memory:" database, which has no file to secure -- is expected and + // silently ignored. +#if !defined(_WIN32) + if (::chmod(filename.c_str(), S_IRUSR | S_IWUSR) != 0 && errno != ENOENT) { + LOG_WARN("Could not restrict database file permissions to 0600 (errno %d)", errno); + } + for (const char* suffix : { "-wal", "-shm", "-journal" }) { + std::string companion = filename + suffix; + if (::chmod(companion.c_str(), S_IRUSR | S_IWUSR) != 0 && errno != ENOENT) { + LOG_WARN("Could not restrict %s file permissions to 0600 (errno %d)", suffix, errno); + } + } +#endif + if (!registerTokenizeFunction()) { shutdown(); return false; diff --git a/tests/unittests/OfflineStorageTests_SQLite.cpp b/tests/unittests/OfflineStorageTests_SQLite.cpp index e90b0a9ae..d5aa6808a 100644 --- a/tests/unittests/OfflineStorageTests_SQLite.cpp +++ b/tests/unittests/OfflineStorageTests_SQLite.cpp @@ -12,6 +12,9 @@ #include "offline/OfflineStorage_SQLite.hpp" #include #include +#if !defined(_WIN32) +#include +#endif #include "NullObjects.hpp" @@ -89,6 +92,12 @@ struct OfflineStorageTests_SQLite : public Test EXPECT_THAT(fileExists(storageFilename), true); ::remove(storageFilename.c_str()); EXPECT_THAT(fileExists(storageFilename), false); + // WAL mode can leave -wal/-shm/-journal companions behind; remove them too + // so they don't leak into other tests that reuse the same storage filename. + for (const char* suffix : { "-wal", "-shm", "-journal" }) + { + ::remove((storageFilename + suffix).c_str()); + } } } @@ -839,4 +848,67 @@ TEST_F(OfflineStorageTests_SQLite, SqliteDbInstancesAreCounted) shutdownAndRemoveFile(); EXPECT_EQ(offlineStorage->GetDbInstanceCount(), 0); } + +#if !defined(_WIN32) +// SECURITY: the offline cache buffers pending telemetry/audit events, so it must +// not be world-readable. SQLite creates the file 0644 by default; SQLiteWrapper +// tightens it to 0600 after open, and the -wal/-journal companions inherit that +// mode from the main database file. POSIX-only (mode bits are meaningless on +// Windows, where access is governed by NTFS ACLs). +TEST_F(OfflineStorageTests_SQLite, CacheFileCreatedOwnerReadWriteOnly) +{ + initializeStorage(); + + struct stat st; + ASSERT_EQ(0, ::stat(storageFilename.c_str(), &st)) << "cache database file was not created"; + EXPECT_EQ(static_cast(S_IRUSR | S_IWUSR), static_cast(st.st_mode & 0777)) + << "offline cache database must be created 0600, not world-readable"; + + // Any WAL/journal/shm companion that exists must not grant group or other access + // (SQLite derives their permissions from the main database file's mode). + for (const char* suffix : { "-wal", "-journal", "-shm" }) + { + struct stat cst; + const std::string companion = storageFilename + suffix; + if (::stat(companion.c_str(), &cst) == 0) + { + EXPECT_EQ(0, static_cast(cst.st_mode & (S_IRWXG | S_IRWXO))) + << "companion file " << suffix << " must not be group/world accessible"; + } + } +} + +// A cache written by an older SDK (or left behind after a crash) can have the +// database and companion files already on disk with the old world-readable 0644 +// mode; SQLite only derives 0600 for companions it creates itself. Opening the +// storage must re-tighten both the database and any pre-existing companion. +TEST_F(OfflineStorageTests_SQLite, ExistingFilesAreTightenedOnOpen) +{ + initializeStorage(); + offlineStorage->Shutdown(); + storageInitialized = false; + + // Simulate an old cache: loosen the database and plant a leftover -wal at 0644. + const mode_t loose = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; // 0644 + ASSERT_EQ(0, ::chmod(storageFilename.c_str(), loose)); + const std::string wal = storageFilename + "-wal"; + std::ofstream(wal, std::ios::binary); // empty leftover companion + ASSERT_EQ(0, ::chmod(wal.c_str(), loose)); + + // Reopen -- the open path must re-tighten both files. + initializeStorage(); + + struct stat st; + ASSERT_EQ(0, ::stat(storageFilename.c_str(), &st)); + EXPECT_EQ(0, static_cast(st.st_mode & (S_IRWXG | S_IRWXO))) + << "reopened database must be tightened to 0600"; + + struct stat wst; + if (::stat(wal.c_str(), &wst) == 0) + { + EXPECT_EQ(0, static_cast(wst.st_mode & (S_IRWXG | S_IRWXO))) + << "pre-existing -wal companion must be tightened to 0600"; + } +} +#endif #endif From 8be7fc61310414690a7852382cb4543522cd4855 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 29 Jul 2026 20:09:48 -0500 Subject: [PATCH 23/30] Cap curl HTTP response body size to prevent memory-amplification DoS (#1507) * Cap curl HTTP response body size to prevent memory-amplification DoS The libcurl transport buffered the entire collector response with no upper bound: WriteMemoryCallback and WriteVectorCallback grew (realloc / vector insert) until the whole body was received. A hostile or MITM'd collector could return an arbitrarily large body and drive unbounded memory growth in the embedding process. OneCollector responses are tiny (status, kill-switch tokens, retry-after, small config), so a generous fixed cap never rejects a legitimate response. Add a 16 MB cap (kMaxResponseBytes) enforced overflow-safely in both curl write callbacks; exceeding it returns a short count so curl aborts the transfer with CURLE_WRITE_ERROR (the upload becomes a network failure and is retried). Add HttpClientCurlResponseCapTests covering the oversized-abort and large-under-cap paths. Files: - lib/http/HttpClient_Curl.hpp - tests/unittests/HttpClientCurlTests.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Guard response cap against overflow and fix test request lifetime (round 1) Address Copilot review on the response-size cap: - Guard the size * nmemb product in both curl write callbacks against size_t multiplication overflow before using it, so the "overflow-safe" cap check cannot operate on a wrapped length. - The response-cap test fixture now owns the IHttpRequest (the client only stores a raw pointer and never frees it) and releases it in TearDown on the main thread, fixing the per-test leak. Freeing it in OnHttpResponse would destroy the CurlHttpOperation from within its own async task (whose destructor waits on that task -- a self-join deadlock), so teardown-time release is used instead. sendAndWait() also resets result state up front so the helper is safe to reuse. Files: - lib/http/HttpClient_Curl.hpp - tests/unittests/HttpClientCurlTests.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Synchronize test response-size field to avoid a data race (round 2) Address Copilot review round 2: m_responseBodySize was written by the test thread in sendAndWait() and read by the HttpServer reactor thread in onHttpRequest() without synchronization -- a data race that would trip TSAN. Write it under the existing mutex and read it under the same lock (into a local) to establish a happens-before edge. Files: - tests/unittests/HttpClientCurlTests.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/http/HttpClient_Curl.hpp | 34 ++++++- tests/unittests/HttpClientCurlTests.cpp | 115 ++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/lib/http/HttpClient_Curl.hpp b/lib/http/HttpClient_Curl.hpp index b1bb5344c..533c522e3 100644 --- a/lib/http/HttpClient_Curl.hpp +++ b/lib/http/HttpClient_Curl.hpp @@ -484,6 +484,14 @@ class CurlHttpOperation { return poll(&pfd, 1, static_cast(timeout)); } + // SECURITY: upper bound on the collector response the client will buffer. The + // OneCollector protocol responses (status, kill-switch tokens, retry-after, small + // config) are tiny, so this generous cap never rejects a legitimate response but + // stops a hostile or MITM'd collector from driving unbounded memory growth by + // returning an oversized body (a memory-amplification DoS of the embedding process). + // Exceeding it aborts the transfer, so the upload is treated as failed and retried. + static constexpr size_t kMaxResponseBytes = 16 * 1024 * 1024; // 16 MB + // Raw response buffer struct MemoryStruct { char *memory; @@ -501,9 +509,21 @@ class CurlHttpOperation { */ static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) { + // Guard the size * nmemb product against size_t overflow before using it. + if (nmemb != 0 && size > static_cast(-1) / nmemb) { + return 0; + } size_t realsize = size * nmemb; struct MemoryStruct *mem = (struct MemoryStruct *)userp; + // SECURITY: bound the buffered response (see kMaxResponseBytes). Compare + // overflow-safely (mem->size is always <= kMaxResponseBytes here). Returning a + // short count aborts the transfer with CURLE_WRITE_ERROR. + if (realsize > kMaxResponseBytes - mem->size) { + TRACE("Response exceeds max buffered size (%zu bytes); aborting transfer\n", kMaxResponseBytes); + return 0; + } + auto* memory = static_cast(realloc(mem->memory, mem->size + realsize + 1)); if(memory == nullptr) { /* out of memory! */ @@ -533,9 +553,21 @@ class CurlHttpOperation { */ static size_t WriteVectorCallback(void *ptr, size_t size, size_t nmemb, std::vector* data) { + // Guard the size * nmemb product against size_t overflow before using it. + if (nmemb != 0 && size > static_cast(-1) / nmemb) { + return 0; + } if (data != nullptr) { + size_t realsize = size * nmemb; + // SECURITY: bound the buffered response (see kMaxResponseBytes). Compare + // overflow-safely (data->size() is always <= kMaxResponseBytes here). + // Returning a short count aborts the transfer with CURLE_WRITE_ERROR. + if (realsize > kMaxResponseBytes - data->size()) { + TRACE("Response exceeds max buffered size (%zu bytes); aborting transfer\n", kMaxResponseBytes); + return 0; + } const auto* begin = static_cast(ptr); - const auto* end = begin + size * nmemb; + const auto* end = begin + realsize; data->insert( data->end(), begin, end); } return size * nmemb; diff --git a/tests/unittests/HttpClientCurlTests.cpp b/tests/unittests/HttpClientCurlTests.cpp index c9894b90d..d494ba2fc 100644 --- a/tests/unittests/HttpClientCurlTests.cpp +++ b/tests/unittests/HttpClientCurlTests.cpp @@ -9,6 +9,7 @@ && !defined(__APPLE__) && !defined(ANDROID) #include "common/Common.hpp" +#include "common/HttpServer.hpp" #include "http/HttpClient_Curl.hpp" #include "config/RuntimeConfig_Default.hpp" @@ -126,4 +127,118 @@ TEST_F(HttpClientCurlTests, SetSslVerification_ConcurrentCallsNoRace) SUCCEED(); } +// --- Response-size cap (memory-amplification DoS hardening) --- + +class HttpClientCurlResponseCapTests : public ::testing::Test, + public HttpServer::Callback, + public IHttpResponseCallback +{ +protected: + HttpServer m_server; + HttpClient_Curl m_client; + // The client never takes ownership of the request (it only stores a raw pointer + // and erases it); the fixture owns it and frees it in TearDown -- on the main + // thread, after the transfer has completed. Freeing it inside OnHttpResponse + // would destroy the CurlHttpOperation from within its own async task, whose + // destructor waits on that task (a self-join deadlock). + std::unique_ptr m_request; + std::string m_hostname; + size_t m_responseBodySize {0}; + + std::mutex m_lock; + bool m_received {false}; + HttpResult m_result {}; + unsigned int m_statusCode {0}; + size_t m_bodySize {0}; + + void SetUp() override + { + int port = m_server.addListeningPort(0); + std::ostringstream os; + os << "127.0.0.1:" << port; + m_hostname = os.str(); + m_server.setServerName(m_hostname); + m_server.addHandler("/huge/", *this); + m_server.start(); + } + + void TearDown() override + { + m_server.stop(); + m_request.reset(); + } + + // HttpServer::Callback -- returns a body of m_responseBodySize bytes. + int onHttpRequest(HttpServer::Request const& /*request*/, HttpServer::Response& response) override + { + size_t bodySize; + { + std::lock_guard lock(m_lock); + bodySize = m_responseBodySize; + } + response.headers["Content-Type"] = "application/octet-stream"; + response.content = std::string(bodySize, 'A'); + return 200; + } + + // IHttpResponseCallback -- the SDK hands over ownership of the response. + void OnHttpResponse(IHttpResponse* response) override + { + std::unique_ptr owned(response); + std::lock_guard lock(m_lock); + m_result = owned->GetResult(); + m_statusCode = owned->GetStatusCode(); + m_bodySize = owned->GetBody().size(); + m_received = true; + } + + bool responseReceived() + { + std::lock_guard lock(m_lock); + return m_received; + } + + void sendAndWait(size_t bodySize) + { + { + std::lock_guard lock(m_lock); + m_received = false; + m_result = HttpResult{}; + m_statusCode = 0; + m_bodySize = 0; + m_responseBodySize = bodySize; // read under the same lock by onHttpRequest + } + m_request.reset(m_client.CreateRequest()); + m_request->SetUrl("http://" + m_hostname + "/huge/"); + m_client.SendRequestAsync(m_request.get(), this); + for (int i = 0; i < 300 && !responseReceived(); i++) + PAL::sleep(100); + } +}; + +TEST_F(HttpClientCurlResponseCapTests, AbortsOversizedResponseBody) +{ + // A response body larger than the client's response-size cap (kMaxResponseBytes, + // 16 MB) must be refused, not buffered in full, so a hostile/MITM'd collector + // cannot exhaust process memory. + sendAndWait(17u * 1024u * 1024u); + ASSERT_TRUE(responseReceived()); + // curl aborts the transfer (CURLE_WRITE_ERROR) once the cap is hit -> NetworkFailure. + EXPECT_EQ(m_result, HttpResult_NetworkFailure); + // The oversized body is never fully buffered. + EXPECT_LE(m_bodySize, static_cast(16u * 1024u * 1024u)); +} + +TEST_F(HttpClientCurlResponseCapTests, AcceptsLargeResponseUnderCap) +{ + // A large-but-legitimate response (well under the cap) must still be received + // in full: the cap must not regress normal responses. + const size_t bodySize = 4u * 1024u * 1024u; + sendAndWait(bodySize); + ASSERT_TRUE(responseReceived()); + EXPECT_EQ(m_result, HttpResult_OK); + EXPECT_EQ(m_statusCode, 200u); + EXPECT_EQ(m_bodySize, bodySize); +} + #endif // MATSDK_PAL_CPP11 && !_MSC_VER && HAVE_MAT_DEFAULT_HTTP_CLIENT From e23275148aade102aca0d84bf3ce82420287b54f Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Wed, 29 Jul 2026 23:03:05 -0500 Subject: [PATCH 24/30] Fix version.js build-version script under the WSH JScript engine (#1505) String.prototype.trim() (added in #1455 to complete the newline sanitization) is not implemented by the Windows Script Host JScript engine that runs version.js via cscript, so every Windows CI build logged 'version.js(48,3) Microsoft JScript runtime error: Object doesn't support this property or method'. Replace trim() with an ES3-compatible global-anchored regex that strips leading/trailing whitespace (including all trailing newlines), so the script runs cleanly while keeping the complete-sanitization behavior CodeQL asked for. Verified: 'cscript //nologo version.js' now exits 0 and regenerates Version.hpp; confirmed the JScript engine rejects .trim() with the same error seen in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/version.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/version.js b/tools/version.js index 4f090e1be..45c4c922e 100644 --- a/tools/version.js +++ b/tools/version.js @@ -44,8 +44,11 @@ function generateVersionHpp() { // Read version tag var ver1 = readAll("..\\Solutions\\version.txt"); - // Remove end-of-line - ver1 = ver1.trim(); + // Remove leading/trailing whitespace. Use a regex rather than String.trim() so this + // runs under the Windows Script Host JScript engine (cscript), which does not + // implement String.prototype.trim(); the global anchored pattern also fully strips + // trailing newlines (the CodeQL incomplete-sanitization concern). + ver1 = ver1.replace(/^\s+|\s+$/g, ""); ver1 = updateYearAndDay(ver1); // console.log("version.txt => " + ver1 + "\n"); var ver2 = ver1.split(".").join(","); From 97e90d124127e32aba824c2140c51533756960e2 Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Thu, 30 Jul 2026 15:22:41 -0500 Subject: [PATCH 25/30] Cap HTTP response body size across WinInet, WinRt, and Apple transports (#1508) * Cap HTTP response body size across WinInet, WinRt, and Apple transports Extends the memory-amplification DoS hardening (a hostile or MITM'd collector returning an oversized body to exhaust process memory) beyond the libcurl transport to the remaining platform transports. Introduces a single shared constant MAX_HTTP_RESPONSE_SIZE (16 MB) in IHttpClient.hpp so every transport uses the same generous ceiling, well above any legitimate OneCollector or config response. - WinInet: bound m_bodyBuffer in the InternetReadFile loop; over-cap aborts the read and the request is reported as a failure (retried). - WinRt: reject a ReadAsBufferAsync buffer whose length exceeds the cap without copying it; report NetworkFailure. - Apple (NSURLSession): reject a completion-handler NSData larger than the cap without copying it; report NetworkFailure. The libcurl transport is capped separately in its own focused change; a later cleanup can unify its constant onto MAX_HTTP_RESPONSE_SIZE. Files: - lib/include/public/IHttpClient.hpp - lib/http/HttpClient_WinInet.cpp - lib/http/HttpClient_WinRt.cpp - lib/http/HttpClient_Apple.mm Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Map WinInet oversize to NetworkFailure and guard Apple nil data (round 1) Address Copilot review: - WinInet: the oversize-response abort set ERROR_NOT_ENOUGH_MEMORY, which fell through to the default case (LocalFailure). Use ERROR_HTTP_INVALID_SERVER_- RESPONSE so it maps to HttpResult_NetworkFailure, consistent with the WinRt and Apple transports (still retried, but correctly classified). - Apple: guard the success-path copy on a non-zero length and cast data.length to size_t, so a nil NSData (bytes == nullptr) never performs pointer arithmetic on nullptr (undefined behavior). Files: - lib/http/HttpClient_WinInet.cpp - lib/http/HttpClient_Apple.mm Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Stream response bodies to enforce the cap before full materialization (round 2) Address Copilot review round 2: the previous checks ran after the framework had already materialized the whole response body, so an oversized response could still drive a large allocation. Rework each transport to bound memory to the cap: - WinInet: check before every append (pre-loop and in-loop) so m_bodyBuffer never exceeds MAX_HTTP_RESPONSE_SIZE, not "cap + one chunk". (Validated: the Windows `mat` library compiles.) - WinRt: request with HttpCompletionOption::ResponseHeadersRead (so the body is not pre-buffered) and stream it via ReadAsInputStreamAsync in 64 KB chunks, aborting the moment the cap would be exceeded. - Apple: replace the completionHandler NSURLSession API (which materializes the full NSData) with a streaming NSURLSessionDataDelegate that accumulates in didReceiveData: and cancels the task once the cap would be exceeded; an over-cap transfer is surfaced as NetworkFailure (retried). The WinRt and Apple rewrites target UWP/macOS toolchains that aren't available locally, so they are review-verified and must be built/tested on-device before merge. Files: - lib/http/HttpClient_WinInet.cpp - lib/http/HttpClient_WinRt.cpp - lib/http/HttpClient_Apple.mm Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Harden streaming transports: guard WinRt read faults, fix Apple block cast (round 3) Address Copilot review round 3 on the streaming rework: - WinRt: concurrency::task::wait()/get() rethrow if ReadAsInputStreamAsync or a chunk ReadAsync faults (e.g., connection reset) even when the status looks completed. Wrap the whole streamed-read in try/catch so a fault maps to HttpResult_NetworkFailure instead of escaping onRequestComplete and crashing. - Apple: cast the dictionary value (stored as id) back to the concrete block type in didCompleteWithError: to avoid an incompatible-pointer-types warning (which fails builds under -Werror). Files: - lib/http/HttpClient_WinRt.cpp - lib/http/HttpClient_Apple.mm Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * WinRt: map cancel to Aborted and clear partial body on rejection (round 4) Address Copilot review round 4: HttpResponseDecoder processes any non-empty response body regardless of HttpResult (processBody runs when GetBody() is non-empty), so a partial body left on a rejected streamed response could be parsed for kill-switch/stats. In the WinRt streaming reader: - Map a caller-initiated cancellation (task_status::canceled, from cancel()) to HttpResult_Aborted instead of NetworkFailure. - Clear response->m_body on every non-success path (cancel, read failure, over-cap, and streaming exceptions) so no partial body is processed. (WinInet and Apple never attach a partial body to the response on rejection, so they need no change.) Files: - lib/http/HttpClient_WinRt.cpp Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix Apple response-cap delegate cleanup: NSMutableSet uses removeObject: The streaming session delegate's didCompleteWithError: cleanup called [_overCap removeObjectForKey:key], but _overCap is an NSMutableSet, which has no removeObjectForKey: selector (that belongs to NSMutableDictionary). This was a copy-paste from the _handlers/_buffers dictionary cleanup two lines above and fails to compile, breaking the entire Apple/macOS mat build. Use the correct NSMutableSet selector, removeObject:. Validation (macOS arm64, Apple HTTP transport): - libmat builds clean; full host UnitTests 518/518 pass. - End-to-end test against a local HttpServer through the real HttpClient_Apple: under-cap (64 KB) -> HttpResult_OK with full body; over-cap (16 MB + 1 MB) -> HttpResult_NetworkFailure with an empty body and no crash; exactly MAX_HTTP_RESPONSE_SIZE (16 MB) -> HttpResult_OK with full body. Stable over 8 repeats (no delegate state races). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- lib/http/HttpClient_Apple.mm | 132 +++++++++++++++++++++++++++-- lib/http/HttpClient_WinInet.cpp | 52 +++++++----- lib/http/HttpClient_WinRt.cpp | 109 ++++++++++++++++++------ lib/include/public/IHttpClient.hpp | 13 +++ 4 files changed, 257 insertions(+), 49 deletions(-) diff --git a/lib/http/HttpClient_Apple.mm b/lib/http/HttpClient_Apple.mm index 449b4c9af..b7d6646a4 100644 --- a/lib/http/HttpClient_Apple.mm +++ b/lib/http/HttpClient_Apple.mm @@ -15,6 +15,111 @@ #include "utils/StringUtils.hpp" #include "utils/Utils.hpp" +// Streams the response body in bounded chunks and enforces MAX_HTTP_RESPONSE_SIZE. +// The completionHandler-based NSURLSession APIs fully materialize the response body +// as an NSData before handing it over, so an attacker-controlled collector could force +// a large allocation. This delegate instead accumulates data incrementally in +// didReceiveData: and cancels the transfer as soon as the cap would be exceeded, so no +// more than the cap is ever buffered. Delegate callbacks may arrive on the session's +// delegate queue while a request thread registers a task, so shared state is guarded. +@interface MATStreamingSessionDelegate : NSObject +- (void)registerTask:(NSURLSessionTask*)task + handler:(void (^)(NSData* data, NSURLResponse* response, NSError* error))handler; +@end + +@implementation MATStreamingSessionDelegate { + NSMutableDictionary* _buffers; + NSMutableDictionary* _handlers; + NSMutableSet* _overCap; +} + +- (instancetype)init +{ + self = [super init]; + if (self) + { + _buffers = [NSMutableDictionary new]; + _handlers = [NSMutableDictionary new]; + _overCap = [NSMutableSet new]; + } + return self; +} + +- (void)registerTask:(NSURLSessionTask*)task + handler:(void (^)(NSData*, NSURLResponse*, NSError*))handler +{ + NSNumber* key = @(task.taskIdentifier); + @synchronized(self) + { + _buffers[key] = [NSMutableData new]; + _handlers[key] = [handler copy]; + } +} + +- (void)URLSession:(NSURLSession*)session + dataTask:(NSURLSessionDataTask*)dataTask + didReceiveData:(NSData*)data +{ + NSNumber* key = @(dataTask.taskIdentifier); + @synchronized(self) + { + if ([_overCap containsObject:key]) + { + return; + } + NSMutableData* buffer = _buffers[key]; + if (buffer == nil) + { + return; + } + if (buffer.length + data.length > MAT::MAX_HTTP_RESPONSE_SIZE) + { + // Refuse the over-large response: stop buffering and cancel the transfer. + [_overCap addObject:key]; + [dataTask cancel]; + return; + } + [buffer appendData:data]; + } +} + +- (void)URLSession:(NSURLSession*)session + task:(NSURLSessionTask*)task +didCompleteWithError:(NSError*)error +{ + NSNumber* key = @(task.taskIdentifier); + void (^handler)(NSData*, NSURLResponse*, NSError*) = nil; + NSData* body = nil; + BOOL overCap = NO; + @synchronized(self) + { + handler = (void (^)(NSData*, NSURLResponse*, NSError*))_handlers[key]; + body = _buffers[key]; + overCap = [_overCap containsObject:key]; + [_handlers removeObjectForKey:key]; + [_buffers removeObjectForKey:key]; + [_overCap removeObject:key]; + } + if (handler == nil) + { + return; + } + if (overCap) + { + // Surface a non-cancellation error so the request maps to NetworkFailure + // (retried), not Aborted (which is reserved for caller-initiated cancels). + NSError* capError = [NSError errorWithDomain:@"MATResponseCap" + code:-1 + userInfo:@{ NSLocalizedDescriptionKey : @"HTTP response exceeds max buffered size" }]; + handler(nil, task.response, capError); + } + else + { + handler(body, task.response, error); + } +} +@end + namespace MAT_NS_BEGIN { static std::string NextReqId() @@ -31,6 +136,7 @@ static dispatch_once_t once; static NSURLSession* session; +static MATStreamingSessionDelegate* sessionDelegate; class HttpRequestApple : public SimpleHttpRequest { @@ -42,7 +148,10 @@ m_parent->Add(static_cast(this)); dispatch_once(&once, ^{ NSURLSessionConfiguration* sessionConfig = [NSURLSessionConfiguration defaultSessionConfiguration]; - session = [NSURLSession sessionWithConfiguration:sessionConfig]; + sessionDelegate = [MATStreamingSessionDelegate new]; + session = [NSURLSession sessionWithConfiguration:sessionConfig + delegate:sessionDelegate + delegateQueue:nil]; }); } @@ -75,15 +184,18 @@ void SendAsync(IHttpResponseCallback* callback) if(equalsIgnoreCase(m_method, "get")) { [m_urlRequest setHTTPMethod:@"GET"]; - m_dataTask = [session dataTaskWithRequest:m_urlRequest completionHandler:m_completionMethod]; + m_dataTask = [session dataTaskWithRequest:m_urlRequest]; } else { [m_urlRequest setHTTPMethod:@"POST"]; NSData* postData = [NSData dataWithBytes:m_body.data() length:m_body.size()]; - m_dataTask = [session uploadTaskWithRequest:m_urlRequest fromData:postData completionHandler:m_completionMethod]; + m_dataTask = [session uploadTaskWithRequest:m_urlRequest fromData:postData]; } + // Register before resume so the streaming delegate has the buffer and + // completion handler in place before any response data arrives. + [sessionDelegate registerTask:m_dataTask handler:m_completionMethod]; [m_dataTask resume]; } } @@ -120,10 +232,18 @@ void HandleResponse(NSData* data, NSURLResponse* response, NSError* error) } else { + // The streaming delegate has already enforced MAX_HTTP_RESPONSE_SIZE + // (an over-cap response arrives here as a cap error, handled above), so + // data is bounded. Guard against a nil/empty body to avoid pointer + // arithmetic on a null [data bytes]. simpleResponse->m_result = HttpResult_OK; - auto body = static_cast([data bytes]); - simpleResponse->m_body.reserve(data.length); - std::copy(body, body + data.length, std::back_inserter(simpleResponse->m_body)); + const size_t length = static_cast(data.length); + if (length > 0) + { + auto body = static_cast([data bytes]); + simpleResponse->m_body.reserve(length); + std::copy(body, body + length, std::back_inserter(simpleResponse->m_body)); + } } m_callback->OnHttpResponse(simpleResponse); } diff --git a/lib/http/HttpClient_WinInet.cpp b/lib/http/HttpClient_WinInet.cpp index eaefb2318..b1d3b4013 100644 --- a/lib/http/HttpClient_WinInet.cpp +++ b/lib/http/HttpClient_WinInet.cpp @@ -324,27 +324,41 @@ class WinInetRequestWrapper // It might potentially be another async operation which will // trigger INTERNET_STATUS_REQUEST_COMPLETE again. - m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); - while (!m_readingData || m_bufferUsed != 0) { - BOOL bResult = ::InternetReadFile(m_hWinInetRequest, m_buffer, sizeof(m_buffer), &m_bufferUsed); - m_readingData = true; - if (!bResult) { - dwError = GetLastError(); - if (dwError == ERROR_IO_PENDING) { - // Do not touch anything from this thread anymore. - // The buffer passed to InternetReadFile() and the - // read count will be filled asynchronously, so they - // must stay valid and writable until the next - // INTERNET_STATUS_REQUEST_COMPLETE callback comes - // (that's why those are member variables). - LOG_TRACE("InternetReadFile() failed: ERROR_IO_PENDING. Waiting for INTERNET_STATUS_REQUEST_COMPLETE to be called again"); - return; + // SECURITY: refuse an over-large response instead of buffering it (see + // MAX_HTTP_RESPONSE_SIZE) so a hostile/MITM'd collector cannot exhaust + // process memory. Checked before every append so the buffer never exceeds + // the cap; reported as an invalid server response -> NetworkFailure (retried). + if (m_bodyBuffer.size() + m_bufferUsed > MAX_HTTP_RESPONSE_SIZE) { + LOG_WARN("HTTP response exceeds max buffered size (%zu bytes); aborting", MAX_HTTP_RESPONSE_SIZE); + dwError = ERROR_HTTP_INVALID_SERVER_RESPONSE; + } else { + m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); + while (!m_readingData || m_bufferUsed != 0) { + BOOL bResult = ::InternetReadFile(m_hWinInetRequest, m_buffer, sizeof(m_buffer), &m_bufferUsed); + m_readingData = true; + if (!bResult) { + dwError = GetLastError(); + if (dwError == ERROR_IO_PENDING) { + // Do not touch anything from this thread anymore. + // The buffer passed to InternetReadFile() and the + // read count will be filled asynchronously, so they + // must stay valid and writable until the next + // INTERNET_STATUS_REQUEST_COMPLETE callback comes + // (that's why those are member variables). + LOG_TRACE("InternetReadFile() failed: ERROR_IO_PENDING. Waiting for INTERNET_STATUS_REQUEST_COMPLETE to be called again"); + return; + } + LOG_WARN("InternetReadFile() failed: %d", dwError); + break; } - LOG_WARN("InternetReadFile() failed: %d", dwError); - break; - } - m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); + if (m_bodyBuffer.size() + m_bufferUsed > MAX_HTTP_RESPONSE_SIZE) { + LOG_WARN("HTTP response exceeds max buffered size (%zu bytes); aborting", MAX_HTTP_RESPONSE_SIZE); + dwError = ERROR_HTTP_INVALID_SERVER_RESPONSE; + break; + } + m_bodyBuffer.insert(m_bodyBuffer.end(), m_buffer, m_buffer + m_bufferUsed); + } } } diff --git a/lib/http/HttpClient_WinRt.cpp b/lib/http/HttpClient_WinRt.cpp index e46e4c49c..1efc1bb22 100644 --- a/lib/http/HttpClient_WinRt.cpp +++ b/lib/http/HttpClient_WinRt.cpp @@ -144,7 +144,7 @@ namespace MAT_NS_BEGIN { void SendHttpAsyncRequest(HttpRequestMessage ^req) { - IAsyncOperationWithProgress^ operation = m_parent.getHttpClient()->SendRequestAsync(req, HttpCompletionOption::ResponseContentRead); + IAsyncOperationWithProgress^ operation = m_parent.getHttpClient()->SendRequestAsync(req, HttpCompletionOption::ResponseHeadersRead); m_cancellationTokenSource = cancellation_token_source(); create_task(operation, m_cancellationTokenSource.get_token()). @@ -202,37 +202,98 @@ namespace MAT_NS_BEGIN { index++; } - auto operation = m_httpResponseMessage->Content->ReadAsBufferAsync(); - auto task = create_task(operation); - if (task.wait() == task_status::completed) + // Read content headers before streaming the body. + IMapView^ contentHeadersView = m_httpResponseMessage->Content->Headers->GetView(); + auto contentHeadersiterator = contentHeadersView->First(); + unsigned int contentHeadersIndex = 0; + while (contentHeadersIndex < contentHeadersView->Size) { - IMapView^ contentHeadersView = m_httpResponseMessage->Content->Headers->GetView(); + String^ Key = contentHeadersiterator->Current->Key; + String^ Value = contentHeadersiterator->Current->Value; - auto contentHeadersiterator = contentHeadersView->First(); - unsigned int contentHeadersIndex = 0; - while (contentHeadersIndex < contentHeadersView->Size) - { - String^ Key = contentHeadersiterator->Current->Key; - String^ Value = contentHeadersiterator->Current->Value; + response->m_headers.add(from_platform_string(Key), from_platform_string(Value)); + contentHeadersiterator->MoveNext(); + contentHeadersIndex++; + } - response->m_headers.add(from_platform_string(Key), from_platform_string(Value)); - contentHeadersiterator->MoveNext(); - contentHeadersIndex++; + // SECURITY: stream the body in bounded chunks and enforce + // MAX_HTTP_RESPONSE_SIZE. SendRequestAsync uses ResponseHeadersRead, so + // the framework does not pre-buffer the whole body; reading it here in + // chunks ensures an oversized response is never fully materialized in + // memory (a hostile/MITM'd collector cannot exhaust process memory). + // task::wait()/get() rethrow if a read faults, so guard the whole stream. + try + { + IInputStream^ inputStream = nullptr; + { + auto streamOp = m_httpResponseMessage->Content->ReadAsInputStreamAsync(); + auto streamTask = create_task(streamOp, m_cancellationTokenSource.get_token()); + auto status = streamTask.wait(); + if (status == task_status::completed) + { + inputStream = streamTask.get(); + } + else + { + // Caller-initiated cancel maps to Aborted; anything else is a failure. + response->m_result = (status == task_status::canceled) ? HttpResult_Aborted : HttpResult_NetworkFailure; + } } - auto buffer = task.get(); - size_t length = buffer->Length; - - if (length > 0) + if (inputStream != nullptr) { - response->m_body.reserve(length); - response->m_body.resize(length); - DataReader^ dataReader = DataReader::FromBuffer(buffer); - dataReader->ReadBytes((Platform::ArrayReference(reinterpret_cast(response->m_body.data()), (DWORD)length))); - dataReader->DetachBuffer(); - delete dataReader; + const unsigned int chunkSize = 64 * 1024; + for (;;) + { + Buffer^ chunk = ref new Buffer(chunkSize); + auto readOp = inputStream->ReadAsync(chunk, chunkSize, InputStreamOptions::Partial); + auto readTask = create_task(readOp, m_cancellationTokenSource.get_token()); + auto status = readTask.wait(); + if (status != task_status::completed) + { + // Drop any partial body; caller cancel -> Aborted, else failure. + response->m_result = (status == task_status::canceled) ? HttpResult_Aborted : HttpResult_NetworkFailure; + response->m_body.clear(); + break; + } + + IBuffer^ readBuffer = readTask.get(); + unsigned int readLength = (readBuffer != nullptr) ? readBuffer->Length : 0; + if (readLength == 0) + { + break; // end of stream + } + + if (response->m_body.size() + readLength > MAX_HTTP_RESPONSE_SIZE) + { + LOG_WARN("HTTP response exceeds max buffered size (%zu bytes); aborting", MAX_HTTP_RESPONSE_SIZE); + response->m_result = HttpResult_NetworkFailure; + response->m_body.clear(); + break; + } + + const size_t oldSize = response->m_body.size(); + response->m_body.resize(oldSize + readLength); + DataReader^ dataReader = DataReader::FromBuffer(readBuffer); + dataReader->ReadBytes((Platform::ArrayReference(reinterpret_cast(response->m_body.data() + oldSize), readLength))); + dataReader->DetachBuffer(); + delete dataReader; + } + delete inputStream; } } + catch (Platform::Exception^ ex) + { + // A faulted read rethrows here; drop any partial body and fail the request. + LOG_WARN("Reading HTTP response body failed: 0x%08x", ex->HResult); + response->m_result = HttpResult_NetworkFailure; + response->m_body.clear(); + } + catch (...) + { + response->m_result = HttpResult_NetworkFailure; + response->m_body.clear(); + } } else { diff --git a/lib/include/public/IHttpClient.hpp b/lib/include/public/IHttpClient.hpp index 89e5e6cf0..7a8678ceb 100644 --- a/lib/include/public/IHttpClient.hpp +++ b/lib/include/public/IHttpClient.hpp @@ -14,11 +14,24 @@ #include #include #include +#include ///@cond INTERNAL_DOCS namespace MAT_NS_BEGIN { class ILogConfiguration; + + /// + /// SECURITY: upper bound (in bytes) on an HTTP response body that a transport + /// will buffer. OneCollector protocol responses are small (status, kill-switch + /// tokens, retry-after, small config), so this generous cap never rejects a + /// legitimate response, but it stops a hostile or MITM'd collector from driving + /// unbounded memory growth by returning an oversized body (a memory-amplification + /// DoS of the embedding process). A transport that would exceed it refuses the + /// response and reports the request as a network failure so it is retried. + /// + static constexpr std::size_t MAX_HTTP_RESPONSE_SIZE = 16u * 1024u * 1024u; // 16 MB + /// /// The HttpHeaders class contains a set of HTTP headers. /// From e3f238516573072ffe51489d6fe4ca6ef5f2843e Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Thu, 30 Jul 2026 19:39:58 -0500 Subject: [PATCH 26/30] Add a public-header CI gate matching stricter warning flags (#1503) * Add a public-header gate CI job; make CompliantByDefaultFilterApi self-contained Consumers that embed the SDK (e.g. ONNX Runtime / Foundry Local) compile their own translation units -- which include our public headers -- under strict warning flags: -Wall -Wextra -Werror on GCC/Clang (plus -Wshorten-64-to-32 on Clang) and /W4 /WX on MSVC, suppressing third-party headers via -isystem / /external:W0. When that suppression is defeated (include order, PCH, or NO_SYSTEM_FROM_IMPORTED), any warning or missing include in our headers breaks the consumer build. This adds a CI gate that compiles every public header on its own, with no -isystem suppression, under those flags on GCC, Clang, and MSVC, so header issues surface here instead of at integration time: - tests/headers/check_public_headers.sh (GCC + Clang) - tests/headers/check_public_headers.cmd (MSVC) - .github/workflows/public-header-gate.yml The gate caught one real self-containment bug: CompliantByDefaultFilterApi.hpp uses std::vector but did not include , so it only compiled when something else pulled in first. Added the include. VariantType.hpp is excluded: it is an implementation fragment included by Variant.hpp (which defines VariantMap/VariantArray and the needed std headers first), not a standalone header. Validated locally: all 40 public headers pass (39 compiled + VariantType excluded) on g++, clang++, and MSVC cl /W4 /WX. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address Copilot round 1 on #1503: gate .h headers, tighten flags, fix MSVC includes - Gate both *.hpp and *.h so the flat C API headers (mat.h, CommonFields.h) are covered, not just *.hpp. Both compile clean standalone under the gate flags. - Drop -Wno-unused-but-set-variable from the GCC/Clang flags: no public header relies on it, so removing it makes the gate stricter. Kept -Wno-unused-parameter (mirrors this repo's WARN_FLAGS; intentional unused params use UNREFERENCED_PARAMETER). - MSVC gate: add /I lib\include so headers that conditionally pull mat/config.h (e.g. CsProtocol_types.hpp) exercise the same include graph as the GCC/Clang gate. Validated locally: 41/41 public headers pass on g++, clang++, and MSVC cl /W4 /WX. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fold the public-header gate into existing CI workflows Instead of a standalone workflow, add the header gate as an isolated job in the workflows that already run on the same triggers: - build-posix-latest.yml gains a 'public-headers' job (GCC/Clang on ubuntu). - test-win-latest.yml gains a 'public-headers' job (MSVC on windows). Removes .github/workflows/public-header-gate.yml. The gate scripts under tests/headers/ are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Header gate: drop -Wno-unused-parameter so it mirrors real consumer flags The gate suppressed -Wunused-parameter, which is not part of the strict consumer flag set it claims to mirror (-Wall -Wextra -Werror). That hid a real break: NullObjects.hpp overrides left parameters unused because the old UNREFERENCED_PARAMETER macro expanded to nothing on GCC/Clang, so a consumer including LogManager.hpp with plain -Wall -Wextra -Werror failed to compile while the gate passed. With the macro now expanding to (void)(...) (merged from main), the headers are clean without the suppression; drop it so the gate actually catches this class of consumer break. Verified: 41/41 public headers pass standalone on g++ and clang++ without -Wno-unused-parameter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fail the public-header gate when no headers are found or PUB is missing The gate could silently pass without compiling anything: with nullglob the shell header globs expand to nothing (and the batch FOR loop runs zero times) when the computed public-header directory is wrong, so a miscomputed path reported success while testing nothing. Both scripts now validate the public-header directory exists and fail if zero headers were compiled. Also give the MSVC script a unique per-invocation work directory (%TEMP%\pubhdrgate_) so concurrent runs on the same machine cannot clobber each other's temporary translation unit, and clean it up on exit. Files: tests/headers/check_public_headers.sh, tests/headers/check_public_headers.cmd Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Strengthen public header gate coverage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-posix-latest.yml | 11 ++ .github/workflows/test-win-latest.yml | 10 ++ .../public/CompliantByDefaultFilterApi.hpp | 1 + tests/headers/check_public_headers.cmd | 129 ++++++++++++++ tests/headers/check_public_headers.sh | 158 ++++++++++++++++++ 5 files changed, 309 insertions(+) create mode 100644 tests/headers/check_public_headers.cmd create mode 100644 tests/headers/check_public_headers.sh diff --git a/.github/workflows/build-posix-latest.yml b/.github/workflows/build-posix-latest.yml index 13ac881ab..8f9320e57 100644 --- a/.github/workflows/build-posix-latest.yml +++ b/.github/workflows/build-posix-latest.yml @@ -47,3 +47,14 @@ jobs: continue-on-error: true - name: Test ${{ matrix.os }} ${{ matrix.config }} run: ./build-tests.sh ${{ matrix.config }} + + public-headers: + name: Public header gate (GCC/Clang) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install clang + run: sudo apt-get update && sudo apt-get install -y clang + - name: Compile each public header standalone under strict flags + run: bash tests/headers/check_public_headers.sh diff --git a/.github/workflows/test-win-latest.yml b/.github/workflows/test-win-latest.yml index 255868a88..4928fc71f 100644 --- a/.github/workflows/test-win-latest.yml +++ b/.github/workflows/test-win-latest.yml @@ -54,3 +54,13 @@ jobs: - name: Test ${{ matrix.arch }} ${{ matrix.build }} shell: cmd run: build-tests.cmd ${{ matrix.arch }} ${{ matrix.build }} + + public-headers: + name: Public header gate (MSVC) + runs-on: windows-2022 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Compile each public header standalone under /W4 /WX + shell: cmd + run: tests\headers\check_public_headers.cmd diff --git a/lib/include/public/CompliantByDefaultFilterApi.hpp b/lib/include/public/CompliantByDefaultFilterApi.hpp index 642aa4c9e..85e779c3f 100644 --- a/lib/include/public/CompliantByDefaultFilterApi.hpp +++ b/lib/include/public/CompliantByDefaultFilterApi.hpp @@ -7,6 +7,7 @@ #include "ctmacros.hpp" +#include #include namespace MAT_NS_BEGIN { namespace Modules { namespace Filtering diff --git a/tests/headers/check_public_headers.cmd b/tests/headers/check_public_headers.cmd new file mode 100644 index 000000000..53860422c --- /dev/null +++ b/tests/headers/check_public_headers.cmd @@ -0,0 +1,129 @@ +@echo off +REM Copyright (c) Microsoft Corporation. All rights reserved. +REM SPDX-License-Identifier: Apache-2.0 +REM +REM Public header gate (MSVC). Compiles each public SDK header on its own under +REM /W4 /WX, mirroring how ONNX Runtime / Foundry Local compile their own C++ +REM translation units on Windows. STL/Windows SDK headers are treated as external +REM (/external:W0) so only the SDK's headers are gated. Exits non-zero if any +REM header fails to compile or emits a warning. Also compiles mat.h as C11 (/TC). +setlocal enabledelayedexpansion + +set "SCRIPT_DIR=%~dp0" +set "REPO_ROOT=%SCRIPT_DIR%..\.." +set "PUB=%REPO_ROOT%\lib\include\public" +set "C_API_HEADER=mat.h" + +REM Fail fast if the public header directory is missing or miscomputed, otherwise +REM the header loop below would run zero times and the gate would silently "pass" +REM without compiling anything -- a false negative. +if not exist "%PUB%" ( + echo error: public header directory not found: %PUB% 1>&2 + exit /b 2 +) + +REM Enter the MSVC x64 developer environment via vswhere (portable across runners). +set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" +if not exist "%VSWHERE%" ( + echo error: vswhere.exe not found 1>&2 + exit /b 2 +) +set "VSPATH=" +for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" -latest -products * -property installationPath`) do set "VSPATH=%%i" +if not defined VSPATH ( + echo error: no Visual Studio installation found 1>&2 + exit /b 2 +) +call "%VSPATH%\VC\Auxiliary\Build\vcvars64.bat" >nul +if errorlevel 1 ( + echo error: failed to initialize the MSVC environment 1>&2 + exit /b 2 +) + +REM Unique work directory under the repository so concurrent invocations on the +REM same machine do not clobber each other's temporary translation units. +set "WORK=%REPO_ROOT%\.public-header-gate_%RANDOM%_%RANDOM%" +if exist "%WORK%" rmdir /s /q "%WORK%" +mkdir "%WORK%" +if errorlevel 1 ( + echo error: failed to create work directory %WORK% 1>&2 + exit /b 2 +) + +REM /W4 /WX matches ORT; /external:W0 suppresses platform/STL warnings so only our headers gate. +set "CXX_COMMON=/nologo /permissive- /W4 /WX /EHsc /experimental:external /external:anglebrackets /external:W0" +set "C_COMMON=/nologo /std:c11 /TC /W4 /WX /experimental:external /external:anglebrackets /external:W0" +set "FAIL=0" +set "TOTAL=0" + +REM MSVC does not expose a /std:c++11 switch; /std:c++14 is its lowest selectable mode. +call :RunCxxHeaders c++14 /std:c++14 "cl (c++14, /W4 /WX)" +call :RunCxxHeaders c++17 /std:c++17 "cl (c++17, /W4 /WX)" +call :RunCHeader + +rmdir /s /q "%WORK%" 2>nul + +if "%FAIL%"=="1" ( + echo Public header gate FAILED. + exit /b 1 +) +echo Public header gate passed. ^(!TOTAL! checks^) +exit /b 0 + +:RunCxxHeaders +set "STD_NAME=%~1" +set "STD_FLAG=%~2" +set "LABEL=%~3" +set "OKC=0" +set "FAILC=0" +echo == %LABEL% == +for %%h in ("%PUB%\*.hpp" "%PUB%\*.h") do ( + set "NAME=%%~nxh" + REM Skip implementation-fragment headers not meant to be included standalone + REM (VariantType.hpp is included by Variant.hpp, which defines VariantMap/VariantArray first). + if /I not "!NAME!"=="VariantType.hpp" ( + > "%WORK%\tu_!STD_NAME!.cpp" echo #include "!NAME!" + >> "%WORK%\tu_!STD_NAME!.cpp" echo int main^(^){return 0;} + cl %CXX_COMMON% %STD_FLAG% /I "%PUB%" /I "%REPO_ROOT%\lib\include" /Zs "%WORK%\tu_!STD_NAME!.cpp" > "%WORK%\err.txt" 2>&1 + if errorlevel 1 ( + echo FAIL: !NAME! + type "%WORK%\err.txt" + set "FAIL=1" + set /a FAILC+=1 + ) else ( + set /a OKC+=1 + set /a TOTAL+=1 + ) + ) +) +REM No headers compiled means PUB matched nothing -- treat it as a failure rather +REM than a silent pass. +if "!OKC!"=="0" if "!FAILC!"=="0" ( + echo error: no public headers found under %PUB% 1>&2 + set "FAIL=1" +) +echo %LABEL%: !OKC! passed, !FAILC! failed +exit /b 0 + +:RunCHeader +set "LABEL=cl (C11 mat.h, /TC, /W4 /WX)" +echo == !LABEL! == +if not exist "%PUB%\%C_API_HEADER%" ( + echo error: C API header not found: %PUB%\%C_API_HEADER% 1>&2 + set "FAIL=1" + echo !LABEL!: 0 passed, 1 failed + exit /b 0 +) +> "%WORK%\tu_mat_c11.c" echo #include "%C_API_HEADER%" +>> "%WORK%\tu_mat_c11.c" echo int main^(void^){return 0;} +cl %C_COMMON% /I "%PUB%" /I "%REPO_ROOT%\lib\include" /Zs "%WORK%\tu_mat_c11.c" > "%WORK%\err.txt" 2>&1 +if errorlevel 1 ( + echo FAIL: %C_API_HEADER% + type "%WORK%\err.txt" + set "FAIL=1" + echo !LABEL!: 0 passed, 1 failed +) else ( + set /a TOTAL+=1 + echo !LABEL!: 1 passed, 0 failed +) +exit /b 0 diff --git a/tests/headers/check_public_headers.sh b/tests/headers/check_public_headers.sh new file mode 100644 index 000000000..79724e588 --- /dev/null +++ b/tests/headers/check_public_headers.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Public header gate (GCC/Clang). +# +# Verifies that every public SDK header is self-contained (compiles on its own, +# in any include order) and warning-clean under strict, consumer-representative +# warning flags. Downstream consumers such as ONNX Runtime / Foundry Local +# compile their own translation units -- which include these headers -- with +# -Wall -Wextra -Werror (plus -Wshorten-64-to-32 on Clang). This gate compiles +# each public header on its own as both C++11 and C++17, with no -isystem +# suppression, so any header issue surfaces here instead of at integration time. +# It also compiles the ABI-stable C API header (mat.h) as C11. +# +# Exits non-zero if any header fails to compile or emits a warning. + +set -uo pipefail +shopt -s nullglob + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PUB="$REPO_ROOT/lib/include/public" + +# Fail fast if the public header directory is missing or miscomputed. With +# nullglob on, a bad PUB would make the header globs below expand to nothing and +# the gate would silently "pass" without compiling anything -- a false negative. +if [ ! -d "$PUB" ]; then + echo "error: public header directory not found: $PUB" >&2 + exit 2 +fi + +# Implementation-fragment headers: intentionally included by another public +# header (which supplies their dependencies first) and not meant to be included +# standalone. They are exercised through their public entry point instead. +EXCLUDES=( + "VariantType.hpp" # included by Variant.hpp, which defines VariantMap/VariantArray first +) + +# Mirror the strict warning flags third-party consumers build with +# (-Wall -Wextra -Werror, matching this repo's pipeline and ORT). Unused +# parameters are deliberately NOT suppressed: intentional ones use the +# UNREFERENCED_PARAMETER macro, which expands to (void)(...) on GCC/Clang, so the +# headers stay warning-clean without a blanket -Wno-unused-parameter that would +# hide a real consumer break (e.g. an override that leaves a parameter unused). +C_API_HEADER="mat.h" + +is_excluded() { + local n="$1" e + for e in "${EXCLUDES[@]}"; do [ "$e" = "$n" ] && return 0; done + return 1 +} + +fail=0 +tmp="" +for _ in 1 2 3 4 5; do + candidate="$REPO_ROOT/.public-header-gate.$$.$RANDOM" + if mkdir "$candidate" 2>/dev/null; then + tmp="$candidate" + break + fi +done +if [ -z "$tmp" ]; then + echo "error: failed to create work directory under $REPO_ROOT" >&2 + exit 2 +fi +trap 'rm -rf "$tmp"' EXIT + +run_cxx_compiler() { + local cc="$1" std="$2" label="$3" + shift 3 + local n_ok=0 n_fail=0 h name base tu out + echo "== $label ==" + for h in "$PUB"/*.hpp "$PUB"/*.h; do + name="$(basename "$h")" + is_excluded "$name" && continue + base="${name%.*}" + tu="$tmp/tu_${base}_${std}.cpp" + printf '#include "%s"\nint main() { return 0; }\n' "$name" > "$tu" + if out="$("$cc" "-std=$std" -Wall -Wextra -Werror "$@" -I"$PUB" -I"$REPO_ROOT/lib/include" -fsyntax-only "$tu" 2>&1)"; then + n_ok=$((n_ok + 1)) + else + n_fail=$((n_fail + 1)); fail=1 + echo " FAIL: $name" + echo "$out" | grep -E 'error:|warning:' | head -4 | sed 's/^/ /' + fi + done + if [ $((n_ok + n_fail)) -eq 0 ]; then + echo " ERROR: no public headers found under $PUB" + fail=1 + fi + echo " $label: $n_ok passed, $n_fail failed" +} + +run_c_compiler() { + local cc="$1" label="$2" + shift 2 + local tu out + echo "== $label ==" + if [ ! -f "$PUB/$C_API_HEADER" ]; then + echo " ERROR: C API header not found: $PUB/$C_API_HEADER" + fail=1 + echo " $label: 0 passed, 1 failed" + return + fi + tu="$tmp/tu_${C_API_HEADER%.h}_c11.c" + printf '#include "%s"\nint main(void) { return 0; }\n' "$C_API_HEADER" > "$tu" + if out="$("$cc" -std=c11 -Wall -Wextra -Werror "$@" -I"$PUB" -I"$REPO_ROOT/lib/include" -fsyntax-only "$tu" 2>&1)"; then + echo " $label: 1 passed, 0 failed" + else + fail=1 + echo " FAIL: $C_API_HEADER" + echo "$out" | grep -E 'error:|warning:' | head -4 | sed 's/^/ /' + echo " $label: 0 passed, 1 failed" + fi +} + +cxx_ran=0 +if command -v g++ >/dev/null 2>&1; then + run_cxx_compiler g++ c++11 "g++ (c++11, -Wall -Wextra -Werror)" + run_cxx_compiler g++ c++17 "g++ (c++17, -Wall -Wextra -Werror)" + cxx_ran=1 +fi +if command -v clang++ >/dev/null 2>&1; then + run_cxx_compiler clang++ c++11 "clang++ (c++11, + -Wshorten-64-to-32)" -Wshorten-64-to-32 + run_cxx_compiler clang++ c++17 "clang++ (c++17, + -Wshorten-64-to-32)" -Wshorten-64-to-32 + cxx_ran=1 +fi + +if [ "$cxx_ran" -eq 0 ]; then + echo "error: neither g++ nor clang++ was found" >&2 + exit 2 +fi + +c_ran=0 +if command -v gcc >/dev/null 2>&1; then + run_c_compiler gcc "gcc (c11 mat.h, -Wall -Wextra -Werror)" + c_ran=1 +fi +if command -v clang >/dev/null 2>&1; then + run_c_compiler clang "clang (c11 mat.h, + -Wshorten-64-to-32)" -Wshorten-64-to-32 + c_ran=1 +fi +if [ "$c_ran" -eq 0 ] && command -v cc >/dev/null 2>&1; then + run_c_compiler cc "cc (c11 mat.h, -Wall -Wextra -Werror)" + c_ran=1 +fi + +if [ "$c_ran" -eq 0 ]; then + echo "error: no C compiler (gcc, clang, or cc) was found" >&2 + exit 2 +fi + +if [ "$fail" -ne 0 ]; then + echo "Public header gate FAILED." + exit 1 +fi +echo "Public header gate passed." From f9d020bd84cdcd76a1b28ea8ab704abc3a65eb9b Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 3 Aug 2026 12:34:13 -0500 Subject: [PATCH 27/30] Preserve sub-millisecond event timestamp precision Use precise wall-clock time where available and retain nanosecond-derived 100 ns ticks on POSIX so record.time no longer truncates every event to milliseconds. Add regression coverage for POSIX timestamp precision.\n\nCo-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>\nCopilot-Session: 05d1030e-75b0-447f-9856-65091d59a97f --- lib/pal/PAL.cpp | 25 ++++++++++++++++++++----- tests/unittests/PalTests.cpp | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/pal/PAL.cpp b/lib/pal/PAL.cpp index 3e667653f..a5291b0e3 100644 --- a/lib/pal/PAL.cpp +++ b/lib/pal/PAL.cpp @@ -430,7 +430,23 @@ namespace PAL_NS_BEGIN { { #ifdef _WIN32 FILETIME tocks; - ::GetSystemTimeAsFileTime(&tocks); + // Resolve the precise API dynamically so the SDK retains its Windows 7 + // runtime compatibility and falls back when the API is unavailable. + using GetSystemTimePreciseAsFileTimeProc = VOID (WINAPI*)(LPFILETIME); + HMODULE kernel32 = ::GetModuleHandleW(L"kernel32.dll"); + auto getSystemTimePreciseAsFileTime = + kernel32 + ? reinterpret_cast( + ::GetProcAddress(kernel32, "GetSystemTimePreciseAsFileTime")) + : nullptr; + if (getSystemTimePreciseAsFileTime) + { + getSystemTimePreciseAsFileTime(&tocks); + } + else + { + ::GetSystemTimeAsFileTime(&tocks); + } ULONGLONG ticks = (ULONGLONG(tocks.dwHighDateTime) << 32) | tocks.dwLowDateTime; // number of days from beginning to 1601 multiplied by ticks per day return ticks + 0x701ce1722770000ULL; @@ -440,10 +456,9 @@ namespace PAL_NS_BEGIN { // This UTC epoch contract has been signed in blood since C++20 std::chrono::time_point now = std::chrono::system_clock::now(); auto duration = now.time_since_epoch(); - auto millis = std::chrono::duration_cast(duration).count(); - uint64_t ticks = millis; - ticks *= 10000; // convert millis to ticks (1 tick = 100ns) - ticks += 0x89F7FF5F7B58000ULL; // UTC time 0 in .NET ticks + auto nanos = std::chrono::duration_cast(duration).count(); + int64_t ticks = nanos / 100; // convert nanoseconds to .NET ticks (1 tick = 100ns) + ticks += static_cast(0x89F7FF5F7B58000ULL); // UTC time 0 in .NET ticks return ticks; #endif } diff --git a/tests/unittests/PalTests.cpp b/tests/unittests/PalTests.cpp index ddf1f6dd2..c931ff376 100644 --- a/tests/unittests/PalTests.cpp +++ b/tests/unittests/PalTests.cpp @@ -122,6 +122,22 @@ TEST_F(PalTests, SystemTime) EXPECT_THAT(t1, Lt(t0 + 1000)); } +#if !defined(_WIN32) && !defined(_WIN64) +TEST_F(PalTests, SystemTimeInTicksPreservesSubMillisecondPrecision) +{ + constexpr int64_t TicksPerMillisecond = 10000; + bool observedSubMillisecondTick = false; + + for (int i = 0; i < 1000 && !observedSubMillisecondTick; ++i) + { + observedSubMillisecondTick = + PAL::getUtcSystemTimeinTicks() % TicksPerMillisecond != 0; + } + + EXPECT_TRUE(observedSubMillisecondTick); +} +#endif + TEST_F(PalTests, FormatUtcTimestampMsAsISO8601) { EXPECT_THAT(PAL::formatUtcTimestampMsAsISO8601(0ll), Eq("1970-01-01T00:00:00.000Z")); From 6a2e9ffe4417ea1c248f7ecef32e8f56c29cd1cc Mon Sep 17 00:00:00 2001 From: bmehta001 Date: Mon, 3 Aug 2026 14:05:40 -0500 Subject: [PATCH 28/30] Stabilize timing-sensitive tests (#1513) * Stabilize timing-sensitive tests Use a monotonic injectable clock for kill-switch deadlines, replace the sleep-heavy expiration functional test with deterministic unit coverage, and simulate expired SQLite leases directly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 77649dd9-3b60-4ca4-861d-8c5ac165d53c * Restore temporary kill-switch integration coverage Rewrite killIsTemporary to observe active drops and eventual server delivery instead of sleeping for a fixed expiration window. Keep every wait bounded without adding test-only access to production internals. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 77649dd9-3b60-4ca4-861d-8c5ac165d53c * Fix Windows CI and harden injected clocks Rename the temporary kill-switch logger so MSVC /WX no longer promotes C4458 into C2220 in both Windows pipelines. Files changed: - lib/offline/KillSwitchManager.hpp: fall back from an empty Clock and invoke injected callbacks outside the mutex. - tests/unittests/KillSwitchManagerTests.cpp: cover the empty-clock fallback. - tests/functests/BasicFuncTests.cpp: avoid shadowing the fixture logger member. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7fe5faca-d77c-45c4-85d3-0d4a00d68a94 * Harden temporary kill-switch polling Decode only newly arrived requests after releasing the HTTP callback mutex, avoiding repeated parsing and preventing the polling helper from delaying incoming requests. Treat kill-switch activation as a fatal prerequisite while preserving teardown on failure. Files changed: - tests/functests/BasicFuncTests.cpp: snapshot new requests outside the decode path and fail fast when activation is not observed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7fe5faca-d77c-45c4-85d3-0d4a00d68a94 * Make bad-network teardown test deterministic Replace external endpoints with an injected HTTP client that holds requests until teardown cancellation, then reports NetworkFailure through the required exactly-once callback. This preserves the real cancellation and callback-drain path without simulator or network timing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 77649dd9-3b60-4ca4-861d-8c5ac165d53c * tests: use SentCount() in WaitForRequest instead of m_sent.load() directly WaitForRequest polled m_sent.load() directly while SentCount() was already the named accessor for the same value. Using SentCount() keeps the implementation consistent with the class's own public API and means any future change to the accessor (e.g. different memory order) is automatically picked up. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7fe5faca-d77c-45c4-85d3-0d4a00d68a94 * tests: clean up kill-switch test and reduce lease TTL in offline storage test BasicFuncTests/killIsTemporary: flatten acceptedAfterKillExpires polling loop. - Remove redundant pre-loop waitForEvent (nothing sent yet at that point, so it always returned false). - Remove redundant post-loop grace-period block; absorb the 100 ms into expiryDeadline so the single loop covers both the poll and the grace. OfflineStorageTests_SQLite/ReservedRecordsAreReleasedAfterTimeout: - Reduce lease TTL from 60000 ms to 5000 ms. The value is the storage reservation duration, not a wall-clock wait (the test fast-forwards expiry via SQL). 5 s is clearer to readers and equally correct. KillSwitchManager::expiryFromNow: add precondition comment documenting that seconds > 0 is required and why all callers must guard it. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7fe5faca-d77c-45c4-85d3-0d4a00d68a94 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 77649dd9-3b60-4ca4-861d-8c5ac165d53c Copilot-Session: 7fe5faca-d77c-45c4-85d3-0d4a00d68a94 --- lib/offline/KillSwitchManager.hpp | 53 +++++-- tests/functests/APITest.cpp | 149 ++++++++++++++---- tests/functests/BasicFuncTests.cpp | 147 ++++++++++------- tests/unittests/KillSwitchManagerTests.cpp | 47 ++++++ .../unittests/OfflineStorageTests_SQLite.cpp | 21 ++- 5 files changed, 309 insertions(+), 108 deletions(-) diff --git a/lib/offline/KillSwitchManager.hpp b/lib/offline/KillSwitchManager.hpp index a70569877..d5f5a1211 100644 --- a/lib/offline/KillSwitchManager.hpp +++ b/lib/offline/KillSwitchManager.hpp @@ -7,11 +7,14 @@ #include "pal/PAL.hpp" +#include +#include #include #include #include #include #include +#include #include #include @@ -21,13 +24,24 @@ namespace MAT_NS_BEGIN { class KillSwitchManager { public: + using Clock = std::function; bool isActive() { return !m_tokenTime.empty(); } - KillSwitchManager() : m_isRetryAfterActive(false), m_retryAfterExpiryTime(0) + KillSwitchManager() + : KillSwitchManager([]() { return static_cast(PAL::getMonotonicTimeMs()); }) + { + } + + explicit KillSwitchManager(Clock clock) + : m_clock(clock + ? std::move(clock) + : Clock([]() { return static_cast(PAL::getMonotonicTimeMs()); })), + m_isRetryAfterActive(false), + m_retryAfterExpiryTime(0) { } @@ -45,8 +59,9 @@ namespace MAT_NS_BEGIN { int64_t timeinSecs = 0; if (tryParseSeconds(timeStr, timeinSecs) && timeinSecs > 0) { + const int64_t expiryTime = expiryFromNow(timeinSecs); std::lock_guard guard(m_lock); - m_retryAfterExpiryTime = PAL::getUtcSystemTime() + timeinSecs; + m_retryAfterExpiryTime = expiryTime; m_isRetryAfterActive = true; } } @@ -101,20 +116,22 @@ namespace MAT_NS_BEGIN { void addToken(const std::string& tokenId, int64_t timeInSeconds) { - std::lock_guard guard(m_lock); if (timeInSeconds > 0) { - m_tokenTime[tokenId] = PAL::getUtcSystemTime() + timeInSeconds; //convert milisec to sec + const int64_t expiryTime = expiryFromNow(timeInSeconds); + std::lock_guard guard(m_lock); + m_tokenTime[tokenId] = expiryTime; } } bool isTokenBlocked(const std::string& tokenId) { + const int64_t now = m_clock(); std::lock_guard guard(m_lock); if (m_isRetryAfterActive) { - if (m_retryAfterExpiryTime > PAL::getUtcSystemTime()) + if (m_retryAfterExpiryTime > now) { return true;//always return true for all tokens } @@ -129,7 +146,7 @@ namespace MAT_NS_BEGIN { {//found, check the time stamp int64_t timeStamp = m_tokenTime[tokenId]; - if (timeStamp > PAL::getUtcSystemTime()) //convert milisec to sec + if (timeStamp > now) { return true; } @@ -169,6 +186,24 @@ namespace MAT_NS_BEGIN { } private: + // Precondition: seconds > 0. All call sites enforce this (handleResponse + // and addToken both guard with `timeinSecs > 0` / `timeInSeconds > 0`). + // Passing a non-positive value is UB: a negative durationMs makes the + // overflow check `now > maxTime - durationMs` wrap (signed overflow), so + // the result is unpredictable — do not relax the call-site guards. + int64_t expiryFromNow(int64_t seconds) const + { + constexpr int64_t millisecondsPerSecond = 1000; + constexpr int64_t maxTime = std::numeric_limits::max(); + const int64_t now = m_clock(); + if (seconds > maxTime / millisecondsPerSecond) + { + return maxTime; + } + const int64_t durationMs = seconds * millisecondsPerSecond; + return now > maxTime - durationMs ? maxTime : now + durationMs; + } + // Parse a count of seconds from a response-header value (Retry-After / // kill-duration). Returns false when the value is malformed or out of // range instead of letting std::stoll throw: the worker thread that drives @@ -225,8 +260,8 @@ namespace MAT_NS_BEGIN { // Either way the std::exception catch below ignores the value rather // than crashing. const long long parsed = std::stoll(value.substr(begin, end - begin)); - // Clamp to a value that cannot overflow when later added to a current - // UTC timestamp (seconds) to compute an expiry time. No legitimate + // Clamp to a value that cannot overflow when later converted to + // milliseconds to compute an expiry time. No legitimate // Retry-After / kill-duration approaches this; an absurd value is // capped instead of wrapping the expiry into the past. const int64_t kMaxSeconds = 100LL * 365 * 24 * 60 * 60; // ~100 years @@ -272,6 +307,7 @@ namespace MAT_NS_BEGIN { return true; } + Clock m_clock; std::map m_tokenTime; std::mutex m_lock; bool m_isRetryAfterActive; @@ -280,4 +316,3 @@ namespace MAT_NS_BEGIN { } MAT_NS_END #endif - diff --git a/tests/functests/APITest.cpp b/tests/functests/APITest.cpp index 0347807f6..baea0112e 100644 --- a/tests/functests/APITest.cpp +++ b/tests/functests/APITest.cpp @@ -210,6 +210,98 @@ class TestDebugEventListener : public DebugEventListener { } }; +// Keep requests in flight until teardown cancels them, then simulate a connection +// reset while honoring IHttpClient's exactly-once callback contract. +class NetworkFailureHttpClient final : public IHttpClient +{ +public: + IHttpRequest* CreateRequest() override + { + return new SimpleHttpRequest("bad-network-" + std::to_string(m_nextRequestId.fetch_add(1))); + } + + void SendRequestAsync(IHttpRequest* request, IHttpResponseCallback* callback) override + { + std::lock_guard lock(m_mutex); + m_pending[request->GetId()] = callback; + m_sent.fetch_add(1); + } + + void CancelRequestAsync(const std::string& id) override + { + IHttpResponseCallback* callback = nullptr; + { + std::lock_guard lock(m_mutex); + auto it = m_pending.find(id); + if (it != m_pending.end()) + { + callback = it->second; + m_pending.erase(it); + } + } + if (callback != nullptr) + { + m_cancelled.fetch_add(1); + CompleteWithNetworkFailure(id, callback); + } + } + + void CancelAllRequests() override + { + std::map pending; + { + std::lock_guard lock(m_mutex); + pending.swap(m_pending); + } + m_cancelled.fetch_add(static_cast(pending.size())); + for (const auto& request : pending) + { + CompleteWithNetworkFailure(request.first, request.second); + } + } + + bool WaitForRequest(unsigned timeoutMs) const + { + const auto deadline = PAL::getMonotonicTimeMs() + timeoutMs; + while (SentCount() == 0 && PAL::getMonotonicTimeMs() < deadline) + { + PAL::sleep(10); + } + return SentCount() > 0; + } + + unsigned SentCount() const + { + return m_sent.load(); + } + + unsigned CancelledCount() const + { + return m_cancelled.load(); + } + + unsigned CompletedCount() const + { + return m_completed.load(); + } + +private: + void CompleteWithNetworkFailure(const std::string& id, IHttpResponseCallback* callback) + { + auto response = new SimpleHttpResponse("failure-" + id); + response->m_result = HttpResult_NetworkFailure; + callback->OnHttpResponse(response); + m_completed.fetch_add(1); + } + + mutable std::mutex m_mutex; + std::map m_pending; + std::atomic m_nextRequestId{0}; + std::atomic m_sent{0}; + std::atomic m_cancelled{0}; + std::atomic m_completed{0}; +}; + /// /// Add all event listeners /// @@ -1204,41 +1296,43 @@ TEST(APITest, LogConfiguration_MsRoot_Check) TEST(APITest, LogManager_BadNetwork_Test) { auto& config = LogManager::GetLogConfiguration(); - - // Clean temp file first const char *cacheFilePath = "bad-network.db"; std::string fileName = MAT::GetTempDirectory(); fileName += cacheFilePath; - printf("remove %s\n", fileName.c_str()); std::remove(fileName.c_str()); std::remove((fileName + "-wal").c_str()); std::remove((fileName + "-shm").c_str()); std::remove((fileName + "-journal").c_str()); - for (auto url : { -#if 0 /* [MG}: Temporary change to avoid GitHub Actions crash #92 */ - "https://0.0.0.0/", - "https://127.0.0.1/", -#endif - "https://mobile.events-sandbox.data.microsoft.com/OneCollector/1.0/", - "https://invalid.host.name.microsoft.com/" - }) - { - printf("--- trying %s", url); - config[CFG_STR_CACHE_FILE_PATH] = cacheFilePath; - config[CFG_INT_TRACE_LEVEL_MASK] = 0; - config[CFG_INT_TRACE_LEVEL_MIN] = ACTTraceLevel_Warn; - config[CFG_INT_SDK_MODE] = SdkModeTypes::SdkModeTypes_CS; - config[CFG_INT_MAX_TEARDOWN_TIME] = 0; - config[CFG_STR_COLLECTOR_URL] = url; - size_t numIterations = 5; - while (numIterations--) - { - printf("."); - EXPECT_GE(StressSingleThreaded(config), MAX_ITERATIONS); - } - printf("\n"); - } + auto httpClient = std::make_shared(); + config.AddModule(CFG_MODULE_HTTP_CLIENT, httpClient); + config[CFG_STR_CACHE_FILE_PATH] = cacheFilePath; + config[CFG_INT_TRACE_LEVEL_MASK] = 0; + config[CFG_INT_TRACE_LEVEL_MIN] = ACTTraceLevel_Warn; + config[CFG_INT_SDK_MODE] = SdkModeTypes::SdkModeTypes_CS; + config[CFG_INT_MAX_TEARDOWN_TIME] = 0; + config[CFG_STR_COLLECTOR_URL] = "https://unused.invalid/"; + + TestDebugEventListener debugListener; + addAllListeners(debugListener); + LogManager::AddEventListener(DebugEventType::EVT_HTTP_FAILURE, debugListener); + auto logger = LogManager::Initialize(TEST_TOKEN, config); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); + logger->LogEvent("badNetworkEvent"); + LogManager::UploadNow(); + + const bool requestStarted = httpClient->WaitForRequest(10000); + LogManager::FlushAndTeardown(); + LogManager::RemoveEventListener(DebugEventType::EVT_HTTP_FAILURE, debugListener); + removeAllListeners(debugListener); + config.AddModule(CFG_MODULE_HTTP_CLIENT, nullptr); + + EXPECT_TRUE(requestStarted); + EXPECT_GE(debugListener.numLogged.load(), 1u); + EXPECT_GE(debugListener.numHttpError.load(), 1u); + EXPECT_GE(httpClient->SentCount(), 1u); + EXPECT_EQ(httpClient->SentCount(), httpClient->CancelledCount()); + EXPECT_EQ(httpClient->CancelledCount(), httpClient->CompletedCount()); } TEST(APITest, LogManager_GetLoggerSameLoggerMultithreaded) @@ -1485,4 +1579,3 @@ TEST(APITest, Custom_Decorator) #endif // HAVE_MAT_DEFAULT_HTTP_CLIENT // TEST_PULL_ME_IN(APITest) - diff --git a/tests/functests/BasicFuncTests.cpp b/tests/functests/BasicFuncTests.cpp index 438411425..bc879d3e6 100644 --- a/tests/functests/BasicFuncTests.cpp +++ b/tests/functests/BasicFuncTests.cpp @@ -541,6 +541,35 @@ class BasicFuncTests : public ::testing::Test, } return result; } + + bool waitForEvent(const std::string& name, unsigned timeoutMs, size_t& nextRequestIndex) + { + const auto deadline = PAL::getMonotonicTimeMs() + timeoutMs; + while (PAL::getMonotonicTimeMs() < deadline) + { + std::vector newRequests; + { + LOCKGUARD(mtx_requests); + while (nextRequestIndex < receivedRequests.size()) + { + newRequests.push_back(receivedRequests[nextRequestIndex]); + ++nextRequestIndex; + } + } + for (const auto& request : newRequests) + { + for (const auto& record : decodeRequest(request, false)) + { + if (record.name == name) + { + return true; + } + } + } + PAL::sleep(10); + } + return false; + } }; @@ -1110,6 +1139,17 @@ public : break; }; } + + bool waitForAtLeast(const std::atomic& counter, unsigned expected, unsigned timeoutMs) + { + const auto deadline = PAL::getMonotonicTimeMs() + timeoutMs; + while (counter.load() < expected && PAL::getMonotonicTimeMs() < deadline) + { + PAL::sleep(10); + } + return counter.load() >= expected; + } + void printStats(){ std::cerr << "[ ] numLogged = " << numLogged << std::endl; std::cerr << "[ ] numSent = " << numSent << std::endl; @@ -1231,84 +1271,71 @@ TEST_F(BasicFuncTests, killSwitchWorks) TEST_F(BasicFuncTests, killIsTemporary) { CleanStorage(); - // Create the configuration to send to fake server auto configuration = LogManager::GetLogConfiguration(); configuration[CFG_INT_TRACE_LEVEL_MASK] = 0xFFFFFFFF; configuration[CFG_INT_TRACE_LEVEL_MIN] = ACTTraceLevel_Warn; configuration[CFG_INT_SDK_MODE] = SdkModeTypes::SdkModeTypes_CS; - configuration[CFG_INT_RAM_QUEUE_SIZE] = 4096 * 20; configuration[CFG_STR_CACHE_FILE_PATH] = TEST_STORAGE_FILENAME; - configuration[CFG_INT_MAX_TEARDOWN_TIME] = 2; // 2 seconds wait on shutdown + configuration[CFG_INT_MAX_TEARDOWN_TIME] = 2; configuration[CFG_STR_COLLECTOR_URL] = serverAddress.c_str(); - configuration[CFG_MAP_HTTP][CFG_BOOL_HTTP_COMPRESSION] = false; // disable compression for now - configuration[CFG_MAP_METASTATS_CONFIG]["interval"] = 30 * 60; // 30 mins - configuration[CFG_MAP_METASTATS_CONFIG]["enabled"] = true; // opt in to stats (disabled by default since #1420) - + configuration[CFG_MAP_HTTP][CFG_BOOL_HTTP_COMPRESSION] = false; + configuration[CFG_MAP_METASTATS_CONFIG]["interval"] = 30 * 60; + configuration[CFG_MAP_METASTATS_CONFIG]["enabled"] = true; configuration["name"] = __FILE__; configuration["version"] = "1.0.0"; - configuration["config"] = { { "host", __FILE__ } }; // Host instance + configuration["config"] = { { "host", __FILE__ } }; - // set the killed token on the server - server.setKilledToken(KILLED_TOKEN, 10); + constexpr unsigned killDurationSec = 5; + server.setKilledToken(KILLED_TOKEN, killDurationSec); KillSwitchListener listener; addListeners(listener); - // Log 100 events from valid and invalid 4 times - int repetitions = 4; - for (int i = 0; i < repetitions; i++) { - // Initialize the logger for the valid token and log 100 events - LogManager::Initialize(TEST_TOKEN, configuration); - LogManager::ResumeTransmission(); - auto myLogger = LogManager::GetLogger(TEST_TOKEN, "killed"); - int numIterations = 100; - while (numIterations--) { - EventProperties event1("fooEvent"); - event1.SetProperty("property", "value"); - myLogger->LogEvent(event1); - } - // Initialize the logger for the killed token and log 100 events - LogManager::Initialize(KILLED_TOKEN, configuration); - LogManager::ResumeTransmission(); - myLogger = LogManager::GetLogger(KILLED_TOKEN, "killed"); - numIterations = 100; - while (numIterations--) { - EventProperties event2("failEvent"); - event2.SetProperty("property", "value"); - myLogger->LogEvent(event2); - } - } - // Try and wait to upload - LogManager::UploadNow(); - PAL::sleep(2000); - // Sleep for 11 seconds so the killed time has expired, clear the killed tokens on server - PAL::sleep(11000); - server.clearKilledTokens(); - // Log 100 events with valid logger - LogManager::Initialize(TEST_TOKEN, configuration); - LogManager::ResumeTransmission(); - auto myLogger = LogManager::GetLogger(TEST_TOKEN, "killed"); - int numIterations = 100; - while (numIterations--) { - EventProperties event1("fooEvent"); - event1.SetProperty("property", "value"); - myLogger->LogEvent(event1); - } LogManager::Initialize(KILLED_TOKEN, configuration); + LogManager::SetTransmitProfile(TransmitProfile_RealTime); LogManager::ResumeTransmission(); - myLogger = LogManager::GetLogger(KILLED_TOKEN, "killed"); - numIterations = 100; - while (numIterations--) { - EventProperties event2("failEvent"); - event2.SetProperty("property", "value"); - myLogger->LogEvent(event2); + + auto killedLogger = LogManager::GetLogger(KILLED_TOKEN, "killed"); + killedLogger->LogEvent("activateKillSwitch"); + LogManager::UploadNow(); + + const bool killSwitchActivated = listener.waitForAtLeast(listener.numHttpOK, 1, 10000); + if (!killSwitchActivated) + { + LogManager::FlushAndTeardown(); + removeListeners(listener); + server.clearKilledTokens(); } - // Expect to 0 events to be dropped - EXPECT_EQ(uint32_t { 0 }, listener.numDropped); - LogManager::FlushAndTeardown(); + ASSERT_TRUE(killSwitchActivated) << "Kill-switch response was not observed before timeout"; + server.clearKilledTokens(); - listener.printStats(); + const unsigned droppedBeforeKill = listener.numDropped.load(); + const auto activeDeadline = PAL::getMonotonicTimeMs() + 2000; + unsigned probe = 0; + while (listener.numDropped.load() == droppedBeforeKill + && PAL::getMonotonicTimeMs() < activeDeadline) + { + killedLogger->LogEvent("blockedWhileKillIsActive" + std::to_string(probe++)); + PAL::sleep(20); + } + EXPECT_GT(listener.numDropped.load(), droppedBeforeKill); + + // Poll until the kill-switch TTL expires and the SDK resumes sending. + // Budget: kill duration + 5 s headroom; the extra 100 ms absorbs any + // request that was dispatched just before the deadline fires. + const auto expiryDeadline = PAL::getMonotonicTimeMs() + (killDurationSec + 5) * 1000 + 100; + size_t nextRequestIndex = 0; + bool acceptedAfterKillExpires = false; + while (!acceptedAfterKillExpires && PAL::getMonotonicTimeMs() < expiryDeadline) + { + killedLogger->LogEvent("acceptedAfterKillExpires"); + LogManager::UploadNow(); + acceptedAfterKillExpires = waitForEvent("acceptedAfterKillExpires", 100, nextRequestIndex); + } + EXPECT_TRUE(acceptedAfterKillExpires); + + LogManager::FlushAndTeardown(); removeListeners(listener); server.clearKilledTokens(); } diff --git a/tests/unittests/KillSwitchManagerTests.cpp b/tests/unittests/KillSwitchManagerTests.cpp index 15aaaee18..ceec1f450 100644 --- a/tests/unittests/KillSwitchManagerTests.cpp +++ b/tests/unittests/KillSwitchManagerTests.cpp @@ -16,6 +16,34 @@ TEST(KillSwitchManagerTests, handleResponse_ValidRetryAfter_ActivatesRetryAfter) ASSERT_TRUE(manager.isRetryAfterActive()); } +TEST(KillSwitchManagerTests, constructor_EmptyClockUsesMonotonicClock) +{ + KillSwitchManager manager(KillSwitchManager::Clock{}); + HttpHeaders headers; + headers.add("Retry-After", "120"); + + ASSERT_NO_THROW(manager.handleResponse(headers)); + EXPECT_TRUE(manager.isTokenBlocked("any-token")); +} + +TEST(KillSwitchManagerTests, handleResponse_RetryAfterExpiresAtDeadline) +{ + int64_t nowMs = 1000; + KillSwitchManager manager([&nowMs]() { return nowMs; }); + HttpHeaders headers; + headers.add("Retry-After", "120"); + + manager.handleResponse(headers); + ASSERT_TRUE(manager.isTokenBlocked("any-token")); + + nowMs += 119999; + EXPECT_TRUE(manager.isTokenBlocked("any-token")); + + nowMs += 1; + EXPECT_FALSE(manager.isTokenBlocked("any-token")); + EXPECT_FALSE(manager.isRetryAfterActive()); +} + TEST(KillSwitchManagerTests, handleResponse_NonNumericRetryAfter_DoesNotThrowAndIsIgnored) { KillSwitchManager manager; @@ -120,6 +148,25 @@ TEST(KillSwitchManagerTests, handleResponse_ValidKillTokenAndDuration_BlocksToke ASSERT_TRUE(manager.isTokenBlocked("tenant-token-1")); } +TEST(KillSwitchManagerTests, handleResponse_KillDurationExpiresAtDeadline) +{ + int64_t nowMs = 1000; + KillSwitchManager manager([&nowMs]() { return nowMs; }); + HttpHeaders headers; + headers.add("kill-tokens", "tenant-token-1"); + headers.add("kill-duration", "10"); + + ASSERT_TRUE(manager.handleResponse(headers)); + ASSERT_TRUE(manager.isTokenBlocked("tenant-token-1")); + + nowMs += 9999; + EXPECT_TRUE(manager.isTokenBlocked("tenant-token-1")); + + nowMs += 1; + EXPECT_FALSE(manager.isTokenBlocked("tenant-token-1")); + EXPECT_FALSE(manager.isActive()); +} + TEST(KillSwitchManagerTests, handleResponse_NonNumericKillDuration_DoesNotThrowAndDoesNotBlock) { KillSwitchManager manager; diff --git a/tests/unittests/OfflineStorageTests_SQLite.cpp b/tests/unittests/OfflineStorageTests_SQLite.cpp index d5aa6808a..015e197d7 100644 --- a/tests/unittests/OfflineStorageTests_SQLite.cpp +++ b/tests/unittests/OfflineStorageTests_SQLite.cpp @@ -312,32 +312,31 @@ TEST_F(OfflineStorageTests_SQLite, ReservedRecordsAreReleasedAfterTimeout) ASSERT_THAT(offlineStorage->StoreRecord({"guid1", "token", EventLatency_Normal, EventPersistence_Normal, 1, {}}), true); ASSERT_THAT(offlineStorage->StoreRecord({"guid2", "token", EventLatency_Normal, EventPersistence_Normal, 1, {}}), true); TestRecordConsumer consumer; - // Reserve first for 2 secs - EXPECT_THAT(offlineStorage->GetAndReserveRecords(consumer, 2000, EventLatency_Unspecified, 1), true); + EXPECT_THAT(offlineStorage->GetAndReserveRecords(consumer, 5000, EventLatency_Unspecified, 1), true); ASSERT_THAT(consumer.records.size(), 1); consumer.records.clear(); - PAL::sleep(500); - - // Reserve second for 1 sec, first still unavailable - EXPECT_THAT(offlineStorage->GetAndReserveRecords(consumer, 1000, EventLatency_Unspecified, 1), true); + // The first record remains reserved, so the second call returns the other record. + EXPECT_THAT(offlineStorage->GetAndReserveRecords(consumer, 5000, EventLatency_Unspecified, 1), true); ASSERT_THAT(consumer.records.size(), 1); consumer.records.clear(); auto records = offlineStorage->GetRecords(true, EventLatency_Unspecified, 0); ASSERT_THAT(records.size(), 2); - int64_t waitUntilMs = 0; for (auto const& record : records) { - waitUntilMs = std::max(waitUntilMs, record.reservedUntil); + EXPECT_GT(record.reservedUntil, 1); } - while (PAL::getUtcSystemTimeMs() <= waitUntilMs + 250) + // Simulate lease expiry without depending on wall-clock sleeps or CI scheduling. + offlineStorage->Execute("UPDATE events SET reserved_until=1"); + records = offlineStorage->GetRecords(true, EventLatency_Unspecified, 0); + ASSERT_THAT(records.size(), 2); + for (auto const& record : records) { - PAL::sleep(50); + EXPECT_EQ(record.reservedUntil, 1); } - // Both records are timed out EXPECT_THAT(offlineStorage->GetAndReserveRecords(consumer, 1000), true); ASSERT_THAT(consumer.records.size(), 2); EXPECT_THAT(consumer.records[0].retryCount, 1); From babeb981262cb88bdf74b036c0da95489c477450 Mon Sep 17 00:00:00 2001 From: Bhagirath Mehta Date: Mon, 3 Aug 2026 15:31:27 -0500 Subject: [PATCH 29/30] Cache precise Windows clock lookup Resolve GetSystemTimePreciseAsFileTime once instead of repeating module and symbol lookups for every event timestamp.\n\nCo-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>\nCopilot-Session: 05d1030e-75b0-447f-9856-65091d59a97f --- lib/pal/PAL.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/pal/PAL.cpp b/lib/pal/PAL.cpp index a5291b0e3..0fc28abfb 100644 --- a/lib/pal/PAL.cpp +++ b/lib/pal/PAL.cpp @@ -433,12 +433,15 @@ namespace PAL_NS_BEGIN { // Resolve the precise API dynamically so the SDK retains its Windows 7 // runtime compatibility and falls back when the API is unavailable. using GetSystemTimePreciseAsFileTimeProc = VOID (WINAPI*)(LPFILETIME); - HMODULE kernel32 = ::GetModuleHandleW(L"kernel32.dll"); - auto getSystemTimePreciseAsFileTime = - kernel32 - ? reinterpret_cast( - ::GetProcAddress(kernel32, "GetSystemTimePreciseAsFileTime")) - : nullptr; + static const GetSystemTimePreciseAsFileTimeProc getSystemTimePreciseAsFileTime = + []() -> GetSystemTimePreciseAsFileTimeProc + { + HMODULE kernel32 = ::GetModuleHandleW(L"kernel32.dll"); + return kernel32 + ? reinterpret_cast( + ::GetProcAddress(kernel32, "GetSystemTimePreciseAsFileTime")) + : nullptr; + }(); if (getSystemTimePreciseAsFileTime) { getSystemTimePreciseAsFileTime(&tocks); From 3a3a83b4927b6e71839e82f5539e5528b0daf44d Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Security Bot Date: Tue, 4 Aug 2026 13:02:39 -0700 Subject: [PATCH 30/30] Pin GitHub Actions to full-length commit SHAs (#1517) --- .github/dependabot.yml | 11 +++++++++++ .github/workflows/build-android.yml | 8 ++++---- .github/workflows/build-ios-mac.yml | 2 +- .github/workflows/build-posix-latest.yml | 4 ++-- .github/workflows/build-ubuntu-2204.yml | 2 +- .github/workflows/build-windows-vs2022.yaml | 2 +- .github/workflows/codeql-analysis.yml | 16 ++++++++-------- .github/workflows/deploy-docs-pages.yml | 10 +++++----- .github/workflows/spellcheck.yml | 2 +- .github/workflows/test-vcpkg.yml | 10 +++++----- .github/workflows/test-win-latest.yml | 6 +++--- 11 files changed, 42 insertions(+), 31 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..2c48305b7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + groups: + github-actions: + patterns: ["*"] + schedule: + interval: "weekly" + cooldown: + default-days: 7 diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 1235e8dc7..1ce8a3aa9 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -35,7 +35,7 @@ jobs: name: Build for Android steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: submodules: false - name: Update submodules @@ -44,7 +44,7 @@ jobs: git config --global submodule.lib/modules.update none git -c protocol.version=2 submodule update --init --force --depth=1 - name: Setup Java - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: 'adopt' java-version: '17' @@ -52,7 +52,7 @@ jobs: # Workaround for: 'Unable to decrypt local Maven settings credentials' run: rm $Env:USERPROFILE\.m2\settings.xml - name: Setup Android SDK - uses: android-actions/setup-android@v3 + uses: android-actions/setup-android@9fc6c4e9069bf8d3d10b2204b1fb8f6ef7065407 # v3.2.2 - name: Install NDK run: | java -version @@ -83,7 +83,7 @@ jobs: working-directory: lib\android_build - name: Upload Reports if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: reports path: lib\android_build\maesdk\build\reports diff --git a/.github/workflows/build-ios-mac.yml b/.github/workflows/build-ios-mac.yml index 7ca85012b..29b3dfc34 100644 --- a/.github/workflows/build-ios-mac.yml +++ b/.github/workflows/build-ios-mac.yml @@ -54,7 +54,7 @@ jobs: - name: Grant write permissions to /usr/local run: | sudo chown -R $USER:staff /usr/local - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: submodules: 'true' continue-on-error: true diff --git a/.github/workflows/build-posix-latest.yml b/.github/workflows/build-posix-latest.yml index 8f9320e57..7a35c5a54 100644 --- a/.github/workflows/build-posix-latest.yml +++ b/.github/workflows/build-posix-latest.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 continue-on-error: true - name: Test ${{ matrix.os }} ${{ matrix.config }} run: ./build-tests.sh ${{ matrix.config }} @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Install clang run: sudo apt-get update && sudo apt-get install -y clang - name: Compile each public header standalone under strict flags diff --git a/.github/workflows/build-ubuntu-2204.yml b/.github/workflows/build-ubuntu-2204.yml index 1fbcc6404..6c779c8b5 100644 --- a/.github/workflows/build-ubuntu-2204.yml +++ b/.github/workflows/build-ubuntu-2204.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 continue-on-error: true - name: Test ${{ matrix.os }} ${{ matrix.config }} run: ./build-tests.sh ${{ matrix.config }} \ No newline at end of file diff --git a/.github/workflows/build-windows-vs2022.yaml b/.github/workflows/build-windows-vs2022.yaml index 222e32e67..e109575e8 100644 --- a/.github/workflows/build-windows-vs2022.yaml +++ b/.github/workflows/build-windows-vs2022.yaml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Build env: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index db7b4870a..a1f7a9c7f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,12 +39,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 continue-on-error: true # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -75,7 +75,7 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 analyze-java: name: Analyze Java @@ -90,7 +90,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 continue-on-error: true - name: Update submodules @@ -100,19 +100,19 @@ jobs: git -c protocol.version=2 submodule update --init --force --depth=1 - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: java - name: Setup Java - uses: actions/setup-java@v5 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 with: distribution: 'adopt' java-version: '17' - name: Remove default github maven configuration run: rm $Env:USERPROFILE\.m2\settings.xml - name: Setup Android SDK - uses: android-actions/setup-android@v3 + uses: android-actions/setup-android@9fc6c4e9069bf8d3d10b2204b1fb8f6ef7065407 # v3.2.2 - name: Install NDK run: | java -version @@ -139,4 +139,4 @@ jobs: working-directory: lib\android_build - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 diff --git a/.github/workflows/deploy-docs-pages.yml b/.github/workflows/deploy-docs-pages.yml index 09ecd2d35..a3f13366f 100644 --- a/.github/workflows/deploy-docs-pages.yml +++ b/.github/workflows/deploy-docs-pages.yml @@ -33,10 +33,10 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -55,7 +55,7 @@ jobs: - name: Upload Pages artifact if: github.event_name != 'pull_request' - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 with: path: docs/public/_build/html @@ -71,8 +71,8 @@ jobs: steps: - name: Configure GitHub Pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index eeedb9c62..261ff567f 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 continue-on-error: true - name: install misspell diff --git a/.github/workflows/test-vcpkg.yml b/.github/workflows/test-vcpkg.yml index bdf37bd2e..59961ce53 100644 --- a/.github/workflows/test-vcpkg.yml +++ b/.github/workflows/test-vcpkg.yml @@ -26,7 +26,7 @@ jobs: runs-on: windows-latest name: Windows (x64-windows-static) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Bootstrap vcpkg run: | @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest name: Linux (x64-linux) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Bootstrap vcpkg run: | @@ -60,7 +60,7 @@ jobs: runs-on: macos-latest name: macOS (native) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Bootstrap vcpkg run: | @@ -78,7 +78,7 @@ jobs: runs-on: macos-latest name: iOS (arm64-ios cross-compile) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Bootstrap vcpkg run: | @@ -96,7 +96,7 @@ jobs: runs-on: ubuntu-latest name: Android (arm64-v8a API 23 cross-compile) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Bootstrap vcpkg run: | diff --git a/.github/workflows/test-win-latest.yml b/.github/workflows/test-win-latest.yml index 4928fc71f..2a77d5e2a 100644 --- a/.github/workflows/test-win-latest.yml +++ b/.github/workflows/test-win-latest.yml @@ -43,11 +43,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 continue-on-error: true - name: setup-msbuild - uses: microsoft/setup-msbuild@v2 + uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # v2.0.0 with: vs-version: '[17,)' @@ -60,7 +60,7 @@ jobs: runs-on: windows-2022 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Compile each public header standalone under /W4 /WX shell: cmd run: tests\headers\check_public_headers.cmd