Perfetto trace format

CMakeLists.txt

@@ -160,12 +160,33 @@ function(calf_set_default_log_dir target)
 endfunction()
 
 function(calf_enable_log target ENABLE)
+    if (${ARGC} GREATER 3)
+        message(FATAL_ERROR "calf_enable_log accepts target, ENABLE, and optional JSON or PROTOBUF format")
+    endif()
+
     if (ENABLE)
         target_compile_definitions(${target} PRIVATE CALF_LOG)
-        target_link_libraries(${target} PRIVATE calf_headers)
+        if(${ARGC} EQUAL 3)
+            string(TOUPPER "${ARGV2}" format)
+        elseif(CALF_PROTOBUF)
+            set(format "PROTOBUF")
+        else()
+            set(format "JSON")
+        endif()
+
+        if(format STREQUAL "PROTOBUF")
+            if(NOT TARGET calf::protobuf)
+                message(FATAL_ERROR "calf_enable_log(${target}) requested PROTOBUF but CALF_PROTOBUF is OFF")
+            endif()
+            target_link_libraries(${target} PRIVATE calf::protobuf)
+        elseif(format STREQUAL "JSON")
+            target_link_libraries(${target} PRIVATE calf_headers)
+        else()
+            message(FATAL_ERROR "calf_enable_log format must be JSON or PROTOBUF, got: ${ARGV2}")
+        endif()
     endif ()
     target_include_directories(${target} PRIVATE "${CMAKE_CURRENT_FUNCTION_LIST_DIR}")
-    message(STATUS "calf_enable_log(${target}) -> include dir: ${CMAKE_CURRENT_FUNCTION_LIST_DIR}")
+    message(STATUS "calf_enable_log(${target}) -> ${format} include dir: ${CMAKE_CURRENT_FUNCTION_LIST_DIR}")
 endfunction()
 
 function(calf_enable_warnings_as_errors target)
@@ -197,40 +218,41 @@ if(CALF_TESTS)
     include(GoogleTest)
 
     add_executable(calf_core_tests tests/core_tests.cpp)
-    calf_enable_log(calf_core_tests ON)
+    calf_enable_log(calf_core_tests ON JSON)
     calf_enable_warnings_as_errors(calf_core_tests)
     target_compile_features(calf_core_tests PRIVATE cxx_std_17)
     target_link_libraries(calf_core_tests PRIVATE GTest::gtest_main nlohmann_json::nlohmann_json)
     gtest_discover_tests(calf_core_tests)
 
     add_executable(calf_stl_tests tests/stl_tests.cpp)
-    calf_enable_log(calf_stl_tests ON)
+    calf_enable_log(calf_stl_tests ON JSON)
     calf_enable_warnings_as_errors(calf_stl_tests)
     target_compile_features(calf_stl_tests PRIVATE cxx_std_17)
     target_link_libraries(calf_stl_tests PRIVATE GTest::gtest_main nlohmann_json::nlohmann_json)
     gtest_discover_tests(calf_stl_tests)
 
     if(CALF_PROTOBUF)
         add_executable(calf_protobuf_tests tests/protobuf_tests.cpp)
+        calf_enable_log(calf_protobuf_tests ON)
         calf_enable_warnings_as_errors(calf_protobuf_tests)
         target_compile_features(calf_protobuf_tests PRIVATE cxx_std_17)
-        target_link_libraries(calf_protobuf_tests PRIVATE GTest::gtest_main calf::protobuf)
+        target_link_libraries(calf_protobuf_tests PRIVATE GTest::gtest_main)
         gtest_discover_tests(calf_protobuf_tests)
 
         if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
             add_executable(calf_syscall_protobuf_tests tests/syscall_protobuf_tests.cpp)
+            calf_enable_log(calf_syscall_protobuf_tests ON)
             calf_enable_warnings_as_errors(calf_syscall_protobuf_tests)
             target_compile_definitions(calf_syscall_protobuf_tests PRIVATE __CALF_POSIX)
             target_compile_features(calf_syscall_protobuf_tests PRIVATE cxx_std_17)
-            target_link_libraries(calf_syscall_protobuf_tests PRIVATE GTest::gtest_main
-                                                                       calf::protobuf)
+            target_link_libraries(calf_syscall_protobuf_tests PRIVATE GTest::gtest_main)
             gtest_discover_tests(calf_syscall_protobuf_tests)
         endif()
     endif()
 
     if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
         add_executable(calf_syscall_tests tests/syscall_tests.cpp)
-        calf_enable_log(calf_syscall_tests ON)
+        calf_enable_log(calf_syscall_tests ON JSON)
         calf_enable_warnings_as_errors(calf_syscall_tests)
         target_compile_definitions(calf_syscall_tests PRIVATE __CALF_POSIX)
         target_compile_features(calf_syscall_tests PRIVATE cxx_std_17)

README.md

@@ -10,7 +10,7 @@
 [![C++ tests](https://github.com/High-Performance-IO/CALF/actions/workflows/cpp-tests.yml/badge.svg?branch=main)](https://github.com/High-Performance-IO/CALF/actions/workflows/cpp-tests.yml?query=branch%3Amain)
 [![Python tests](https://github.com/High-Performance-IO/CALF/actions/workflows/python-tests.yml/badge.svg?branch=main)](https://github.com/High-Performance-IO/CALF/actions/workflows/python-tests.yml?query=branch%3Amain)
 
-Calf is a structured, header-only C++17 logging library for the [CAPIO](https://github.com/High-Performance-IO/capio) ecosystem. It supports indented JSON output and an opt-in streaming protobuf format for lower-overhead trace creation.
+Calf is a structured, header-only C++17 logging library for the [CAPIO](https://github.com/High-Performance-IO/capio) ecosystem. It supports indented JSON output and opt-in streaming Perfetto traces for lower-overhead trace creation.
 
 Calf is designed to work in two distinct environments:
 
@@ -19,12 +19,11 @@ Calf is designed to work in two distinct environments:
 - **NON STL safe processes**: uses raw syscalls whenever STL is not safe (`SyscallLogger`)
 Both backends share the same JSON structure and produce identical output formats.
 
-The C++ protobuf format works with both `StlLogger` and `SyscallLogger`. It uses
-Google's generated Protobuf C++ classes and writes `.pb` files matching
-`calf/protobuf/calf_trace.proto`.
-Each file is a valid `calf.proto.TraceFile`; flat
-scope-enter, event, and scope-exit records can be streamed and reconstructed
-through `scope_id` and `parent_scope_id`.
+The C++ Perfetto format works with both `StlLogger` and `SyscallLogger`. It
+writes standard `.perfetto-trace` protobuf streams that open directly in the
+[Perfetto UI](https://ui.perfetto.dev/). Scopes are nested TrackEvent slices,
+while `LOG` messages are instant TrackEvents with source locations and an
+`args` debug annotation.
 
 ## CALF Inspector
 
@@ -84,17 +83,23 @@ Each log scope opened by `START_LOG` becomes one JSON object in a root array. In
 ]
 ```
 
-Each thread writes to its own log file under the log directory, named after the thread ID.
+JSON output writes one file per thread. Perfetto output combines all threads in
+one process into
+`$CALF_LOG_DIR/<hostname>/<component>_<pid>.perfetto-trace`.
 
-### Protobuf output
+### Perfetto output
 
-Select protobuf by linking the application target with `calf::protobuf`. The
+Select Perfetto output by linking the application target with `calf::protobuf`. The
 existing logger include and macro interface remain unchanged:
 
 ```cmake
 target_link_libraries(my_server PRIVATE calf::protobuf)
 ```
 
+Projects using `calf_enable_log(target ON)` select Perfetto automatically when
+`CALF_PROTOBUF=ON`. Use `calf_enable_log(target ON JSON)` only for targets that
+intentionally require JSON output.
+
 ```cpp
 #include <calf/StlLogger.h>
 
@@ -104,11 +109,14 @@ void handle_request(const char *path) {
 }
 ```
 
-CMake uses an installed Google Protobuf package when available and otherwise
-downloads it from source through `FetchContent`. Without the `calf::protobuf`
-link dependency, `StlLogger.h` and `SyscallLogger.h` produce JSON. The syscall
-backend writes serialized protobuf bytes through raw syscalls; constructing the
-generated messages still uses Google's C++ runtime and its allocations.
+CMake uses an installed Google Protobuf package for its minimal Perfetto schema
+and tests, and otherwise downloads it through `FetchContent`. Without the
+`calf::protobuf` link dependency, `StlLogger.h` and `SyscallLogger.h` produce
+JSON. The syscall backend writes Perfetto protobuf bytes directly through raw
+syscalls without constructing generated messages or allocating per event.
+STL writes are synchronized across threads, while syscall writes use file locks
+so packets cannot interleave. A process truncates its trace on the first write;
+different processes always write separate files.
 
 
 ## Integration
@@ -150,7 +158,7 @@ target_link_libraries(my_interceptor PRIVATE calf::syscall)
 | `CALF_TESTS` | `OFF` | Build and register the GoogleTest suites with CTest. |
 | `CALF_PYTHON_TESTS` | Value of `CALF_TESTS` | Build the Python extension and register its binding tests. |
 | `CALF_BUILD_PYTHON_BINDINGS` | `OFF` | Build and install the private `calf._py_calf` Python extension. |
-| `CALF_PROTOBUF` | `ON` | Build the Google Protobuf C++ logger and `calf::protobuf` target. |
+| `CALF_PROTOBUF` | `ON` | Build the Perfetto trace logger and `calf::protobuf` target. |
 | `CALF_DEFAULT_COMPONENT_NAME` | `calf` | Component directory and CLI header name used by configured targets. |
 | `CALF_DEFAULT_LOG_DIR_NAME` | `./calf_logs` | Default log root when `CALF_LOG_DIR` is unset. |
 
@@ -333,6 +341,8 @@ void internal_op() {
 | Variable             | Description                                                   | Default          |
 |----------------------|---------------------------------------------------------------|------------------|
 | `CALF_LOG_DIR`    | Root directory for all log output.                            | `./calf_logs` |
-| `CALF_LOG_PREFIX` | Filename prefix for per-thread log files.                     | Backend-specific |
+| `CALF_LOG_PREFIX` | Filename prefix for per-thread JSON files; ignored by Perfetto output. | Backend-specific |
 
-Log files are written to `$CALF_LOG_DIR/<backend>/<hostname>/<prefix><tid>.log`.
+JSON files are written to
+`$CALF_LOG_DIR/<component>/<hostname>/<prefix><tid>.log`. Perfetto files are
+written to `$CALF_LOG_DIR/<hostname>/<component>_<pid>.perfetto-trace`.

calf/StlLogger.h

@@ -7,25 +7,69 @@
 #include <fstream>
 #include <climits>
 #include <memory>
+#include <mutex>
 #include <string>
 #include <unistd.h>
 
 #include "format/BaseLogger.h"
 #include "format/LogFormat.h"
 #include "utils//ThreadId.h"
 
-struct StlLogger : CalfLogBase<StlLogger> {
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+#define CALF_STL_LOGGER_TYPE PerfettoStlLogger
+#else
+#define CALF_STL_LOGGER_TYPE JsonStlLogger
+#endif
+
+struct CALF_STL_LOGGER_TYPE : CalfLogBase<CALF_STL_LOGGER_TYPE> {
 
-    inline static thread_local std::unique_ptr<std::fstream> logfile    = nullptr;
+    inline static thread_local std::unique_ptr<std::fstream> logfile     = nullptr;
     inline static thread_local std::unique_ptr<std::string> logFileName = nullptr;
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+    inline static std::unique_ptr<std::fstream> perfettoLogfile     = nullptr;
+    inline static std::unique_ptr<std::string> perfettoLogFileName = nullptr;
+    inline static std::recursive_mutex perfettoLogfileMutex;
+    inline static pid_t logfilePid = 0;
+#endif
+
+    explicit CALF_STL_LOGGER_TYPE() { ensureFileOpen(); }
+
+    static unsigned long currentTimestamp() {
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        timespec now{};
+        ::clock_gettime(CLOCK_MONOTONIC, &now);
+        return static_cast<unsigned long>(now.tv_sec) * 1000000000UL +
+               static_cast<unsigned long>(now.tv_nsec);
+#else
+        return static_cast<unsigned long>(current_time_in_millis());
+#endif
+    }
+
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+    static pid_t processId() { return ::getpid(); }
 
-    explicit StlLogger() { ensureFileOpen(); }
+    static long threadId() { return calf_current_tid(); }
+#endif
 
-    static std::string getLogFileName() { return logFileName ? *logFileName : std::string{}; }
+    static std::string getLogFileName() {
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        const std::lock_guard<std::recursive_mutex> lock(perfettoLogfileMutex);
+        return perfettoLogFileName ? *perfettoLogFileName : std::string{};
+#else
+        return logFileName ? *logFileName : std::string{};
+#endif
+    }
 
     static void rawWriteBytes(const char *buf, const int len) {
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        const std::lock_guard<std::recursive_mutex> lock(perfettoLogfileMutex);
+#endif
         ensureFileOpen();
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        perfettoLogfile->write(buf, len);
+#else
         logfile->write(buf, len);
+#endif
         if (!CALF_LOG_FILE_BINARY) {
             logfile->flush();
         }
@@ -36,9 +80,16 @@ struct StlLogger : CalfLogBase<StlLogger> {
     }
 
     static void flush() {
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        const std::lock_guard<std::recursive_mutex> lock(perfettoLogfileMutex);
+        if (perfettoLogfile) {
+            perfettoLogfile->flush();
+        }
+#else
         if (logfile) {
             logfile->flush();
         }
+#endif
     }
 
     static void reopenRootArray() {
@@ -50,46 +101,87 @@ struct StlLogger : CalfLogBase<StlLogger> {
 
   private:
     static void ensureFileOpen() {
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        const std::lock_guard<std::recursive_mutex> lock(perfettoLogfileMutex);
+        const auto currentPid = ::getpid();
+        if (perfettoLogfile != nullptr && perfettoLogfile->is_open() &&
+            logfilePid != currentPid) {
+            perfettoLogfile->close();
+            perfettoLogfile.reset();
+            perfettoLogFileName.reset();
+        }
+        if (perfettoLogfile != nullptr && perfettoLogfile->is_open()) {
+            return;
+        }
+#else
         if (logfile != nullptr && logfile->is_open()) {
             return;
         }
+#endif
 
         std::string logDir;
+#ifndef CALF_LOG_FORMAT_PROTOBUF
         std::string prefix;
+#endif
 
         if (const char *env = std::getenv("CALF_LOG_DIR"); env != nullptr) {
             logDir = env;
         } else {
             logDir = CALF_DEFAULT_LOG_FOLDER;
         }
 
+#ifndef CALF_LOG_FORMAT_PROTOBUF
         if (const char *env = std::getenv("CALF_LOG_PREFIX"); env != nullptr) {
             prefix = env;
         } else {
             prefix = CALF_STL_DEFAULT_LOG_FILE_PREFIX;
         }
+#endif
 
         char hostname[CALF_HOSTNAME_BUFFER_SIZE]{};
         ::gethostname(hostname, sizeof(hostname));
         hostname[sizeof(hostname) - 1] = '\0';
 
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        const std::filesystem::path outputFolder{logDir + "/" + hostname};
+#else
         const std::filesystem::path outputFolder{logDir + "/" + CALF_COMPONENT_NAME + "/" +
                                                  hostname};
+#endif
         std::filesystem::create_directories(outputFolder);
 
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        const std::filesystem::path path =
+            outputFolder / (std::string(CALF_COMPONENT_NAME) + "_" + std::to_string(::getpid()) +
+                            CALF_LOG_FILE_EXTENSION);
+#else
         const std::filesystem::path path =
             outputFolder /
             (prefix + std::to_string(calf_current_tid()) + CALF_LOG_FILE_EXTENSION);
+#endif
 
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        std::ios::openmode mode = std::ios::out | std::ios::trunc | std::ios::binary;
+#else
         std::ios::openmode mode = std::ios::in | std::ios::out | std::ios::trunc;
         if (CALF_LOG_FILE_BINARY) {
             mode |= std::ios::binary;
         }
+#endif
+#ifdef CALF_LOG_FORMAT_PROTOBUF
+        perfettoLogfile = std::make_unique<std::fstream>(path, mode);
+        perfettoLogFileName = std::make_unique<std::string>(path.string());
+        logfilePid = currentPid;
+#else
         logfile = std::make_unique<std::fstream>(path, mode);
         logFileName = std::make_unique<std::string>(path.string());
+#endif
     }
 };
 
+using StlLogger = CALF_STL_LOGGER_TYPE;
+#undef CALF_STL_LOGGER_TYPE
+
 using Logger = TemplateLogger<StlLogger>;
 
 #ifdef CALF_LOG