Merge pull request #42 from Fluorescence-Tools/development

Add support for CZ and sm files

Author: tpeulen

doc/whats_new/changes.rst

@@ -5,7 +5,8 @@
 Version 0.24
 ============
 
-**January, 2024**
+* Add support for sm files
+* Add support for CZ CF3 FCS files
 
 
 Version 0.23

include/TTTR.h

@@ -343,6 +343,16 @@ class TTTR : public std::enable_shared_from_this<TTTR>{
      */
     int read_hdf_file(const char *fn);
 
+    /*!
+     * \brief Reads the essential content from a SM file.
+     *
+     * Reads the macro time, and routing channel number from the specified Photon SM file.
+     *
+     * \param fn Filename pointing to the Photon HDF file.
+     * \return Returns an integer indicating the success or failure of the file reading operation.
+     */
+    int read_sm_file(const char *fn);
+
     /*!
      * \brief Reads a specified number of records from the file.
      *

include/TTTRHeader.h

@@ -18,6 +18,7 @@
 #include <sstream>      // std::stringstream
 #include <iomanip> /* std::setfill */
 #include <fstream> /* ifstream */
+#include <cstring> /* std::memcpy */
 
 #include <any>
 // #include <boost/any.hpp>
@@ -44,6 +45,37 @@ const std::string TTTRTagBits = "TTResultFormat_BitsPerRecord";    // Bits per T
 const std::string FileTagEnd = "Header_End";                       // Always appended as last tag (BLOCKEND)
 
 
+/**
+ * Swaps the endianness of a given value.
+ *
+ * This function takes a reference to a value of any type `T` and swaps its byte order
+ * between little-endian and big-endian formats. It uses a union to access the raw bytes
+ * of the value and reverses the byte order using `std::reverse_copy`.
+ *
+ * @tparam T The type of the value whose endianness is to be swapped. Must be trivially
+ *            copyable and have a defined byte size.
+ * @param val A reference to the value whose endianness is to be swapped. The value is
+ *            modified in-place.
+ *
+ * Example:
+ *
+ * int32_t original = 0x12345678;
+ * SwapEndian(original);
+ * // original now contains 0x78563412
+ */
+template <typename T>
+void SwapEndian(T &val) {
+    union U {
+        T val;
+        std::array<std::uint8_t, sizeof(T)> raw;
+    } src, dst;
+
+    src.val = val;
+    std::reverse_copy(src.raw.begin(), src.raw.end(), dst.raw.begin());
+    val = dst.val;
+}
+
+
 class TTTRHeader {
 
     friend class TTTR;
@@ -275,8 +307,22 @@ class TTTRHeader {
             bool rewind = true
     );
 
+    /**
+     * Reads and parses the header of an SM (Single molecule) record from the given file.
+     * The parsed information is stored in the provided JSON object `j`.
+     *
+     * This function performs the following tasks:
+     * 1. Adds a tag to the JSON object `j` for the record type.
+     * 2. Reads and processes the header information from the file.
+     *
+     * @param file A pointer to the file from which the header is read.
+     * @param j A reference to a nlohmann::json object where parsed information will be stored.
+     * @return The file position after reading the header.
+     */
+    static size_t read_sm_header(FILE* file, nlohmann::json &j);
+
     /*!
-     * @brief Reads the header of a Becker & Hickel SPC132 file and sets the reading routing.
+     * @brief Reads the header of a Becker & Hickl SPC132 file and sets the reading routing.
      *
      * @param fpin File pointer to the SPC132 file.
      * @param data Output parameter for JSON data.

include/TTTRHeaderTypes.h

@@ -38,6 +38,7 @@
 #define BH_SPC600_4096_CONTAINER  4
 #define PHOTON_HDF_CONTAINER      5
 #define CZ_CONFOCOR3_CONTAINER    6
+#define SM_CONTAINER              7
 
 // tttrlib record type identifier definitions
 #define PQ_RECORD_TYPE_HHT2v2       1
@@ -50,6 +51,7 @@
 #define BH_RECORD_TYPE_SPC600_256   8
 #define BH_RECORD_TYPE_SPC600_4096  9
 #define CZ_RECORD_TYPE_CONFOCOR3    10
+#define SM_RECORD_TYPE              11
 
 
 /*
@@ -139,6 +141,30 @@ typedef struct {
 } pq_ht3_TTModeHeader_t;
 
 
+
+// Header information structure
+typedef struct {
+    uint32_t version;
+    std::string comment;
+    std::string simple_str;
+    uint32_t pointer1;
+    std::string file_section_type;
+    uint32_t magic1;
+    uint32_t magic2;
+    std::string col1_name;
+    double col1_resolution;
+    double col1_offset;
+    uint32_t col1_bho;
+    std::string col2_name;
+    double col2_resolution;
+    double col2_offset;
+    uint32_t col2_bho;
+    std::string col3_name;
+    double col3_resolution;
+    double col3_offset;
+    std::vector<std::string> channel_labels;
+} sm_header_t;
+
 /// Carl Zeiss Confocor3 raw data
 typedef union cz_confocor3_settings{
     uint32_t allbits;

include/TTTRRecordTypes.h

@@ -3,6 +3,12 @@
 
 #include <cstdint>
 
+// SM files
+typedef union sm_record {
+    uint64_t time;
+    uint32_t channel;
+} sm_record_t;
+
 
 // HydraHarp/TimeHarp260 T2 record
 typedef union pq_hh_t2_record {

pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-version = "0.24.3"
+version = "0.24.4"
 name = "tttrlib"
 requires-python = ">=3.8"
 description = "Read, write & process time-tagged time-resolved (TTTR) data."

src/TTTR.cpp

@@ -27,6 +27,7 @@ TTTR::TTTR() :
     container_names.insert({std::string("SPC-600_4096"), BH_SPC600_4096_CONTAINER});
     container_names.insert({std::string("PHOTON-HDF5"), PHOTON_HDF_CONTAINER});
     container_names.insert({std::string("CZ-RAW"), CZ_CONFOCOR3_CONTAINER});
+    container_names.insert({std::string("SM"), SM_CONTAINER});
     header = new TTTRHeader(tttr_container_type);
     allocate_memory_for_records(0);
 }
@@ -242,6 +243,96 @@ int TTTR::read_hdf_file(const char *fn){
     return 0;
 }
 
+int TTTR::read_sm_file(const char *filename){
+    // Function to read a 64-bit big-endian value
+
+    // Open the file using fopen
+    FILE* fp = fopen(filename, "rb");
+    if (fp == nullptr) {
+        std::cerr << "Error opening file: " << filename << std::endl;
+        return 1;
+    }
+
+    if (!fp) {
+        std::cerr << "Error opening file: " << filename << std::endl;
+        return 1;
+    }
+
+    // Decode header
+    header = new TTTRHeader(fp, SM_CONTAINER);
+
+    // Skip the header (165 bytes)
+    size_t HEADER_SIZE = header->header_end;
+    if (fseek(fp, HEADER_SIZE, SEEK_SET) != 0) {
+        std::cerr << "Error seeking past the header." << std::endl;
+        fclose(fp);
+        return 1;
+    }
+
+    // Determine file size to calculate remaining data size
+    fseek(fp, 0, SEEK_END);
+    long fileSize = ftell(fp);
+    fseek(fp, HEADER_SIZE, SEEK_SET); // Return to start of data after header
+
+    if (fileSize < HEADER_SIZE + 26) {
+        std::cerr << "Error: File is too short to contain expected data and trailing bytes." << std::endl;
+        fclose(fp);
+        return 1;
+    }
+
+    size_t dataSize = fileSize - HEADER_SIZE - 26;
+
+    // Allocate buffer to hold the data
+    std::vector<uint8_t> buffer(dataSize);
+    if (fread(buffer.data(), 1, dataSize, fp) != dataSize) {
+        std::cerr << "Error reading data from file." << std::endl;
+        fclose(fp);
+        return 1;
+    }
+
+    fclose(fp);  // Close the file after reading
+
+    // Define inline lambda functions for endian conversion
+    auto readBigEndian64 = [](const uint8_t* data) -> uint64_t {
+        return (static_cast<uint64_t>(data[0]) << 56) |
+               (static_cast<uint64_t>(data[1]) << 48) |
+               (static_cast<uint64_t>(data[2]) << 40) |
+               (static_cast<uint64_t>(data[3]) << 32) |
+               (static_cast<uint64_t>(data[4]) << 24) |
+               (static_cast<uint64_t>(data[5]) << 16) |
+               (static_cast<uint64_t>(data[6]) << 8)  |
+               (static_cast<uint64_t>(data[7]));
+    };
+
+    auto readBigEndian16 = [](const uint8_t* data) -> uint16_t {
+        return (static_cast<uint16_t>(data[0]) << 8) | static_cast<uint16_t>(data[1]);
+    };
+
+    // Process the data in 12-byte records
+    const size_t RECORD_SIZE = 12;
+    if (buffer.size() % RECORD_SIZE != 0) {
+        std::cerr << "Error: Data size is not a multiple of record size." << std::endl;
+        return 1;
+    }
+
+    size_t numRecords = buffer.size() / RECORD_SIZE;
+    n_valid_events = numRecords;
+    n_records_in_file = n_valid_events;
+    allocate_memory_for_records(numRecords);
+    for (size_t i = 0; i < numRecords; ++i) {
+        const uint8_t* record = buffer.data() + i * RECORD_SIZE;
+
+        // Read and interpret the 64-bit PH time
+        macro_times[i] = readBigEndian64(record);
+
+        // Read and interpret the 16-bit detector
+        routing_channels[i] = readBigEndian16(record + 8);
+    }
+
+    return 0;
+
+}
+
 int TTTR::read_file(const char *fn, int container_type) {
 #ifdef VERBOSE_TTTRLIB
     std::clog << "READING TTTR FILE" << std::endl;
@@ -263,7 +354,10 @@ int TTTR::read_file(const char *fn, int container_type) {
         fn = filename.c_str();
         if (container_type == PHOTON_HDF_CONTAINER) {
             read_hdf_file(fn);
-        } else{
+        } else if (container_type == SM_CONTAINER){
+            read_sm_file(fn);
+        }
+        else{
             fp = fopen(fn, "rb");
             header = new TTTRHeader(fp, container_type);
             fp_records_begin = header->end();

src/TTTRHeader.cpp

@@ -74,6 +74,9 @@ TTTRHeader::TTTRHeader(
     } else if(tttr_container_type == CZ_CONFOCOR3_CONTAINER) {
         header_end = read_cz_confocor3_header(fpin, json_data);
         tttr_record_type = get_tag(json_data, TTTRRecordType)["value"];
+    } else if(tttr_container_type == SM_CONTAINER){
+        header_end = read_sm_header(fpin, json_data);
+        tttr_record_type = get_tag(json_data, TTTRRecordType)["value"];
     }
     else if(tttr_container_type == PQ_HT3_CONTAINER){
         header_end = read_ht3_header(fpin, json_data);
@@ -144,6 +147,88 @@ size_t TTTRHeader::read_bh132_header(
 }
 
 
+size_t TTTRHeader::read_sm_header(FILE* file, nlohmann::json &j) {
+
+    add_tag(j, TTTRRecordType, (int) SM_RECORD_TYPE, tyInt8);
+
+    // Helper lambda to read and swap endianness
+    auto read_and_swap = [&](auto& value) {
+        fread(&value, sizeof(value), 1, file);
+        SwapEndian(value);
+    };
+
+    // Helper lambda to read a string with its size
+    auto read_string = [&](const std::string& tag_name) {
+        uint32_t size;
+        read_and_swap(size);
+        char* buffer = new char[size];
+        fread(buffer, sizeof(char), size, file);
+        add_tag(j, tag_name, buffer, tyAnsiString);
+        delete[] buffer;
+    };
+    sm_header_t header;  // Use only the 'header' structure
+
+    // Read and swap the version
+    read_and_swap(header.version);
+    add_tag(j, "version", (int) header.version, tyInt8);
+
+    read_string("comment");
+    read_string("simple");
+
+    read_and_swap(header.pointer1);
+    add_tag(j, "pointer1", (int) header.pointer1, tyInt8);
+
+    read_string("file_section_type");
+
+    read_and_swap(header.magic1);
+    add_tag(j, "magic1", (int) header.magic1, tyInt8);
+    read_and_swap(header.magic2);
+    add_tag(j, "magic2", (int) header.magic2, tyInt8);
+
+    read_string("col1_name");
+    read_and_swap(header.col1_resolution);
+    add_tag(j, "col1_resolution", (double) header.col1_resolution, tyFloat8);
+    read_and_swap(header.col1_offset);
+    add_tag(j, "col1_offset", (double) header.col1_offset, tyFloat8);
+    read_and_swap(header.col1_bho);
+    add_tag(j, "col1_bho", (int) header.col1_bho, tyInt8);
+
+    // Read the column 2 information
+    read_string("col2_name");
+    read_and_swap(header.col2_resolution);
+    add_tag(j, "col2_resolution", (double) header.col2_resolution, tyFloat8);
+    read_and_swap(header.col2_offset);
+    add_tag(j, "col2_offset", (double) header.col2_offset, tyFloat8);
+    read_and_swap(header.col2_bho);
+    add_tag(j, "col2_bho", (int) header.col2_bho, tyInt8);
+
+    read_string("col3_name");
+    read_and_swap(header.col3_resolution);
+    add_tag(j, "col3_resolution", (double) header.col3_resolution, tyFloat8);
+    read_and_swap(header.col3_offset);
+    add_tag(j, "col3_offset", (double) header.col3_offset, tyFloat8);
+
+    // Read the number of channels
+    int32_t num_channels;
+    read_and_swap(num_channels);
+    add_tag(j, "num_channels", (int) num_channels, tyInt8);
+
+    header.channel_labels.resize(num_channels);
+    for (uint32_t i = 0; i < num_channels; ++i) {
+        uint32_t size;
+        fread(&size, sizeof(size), 1, file);
+        SwapEndian(size);
+        fseek(file, size, SEEK_CUR);
+    }
+
+    add_tag(j, TTTRTagGlobRes, (double) header.col2_resolution, tyFloat8);
+
+    // Return the current file position, which is the cursor
+    return ftell(file);
+}
+
+
+
 size_t TTTRHeader::read_cz_confocor3_header(
         std::FILE *fpin,
         nlohmann::json &data,
@@ -484,6 +569,7 @@ void TTTRHeader::write_spc132_header(
     fclose(fp);
 }
 
+
 void TTTRHeader::write_ptu_header(std::string fn, TTTRHeader* header, std::string modes){
     #ifdef VERBOSE_TTTRLIB
     std::clog << "TTTRHeader::write_ptu_header" << std::endl;

tools/conda_build_config.yaml

@@ -1,5 +1,5 @@
 CONDA_BUILD_SYSROOT:
-  - /opt/MacOSX10.13.sdk        # [osx]
+  - /opt/MacOSX10.9.sdk        # [osx]
 c_compiler:
   - gcc                        # [linux]
   - clang                      # [osx]