From 7734e6381c32b5d8016bbd37c7e5bc67bd83d401 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Jul 2023 22:57:02 +0200 Subject: [PATCH 01/20] cmake project --- CMakeLists.txt | 29 ++++++ CMakePresets.json | 125 +++++++++++++++++++++++++ cmd/CMakeLists.txt | 14 +++ lib/CMakeLists.txt | 24 +++++ lib/archiveblockstore/CMakeLists.txt | 1 + lib/atomiccancel/CMakeLists.txt | 1 + lib/bikeshed/CMakeLists.txt | 1 + lib/blake2/CMakeLists.txt | 19 ++++ lib/blake3/CMakeLists.txt | 54 +++++++++++ lib/blockstorestorage/CMakeLists.txt | 1 + lib/brotli/CMakeLists.txt | 94 +++++++++++++++++++ lib/cacheblockstore/CMakeLists.txt | 1 + lib/compressblockstore/CMakeLists.txt | 1 + lib/compressionregistry/CMakeLists.txt | 15 +++ lib/filestorage/CMakeLists.txt | 1 + lib/fsblockstore/CMakeLists.txt | 1 + lib/hashregistry/CMakeLists.txt | 10 ++ lib/hpcdcchunker/CMakeLists.txt | 1 + lib/lrublockstore/CMakeLists.txt | 1 + lib/lz4/CMakeLists.txt | 14 +++ lib/memstorage/CMakeLists.txt | 1 + lib/memtracer/CMakeLists.txt | 1 + lib/meowhash/CMakeLists.txt | 15 +++ lib/ratelimitedprogress/CMakeLists.txt | 1 + lib/shareblockstore/CMakeLists.txt | 1 + lib/zstd/CMakeLists.txt | 72 ++++++++++++++ shared_lib/CMakeLists.txt | 31 ++++++ src/CMakeLists.txt | 1 + static_lib/CMakeLists.txt | 31 ++++++ test/CMakeLists.txt | 8 ++ 30 files changed, 570 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 CMakePresets.json create mode 100644 cmd/CMakeLists.txt create mode 100644 lib/CMakeLists.txt create mode 100644 lib/archiveblockstore/CMakeLists.txt create mode 100644 lib/atomiccancel/CMakeLists.txt create mode 100644 lib/bikeshed/CMakeLists.txt create mode 100644 lib/blake2/CMakeLists.txt create mode 100644 lib/blake3/CMakeLists.txt create mode 100644 lib/blockstorestorage/CMakeLists.txt create mode 100644 lib/brotli/CMakeLists.txt create mode 100644 lib/cacheblockstore/CMakeLists.txt create mode 100644 lib/compressblockstore/CMakeLists.txt create mode 100644 lib/compressionregistry/CMakeLists.txt create mode 100644 lib/filestorage/CMakeLists.txt create mode 100644 lib/fsblockstore/CMakeLists.txt create mode 100644 lib/hashregistry/CMakeLists.txt create mode 100644 lib/hpcdcchunker/CMakeLists.txt create mode 100644 lib/lrublockstore/CMakeLists.txt create mode 100644 lib/lz4/CMakeLists.txt create mode 100644 lib/memstorage/CMakeLists.txt create mode 100644 lib/memtracer/CMakeLists.txt create mode 100644 lib/meowhash/CMakeLists.txt create mode 100644 lib/ratelimitedprogress/CMakeLists.txt create mode 100644 lib/shareblockstore/CMakeLists.txt create mode 100644 lib/zstd/CMakeLists.txt create mode 100644 shared_lib/CMakeLists.txt create mode 100644 src/CMakeLists.txt create mode 100644 static_lib/CMakeLists.txt create mode 100644 test/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..851f9ca8 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,29 @@ +# CMakeList.txt : Top-level CMake project file, do global configuration +# and include sub-projects here. +# +cmake_minimum_required (VERSION 3.8) + +# Enable Hot Reload for MSVC compilers if supported. +if (POLICY CMP0141) + cmake_policy(SET CMP0141 NEW) + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "$,$>,$<$:EditAndContinue>,$<$:ProgramDatabase>>") +endif() + +project ("longtail") + +# Include sub-projects. +add_subdirectory("lib") + +add_subdirectory("src") + +add_subdirectory("static_lib") + +add_subdirectory("shared_lib") + +add_subdirectory ("cmd") + +add_subdirectory("test") + +enable_testing() + +add_test(NAME test COMMAND test_base WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/test) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000..351f244e --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,125 @@ +{ + "version": 3, + "configurePresets": [ + { + "name": "windows-gcc-base", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_C_COMPILER": "gcc.exe", + "CMAKE_CXX_COMPILER": "g++.exe" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "windows-base", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_C_COMPILER": "cl.exe", + "CMAKE_CXX_COMPILER": "cl.exe" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "x64-debug", + "displayName": "x64 Debug", + "inherits": "windows-base", + "architecture": { + "value": "x64", + "strategy": "external" + }, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "x64-release", + "displayName": "x64 Release", + "inherits": "x64-debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "x64-gcc-debug", + "displayName": "x64 GCC Debug", + "inherits": "windows-gcc-base", + "architecture": { + "value": "x64", + "strategy": "external" + }, + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "x64-gcc-release", + "displayName": "x64 GCC Release", + "inherits": "x64-gcc-debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "linux-debug", + "displayName": "Linux Debug", + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Linux" + }, + "vendor": { + "microsoft.com/VisualStudioRemoteSettings/CMake/1.0": { + "sourceDir": "$env{HOME}/.vs/$ms{projectDirName}" + } + } + }, + { + "name": "macos-debug", + "displayName": "macOS Debug", + "generator": "Ninja", + "binaryDir": "${sourceDir}/out/build/${presetName}", + "installDir": "${sourceDir}/out/install/${presetName}", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + }, + "vendor": { + "microsoft.com/VisualStudioRemoteSettings/CMake/1.0": { + "sourceDir": "$env{HOME}/.vs/$ms{projectDirName}" + } + } + } + ], + "testPresets": [ + { + "name": "testPresets", + "displayName": "Custom test preset", + "description": "Custom test preset description", + "configurePreset": "x64-debug" + } + ] +} \ No newline at end of file diff --git a/cmd/CMakeLists.txt b/cmd/CMakeLists.txt new file mode 100644 index 00000000..03bd1d37 --- /dev/null +++ b/cmd/CMakeLists.txt @@ -0,0 +1,14 @@ +# CMakeList.txt : CMake project for longtail, include source and define +# project specific logic here. +# + +# Add source to this project's executable. +add_executable (longtail "main.c" "ext/kgflags.h") + +if (CMAKE_VERSION VERSION_GREATER 3.12) + set_property(TARGET longtail PROPERTY CXX_STANDARD 20) +endif() + +# TODO: Add tests and install targets if needed. +target_link_libraries(longtail PRIVATE static_lib) + diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 00000000..617d2c3c --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,24 @@ +add_library(platform STATIC "longtail_platform.c" "longtail_platform.h") + +add_subdirectory ("archiveblockstore") +add_subdirectory ("atomiccancel") +add_subdirectory ("bikeshed") +add_subdirectory ("blake2") +add_subdirectory ("blake3") +add_subdirectory ("blockstorestorage") +add_subdirectory ("brotli") +add_subdirectory ("cacheblockstore") +add_subdirectory ("compressblockstore") +add_subdirectory ("compressionregistry") +add_subdirectory ("filestorage") +add_subdirectory ("fsblockstore") +add_subdirectory ("hashregistry") +add_subdirectory ("hpcdcchunker") +add_subdirectory ("lrublockstore") +add_subdirectory ("lz4") +add_subdirectory ("memstorage") +add_subdirectory ("memtracer") +add_subdirectory ("meowhash") +add_subdirectory ("ratelimitedprogress") +add_subdirectory ("shareblockstore") +add_subdirectory ("zstd") diff --git a/lib/archiveblockstore/CMakeLists.txt b/lib/archiveblockstore/CMakeLists.txt new file mode 100644 index 00000000..7c987bc0 --- /dev/null +++ b/lib/archiveblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(archiveblockstore OBJECT "longtail_archiveblockstore.c" "longtail_archiveblockstore.h") diff --git a/lib/atomiccancel/CMakeLists.txt b/lib/atomiccancel/CMakeLists.txt new file mode 100644 index 00000000..e81a4d49 --- /dev/null +++ b/lib/atomiccancel/CMakeLists.txt @@ -0,0 +1 @@ +add_library(atomiccancel OBJECT "longtail_atomiccancel.c" "longtail_atomiccancel.h") diff --git a/lib/bikeshed/CMakeLists.txt b/lib/bikeshed/CMakeLists.txt new file mode 100644 index 00000000..2272a116 --- /dev/null +++ b/lib/bikeshed/CMakeLists.txt @@ -0,0 +1 @@ +add_library(bikeshed OBJECT "longtail_bikeshed.c" "longtail_bikeshed.h" "ext/bikeshed.h") diff --git a/lib/blake2/CMakeLists.txt b/lib/blake2/CMakeLists.txt new file mode 100644 index 00000000..dc0cf0fe --- /dev/null +++ b/lib/blake2/CMakeLists.txt @@ -0,0 +1,19 @@ +add_library(blake2 OBJECT + "longtail_blake2.c" + "longtail_blake2.h" + "ext/blake2-config.h" + "ext/blake2-impl.h" + "ext/blake2.h" + "ext/blake2s-load-sse2.h" + "ext/blake2s-load-sse41.h" + "ext/blake2s-round.h" + "ext/blake2s.c" +) + +if(MSVC) + target_compile_options(blake2 INTERFACE /arch:SSE2) + target_compile_options(blake2 PUBLIC -DHAVE_SSE2) +else() + target_compile_options(blake2 PUBLIC -msse2) +endif(MSVC) + \ No newline at end of file diff --git a/lib/blake3/CMakeLists.txt b/lib/blake3/CMakeLists.txt new file mode 100644 index 00000000..e84d2a21 --- /dev/null +++ b/lib/blake3/CMakeLists.txt @@ -0,0 +1,54 @@ +if(CMAKE_ANDROID_ARM_NEON) + set ($BLAKE2_platform_sources "ext/blake3_neon.c") +elseif (MSVC OR MSYS OR MINGW) + set ($BLAKE2_platform_sources "ext/blake3_avx2_x86-64_windows_msvc.asm" "ext/blake3_avx512_x86-64_windows_msvc.asm" "ext/blake3_sse2_x86-64_windows_msvc.asm" "ext/blake3_sse41_x86-64_windows_msvc.asm") +elseif (WIN32) + set ($BLAKE2_platform_sources "ext/blake3_avx2_x86-64_windows_gnu.S" "ext/blake3_avx512_x86-64_windows_gnu.S" "ext/blake3_sse2_x86-64_windows_gnu.S" "ext/blake3_sse41_x86-64_windows_gnu.S") +else() + set ($BLAKE2_platform_sources "ext/blake3_avx2_x86-64_unix.S" "ext/blake3_avx512_x86-64_unix.S" "ext/blake3_sse2_x86-64_unix.S" "ext/blake3_sse41_x86-64_unix.S") +endif() + +add_library( + blake3 OBJECT + "longtail_blake3.c" + "longtail_blake3.h" + + "ext/blake3.c" + "ext/blake3.h" + "ext/blake3_dispatch.c" + "ext/blake3_impl.h" + "ext/blake3_portable.c" + ${BLAKE2_platform_sources} +) + +add_library( + blake3_sse2 OBJECT + "ext/blake3_sse2.c" +) + +add_library( + blake3_sse4_1 OBJECT + "ext/blake3_sse41.c" +) + +add_library( + blake3_avx2 OBJECT + "ext/blake3_avx2.c" +) + +add_library( + blake3_avx512 OBJECT + "ext/blake3_avx512.c" +) + +if(MSVC) + target_compile_options(blake3_sse2 INTERFACE /arch:SSE2) + target_compile_options(blake3_sse4_1 INTERFACE /arch:SSE4.1) + target_compile_options(blake3_avx2 INTERFACE /arch:AVX2) + target_compile_options(blake3_avx512 INTERFACE /arch:AVX512) +else() + target_compile_options(blake3_sse2 PUBLIC -msse2) + target_compile_options(blake3_sse4_1 PUBLIC -msse4.1) + target_compile_options(blake3_avx2 PUBLIC -mavx2) + target_compile_options(blake3_avx512 PUBLIC -mavx512vl -mavx512f) +endif(MSVC) diff --git a/lib/blockstorestorage/CMakeLists.txt b/lib/blockstorestorage/CMakeLists.txt new file mode 100644 index 00000000..2fb607cc --- /dev/null +++ b/lib/blockstorestorage/CMakeLists.txt @@ -0,0 +1 @@ +add_library(blockstorestorage OBJECT "longtail_blockstorestorage.c" "longtail_blockstorestorage.h") diff --git a/lib/brotli/CMakeLists.txt b/lib/brotli/CMakeLists.txt new file mode 100644 index 00000000..08c64e2d --- /dev/null +++ b/lib/brotli/CMakeLists.txt @@ -0,0 +1,94 @@ +add_library( + brotli OBJECT + "longtail_brotli.c" + "longtail_brotli.h" + + "ext/common/constants.c" + "ext/common/constants.h" + "ext/common/context.c" + "ext/common/context.h" + "ext/common/dictionary.c" + "ext/common/dictionary.h" + "ext/common/platform.c" + "ext/common/platform.h" + "ext/common/transform.c" + "ext/common/transform.h" + "ext/common/version.h" + + "ext/dec/bit_reader.h" + "ext/dec/bit_reader.c" + "ext/dec/decode.c" + "ext/dec/huffman.h" + "ext/dec/huffman.c" + "ext/dec/prefix.h" + "ext/dec/state.h" + "ext/dec/state.c" + + "ext/enc/backward_references.c" + "ext/enc/backward_references.h" + "ext/enc/backward_references_hq.c" + "ext/enc/backward_references_hq.h" + "ext/enc/backward_references_inc.h" + "ext/enc/bit_cost.c" + "ext/enc/bit_cost.h" + "ext/enc/bit_cost_inc.h" + "ext/enc/block_encoder_inc.h" + "ext/enc/block_splitter.c" + "ext/enc/block_splitter.h" + "ext/enc/block_splitter_inc.h" + "ext/enc/brotli_bit_stream.c" + "ext/enc/brotli_bit_stream.h" + "ext/enc/cluster.c" + "ext/enc/cluster.h" + "ext/enc/cluster_inc.h" + "ext/enc/command.c" + "ext/enc/command.h" + "ext/enc/compress_fragment.c" + "ext/enc/compress_fragment.h" + "ext/enc/compress_fragment_two_pass.c" + "ext/enc/compress_fragment_two_pass.h" + "ext/enc/dictionary_hash.c" + "ext/enc/dictionary_hash.h" + "ext/enc/encode.c" + "ext/enc/encoder_dict.c" + "ext/enc/encoder_dict.h" + "ext/enc/entropy_encode.c" + "ext/enc/entropy_encode.h" + "ext/enc/entropy_encode_static.h" + "ext/enc/fast_log.c" + "ext/enc/fast_log.h" + "ext/enc/find_match_length.h" + "ext/enc/hash.h" + "ext/enc/hash_composite_inc.h" + "ext/enc/hash_forgetful_chain_inc.h" + "ext/enc/hash_longest_match64_inc.h" + "ext/enc/hash_longest_match_inc.h" + "ext/enc/hash_longest_match_quickly_inc.h" + "ext/enc/hash_rolling_inc.h" + "ext/enc/hash_to_binary_tree_inc.h" + "ext/enc/histogram.c" + "ext/enc/histogram.h" + "ext/enc/histogram_inc.h" + "ext/enc/literal_cost.c" + "ext/enc/literal_cost.h" + "ext/enc/memory.c" + "ext/enc/memory.h" + "ext/enc/metablock.c" + "ext/enc/metablock.h" + "ext/enc/metablock_inc.h" + "ext/enc/params.h" + "ext/enc/prefix.h" + "ext/enc/quality.h" + "ext/enc/ringbuffer.h" + "ext/enc/static_dict.c" + "ext/enc/static_dict.h" + "ext/enc/static_dict_lut.h" + "ext/enc/utf8_util.c" + "ext/enc/utf8_util.h" + "ext/enc/write_bits.h" + + "ext/include/brotli/decode.h" + "ext/include/brotli/encode.h" + "ext/include/brotli/port.h" + "ext/include/brotli/types.h" +) diff --git a/lib/cacheblockstore/CMakeLists.txt b/lib/cacheblockstore/CMakeLists.txt new file mode 100644 index 00000000..f9bdabce --- /dev/null +++ b/lib/cacheblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(cacheblockstore OBJECT "longtail_cacheblockstore.c" "longtail_cacheblockstore.h") diff --git a/lib/compressblockstore/CMakeLists.txt b/lib/compressblockstore/CMakeLists.txt new file mode 100644 index 00000000..54146c95 --- /dev/null +++ b/lib/compressblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(compressblockstore OBJECT "longtail_compressblockstore.c" "longtail_compressblockstore.h") diff --git a/lib/compressionregistry/CMakeLists.txt b/lib/compressionregistry/CMakeLists.txt new file mode 100644 index 00000000..8007f405 --- /dev/null +++ b/lib/compressionregistry/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(compressionregistry_full + OBJECT + "longtail_compression_registry.c" + "longtail_compression_registry.h" + "longtail_full_compression_registry.c" + "longtail_full_compression_registry.h" +) + +add_library(compressionregistry_zstd + OBJECT + "longtail_compression_registry.c" + "longtail_compression_registry.h" + "longtail_zstd_compression_registry.c" + "longtail_zstd_compression_registry.h" +) diff --git a/lib/filestorage/CMakeLists.txt b/lib/filestorage/CMakeLists.txt new file mode 100644 index 00000000..ed5f6a5a --- /dev/null +++ b/lib/filestorage/CMakeLists.txt @@ -0,0 +1 @@ +add_library(filestorage OBJECT "longtail_filestorage.c" "longtail_filestorage.h") diff --git a/lib/fsblockstore/CMakeLists.txt b/lib/fsblockstore/CMakeLists.txt new file mode 100644 index 00000000..e2e5a267 --- /dev/null +++ b/lib/fsblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(fsblockstore OBJECT "longtail_fsblockstore.c" "longtail_fsblockstore.h") diff --git a/lib/hashregistry/CMakeLists.txt b/lib/hashregistry/CMakeLists.txt new file mode 100644 index 00000000..5aa29185 --- /dev/null +++ b/lib/hashregistry/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library( + hashregistry OBJECT + "longtail_hash_registry.c" + "longtail_hash_registry.h" + + "longtail_blake3_hash_registry.c" + "longtail_blake3_hash_registry.h" + "longtail_full_hash_registry.c" + "longtail_full_hash_registry.h" +) diff --git a/lib/hpcdcchunker/CMakeLists.txt b/lib/hpcdcchunker/CMakeLists.txt new file mode 100644 index 00000000..9557f853 --- /dev/null +++ b/lib/hpcdcchunker/CMakeLists.txt @@ -0,0 +1 @@ +add_library(hpcdcchunker OBJECT "longtail_hpcdcchunker.c" "longtail_hpcdcchunker.h") diff --git a/lib/lrublockstore/CMakeLists.txt b/lib/lrublockstore/CMakeLists.txt new file mode 100644 index 00000000..1f5614d1 --- /dev/null +++ b/lib/lrublockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(lrublockstore OBJECT "longtail_lrublockstore.c" "longtail_lrublockstore.h") diff --git a/lib/lz4/CMakeLists.txt b/lib/lz4/CMakeLists.txt new file mode 100644 index 00000000..1069dae9 --- /dev/null +++ b/lib/lz4/CMakeLists.txt @@ -0,0 +1,14 @@ +add_library(lz4 OBJECT + "longtail_lz4.c" + "longtail_lz4.h" + + "ext/lz4.c" + "ext/lz4.h" + "ext/lz4frame.c" + "ext/lz4frame.h" + "ext/lz4frame_static.h" + "ext/lz4hc.c" + "ext/lz4hc.h" + "ext/xxhash.c" + "ext/xxhash.h" +) diff --git a/lib/memstorage/CMakeLists.txt b/lib/memstorage/CMakeLists.txt new file mode 100644 index 00000000..f3bee5b9 --- /dev/null +++ b/lib/memstorage/CMakeLists.txt @@ -0,0 +1 @@ +add_library(memstorage OBJECT "longtail_memstorage.c" "longtail_memstorage.h") diff --git a/lib/memtracer/CMakeLists.txt b/lib/memtracer/CMakeLists.txt new file mode 100644 index 00000000..e76455e2 --- /dev/null +++ b/lib/memtracer/CMakeLists.txt @@ -0,0 +1 @@ +add_library(memtracer OBJECT "longtail_memtracer.c" "longtail_memtracer.h") diff --git a/lib/meowhash/CMakeLists.txt b/lib/meowhash/CMakeLists.txt new file mode 100644 index 00000000..c4e92470 --- /dev/null +++ b/lib/meowhash/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library( + meowhash OBJECT + + "longtail_meowhash.c" + "longtail_meowhash.h" + + "ext/meow_hash_x64_aesni.h" +) + +if(MSVC) + target_compile_options(blake2 INTERFACE /arch:SSE2) + target_compile_options(blake2 PUBLIC -DHAVE_SSE2) +else() + target_compile_options(meowhash PUBLIC -maes -mavx2) +endif(MSVC) diff --git a/lib/ratelimitedprogress/CMakeLists.txt b/lib/ratelimitedprogress/CMakeLists.txt new file mode 100644 index 00000000..ce5d13ff --- /dev/null +++ b/lib/ratelimitedprogress/CMakeLists.txt @@ -0,0 +1 @@ +add_library(ratelimitedprogress OBJECT "longtail_ratelimitedprogress.c" "longtail_ratelimitedprogress.h") diff --git a/lib/shareblockstore/CMakeLists.txt b/lib/shareblockstore/CMakeLists.txt new file mode 100644 index 00000000..031f1142 --- /dev/null +++ b/lib/shareblockstore/CMakeLists.txt @@ -0,0 +1 @@ +add_library(shareblockstore OBJECT "longtail_shareblockstore.c" "longtail_shareblockstore.h") diff --git a/lib/zstd/CMakeLists.txt b/lib/zstd/CMakeLists.txt new file mode 100644 index 00000000..ea1ef070 --- /dev/null +++ b/lib/zstd/CMakeLists.txt @@ -0,0 +1,72 @@ +add_library( + zstd OBJECT + + "longtail_zstd.c" + "longtail_zstd.h" + + "ext/common/bits.h" + "ext/common/bitstream.h" + "ext/common/compiler.h" + "ext/common/cpu.h" + "ext/common/debug.c" + "ext/common/debug.h" + "ext/common/entropy_common.c" + "ext/common/error_private.c" + "ext/common/error_private.h" + "ext/common/fse.h" + "ext/common/fse_decompress.c" + "ext/common/huf.h" + "ext/common/mem.h" + "ext/common/pool.c" + "ext/common/pool.h" + "ext/common/portability_macros.h" + "ext/common/threading.c" + "ext/common/threading.h" + "ext/common/xxhash.c" + "ext/common/xxhash.h" + "ext/common/zstd_common.c" + "ext/common/zstd_deps.h" + "ext/common/zstd_internal.h" + "ext/common/zstd_trace.h" + + "ext/compress/clevels.h" + "ext/compress/fse_compress.c" + "ext/compress/hist.c" + "ext/compress/hist.h" + "ext/compress/huf_compress.c" + "ext/compress/zstdmt_compress.c" + "ext/compress/zstdmt_compress.h" + "ext/compress/zstd_compress.c" + "ext/compress/zstd_compress_internal.h" + "ext/compress/zstd_compress_literals.c" + "ext/compress/zstd_compress_literals.h" + "ext/compress/zstd_compress_sequences.c" + "ext/compress/zstd_compress_sequences.h" + "ext/compress/zstd_compress_superblock.c" + "ext/compress/zstd_compress_superblock.h" + "ext/compress/zstd_cwksp.h" + "ext/compress/zstd_double_fast.c" + "ext/compress/zstd_double_fast.h" + "ext/compress/zstd_fast.c" + "ext/compress/zstd_fast.h" + "ext/compress/zstd_lazy.c" + "ext/compress/zstd_lazy.h" + "ext/compress/zstd_ldm.c" + "ext/compress/zstd_ldm.h" + "ext/compress/zstd_ldm_geartab.h" + "ext/compress/zstd_opt.c" + "ext/compress/zstd_opt.h" + + "ext/decompress/huf_decompress.c" + "ext/decompress/huf_decompress_amd64.S" + "ext/decompress/zstd_ddict.c" + "ext/decompress/zstd_ddict.h" + "ext/decompress/zstd_decompress.c" + "ext/decompress/zstd_decompress_block.c" + "ext/decompress/zstd_decompress_block.h" + "ext/decompress/zstd_decompress_internal.h" + + "ext/zdict.h" + "ext/zstd.h" + "ext/zstd_errors.h" +) diff --git a/shared_lib/CMakeLists.txt b/shared_lib/CMakeLists.txt new file mode 100644 index 00000000..4e6bf7cb --- /dev/null +++ b/shared_lib/CMakeLists.txt @@ -0,0 +1,31 @@ +add_library(shared_lib + SHARED + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..3e59c1d3 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1 @@ +add_library(src STATIC "longtail.c" "longtail.h" "ext/stb_ds.c" "ext/stb_ds.h") diff --git a/static_lib/CMakeLists.txt b/static_lib/CMakeLists.txt new file mode 100644 index 00000000..7b58854e --- /dev/null +++ b/static_lib/CMakeLists.txt @@ -0,0 +1,31 @@ +add_library(static_lib + STATIC + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 00000000..1db477cf --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(test_base + "main.cpp" + "test.cpp" + + "ext/jc_test.h" +) + +target_link_libraries(test_base PRIVATE static_lib) From 64f2def0c6dfd83ec006fc1a9a8a6c45757f857e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Jul 2023 23:01:36 +0200 Subject: [PATCH 02/20] test MSVC Code Analysis --- .github/workflows/pr-build.yml | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 8a00e6a2..f0498733 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -4,6 +4,11 @@ on: pull_request: branches: [ main ] +env: + # Path to the CMake build directory. + build: '${{ github.workspace }}/build' + config: 'Debug' + jobs: cancel-old-build: @@ -14,6 +19,46 @@ jobs: with: access_token: ${{ github.token }} + analyze: + name: Analyze + runs-on: windows-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Configure CMake + run: cmake -B ${{ env.build }} -DCMAKE_BUILD_TYPE=${{ env.config }} + + # Build is not required unless generated source files are used + # - name: Build CMake + # run: cmake --build ${{ env.build }} --config ${{ env.config }} + + - name: Run MSVC Code Analysis + uses: microsoft/msvc-code-analysis-action@v0.1.1 + # Provide a unique ID to access the sarif output path + id: run-analysis + with: + cmakeBuildDirectory: ${{ env.build }} + buildConfiguration: ${{ env.config }} + # Ruleset file that will determine what checks will be run + ruleset: NativeRecommendedRules.ruleset + # Paths to ignore analysis of CMake targets and includes + # ignoredPaths: ${{ github.workspace }}/dependencies;${{ github.workspace }}/test + + # Upload SARIF file to GitHub Code Scanning Alerts + - name: Upload SARIF to GitHub + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: ${{ steps.run-analysis.outputs.sarif }} + + # Upload SARIF file as an Artifact to download and view + - name: Upload SARIF as an Artifact + uses: actions/upload-artifact@v2 + with: + name: sarif-file + path: ${{ steps.run-analysis.outputs.sarif }} + build-linux: runs-on: ubuntu-latest From cb59aa37af28068caf24e78184dd406bb59dfee4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Jul 2023 23:26:17 +0200 Subject: [PATCH 03/20] more analisys tools --- .github/workflows/pr-build.yml | 99 +++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index f0498733..8106d610 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -19,8 +19,8 @@ jobs: with: access_token: ${{ github.token }} - analyze: - name: Analyze + msvc-analyze: + name: MSVC Analyze runs-on: windows-latest steps: @@ -59,6 +59,101 @@ jobs: name: sarif-file path: ${{ steps.run-analysis.outputs.sarif }} + # A workflow run is made up of one or more jobs that can run sequentially or in parallel. + run-cpptest-make: + name: C/C++test Analyze + + # Specifies the type of runner that the job will run on. + runs-on: ubuntu-latest + + # Specifies required permissions for upload-sarif action + permissions: + # required for all workflows + security-events: write + # only required for workflows in private repositories + actions: read + contents: read + + # Steps represent a sequence of tasks that will be executed as part of the job. + steps: + + # Checks out your repository under $GITHUB_WORKSPACE, so that your job can access it. + - name: Checkout code + uses: actions/checkout@v3 + + # Builds your Make project using 'cpptesttrace' to collect input data for code analysis. + # Be sure 'cpptesttrace' is available on $PATH. + - name: Build project + run: cpptesttrace make clean all + + # Runs code analysis with C/C++test. + - name: Run C/C++test + # Use the 'run-cpptest-action' GitHub action. + uses: parasoft/run-cpptest-action@2.0.1 + # Uncomment if you are using C/C++test 2020.2 to generate a SARIF report: + # with: + # reportFormat: xml,html,custom + # additionalParams: '-property report.custom.extension=sarif -property report.custom.xsl.file=${PARASOFT_SARIF_XSL}' + + # Uploads analysis results in the SARIF format, so that they are displayed as GitHub code scanning alerts. + - name: Upload results (SARIF) + if: always() + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: reports/report.sarif + + # Uploads an archive that includes all report files (.xml, .html, .sarif). + - name: Archive reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: CpptestReports + path: reports/*.* + + codeql-analyze: + name: CodeQL Analyze + runs-on: 'ubuntu-latest' + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: cpp + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # Command-line programs to run using the OS shell. + # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:cpp" + build-linux: runs-on: ubuntu-latest From 424f942bd00c26f2c98aab320fd44edc35f54c45 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Jul 2023 23:31:18 +0200 Subject: [PATCH 04/20] test enable -fPIC --- shared_lib/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/shared_lib/CMakeLists.txt b/shared_lib/CMakeLists.txt index 4e6bf7cb..008a7135 100644 --- a/shared_lib/CMakeLists.txt +++ b/shared_lib/CMakeLists.txt @@ -29,3 +29,8 @@ $ $ ) + +if(MSVC) +else() + target_compile_options(shared_lib PUBLIC -fPIC) +endif(MSVC) From c91c811ffe7591c45f84acd5da7236e7bca3331e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Jul 2023 23:36:02 +0200 Subject: [PATCH 05/20] top level -fPIC --- CMakeLists.txt | 5 +++++ shared_lib/CMakeLists.txt | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 851f9ca8..138244bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,11 @@ endif() project ("longtail") +if(MSVC) +else() + add_compile_options(-fPIC -fvisibility=hidden) +endif(MSVC) + # Include sub-projects. add_subdirectory("lib") diff --git a/shared_lib/CMakeLists.txt b/shared_lib/CMakeLists.txt index 008a7135..4e6bf7cb 100644 --- a/shared_lib/CMakeLists.txt +++ b/shared_lib/CMakeLists.txt @@ -29,8 +29,3 @@ $ $ ) - -if(MSVC) -else() - target_compile_options(shared_lib PUBLIC -fPIC) -endif(MSVC) From d5f625dfd2bcc7613006f5429926374d6f87f795 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Jul 2023 23:39:17 +0200 Subject: [PATCH 06/20] disable shared lib --- CMakeLists.txt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 138244bc..03430c4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,11 +11,6 @@ endif() project ("longtail") -if(MSVC) -else() - add_compile_options(-fPIC -fvisibility=hidden) -endif(MSVC) - # Include sub-projects. add_subdirectory("lib") @@ -23,7 +18,7 @@ add_subdirectory("src") add_subdirectory("static_lib") -add_subdirectory("shared_lib") +#add_subdirectory("shared_lib") add_subdirectory ("cmd") From d2b543f5b7dece31862810add0434e16c1aa5d4d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Jul 2023 23:55:32 +0200 Subject: [PATCH 07/20] disable brotli --- lib/CMakeLists.txt | 2 +- static_lib/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 617d2c3c..d007d8ac 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -6,7 +6,7 @@ add_subdirectory ("bikeshed") add_subdirectory ("blake2") add_subdirectory ("blake3") add_subdirectory ("blockstorestorage") -add_subdirectory ("brotli") +#add_subdirectory ("brotli") add_subdirectory ("cacheblockstore") add_subdirectory ("compressblockstore") add_subdirectory ("compressionregistry") diff --git a/static_lib/CMakeLists.txt b/static_lib/CMakeLists.txt index 7b58854e..58155d87 100644 --- a/static_lib/CMakeLists.txt +++ b/static_lib/CMakeLists.txt @@ -12,10 +12,10 @@ $ $ $ - $ +# $ $ $ - $ + $ $ $ $ From 90c6e92ced759807505fc5b4f01edb2e377781d4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 00:07:33 +0200 Subject: [PATCH 08/20] add huf_decompress_amd64.S to non-windows builds --- lib/zstd/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/zstd/CMakeLists.txt b/lib/zstd/CMakeLists.txt index ea1ef070..ca967fb9 100644 --- a/lib/zstd/CMakeLists.txt +++ b/lib/zstd/CMakeLists.txt @@ -1,4 +1,10 @@ -add_library( +if(MSVC) +else() + set ($zstd_platform_sources "ext/decompres/huf_decompress_amd64.S") +endif() + + +add_library( zstd OBJECT "longtail_zstd.c" @@ -69,4 +75,6 @@ "ext/zdict.h" "ext/zstd.h" "ext/zstd_errors.h" + + ${zstd_platform_sources} ) From e49600b5a86f2fca3cf9fa463c385cec0e814b43 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 00:13:10 +0200 Subject: [PATCH 09/20] disable brotli differently --- .../longtail_full_compression_registry.c | 8 ++++---- shared_lib/CMakeLists.txt | 2 +- static_lib/CMakeLists.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/compressionregistry/longtail_full_compression_registry.c b/lib/compressionregistry/longtail_full_compression_registry.c index 4667a45f..77abf241 100644 --- a/lib/compressionregistry/longtail_full_compression_registry.c +++ b/lib/compressionregistry/longtail_full_compression_registry.c @@ -2,18 +2,18 @@ #include "longtail_compression_registry.h" -#include "../brotli/longtail_brotli.h" +//#include "../brotli/longtail_brotli.h" #include "../lz4/longtail_lz4.h" #include "../zstd/longtail_zstd.h" struct Longtail_CompressionRegistryAPI* Longtail_CreateFullCompressionRegistry() { - Longtail_CompressionRegistry_CreateForTypeFunc compression_create_api_funcs[3] = { - Longtail_CompressionRegistry_CreateForBrotli, + Longtail_CompressionRegistry_CreateForTypeFunc compression_create_api_funcs[2] = { + //Longtail_CompressionRegistry_CreateForBrotli, Longtail_CompressionRegistry_CreateForLZ4, Longtail_CompressionRegistry_CreateForZstd}; return Longtail_CreateDefaultCompressionRegistry( - 3, + 2, (const Longtail_CompressionRegistry_CreateForTypeFunc*)compression_create_api_funcs); } diff --git a/shared_lib/CMakeLists.txt b/shared_lib/CMakeLists.txt index 4e6bf7cb..16c21619 100644 --- a/shared_lib/CMakeLists.txt +++ b/shared_lib/CMakeLists.txt @@ -12,7 +12,7 @@ $ $ $ - $ +# $ $ $ $ diff --git a/static_lib/CMakeLists.txt b/static_lib/CMakeLists.txt index 58155d87..dce4bcc4 100644 --- a/static_lib/CMakeLists.txt +++ b/static_lib/CMakeLists.txt @@ -15,7 +15,7 @@ # $ $ $ - $ + $ $ $ $ From 09ab393dc4bc1e04ece2e74a2bf1b2e8d70cedc3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 00:16:43 +0200 Subject: [PATCH 10/20] set fix --- lib/zstd/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd/CMakeLists.txt b/lib/zstd/CMakeLists.txt index ca967fb9..cab9e71e 100644 --- a/lib/zstd/CMakeLists.txt +++ b/lib/zstd/CMakeLists.txt @@ -1,6 +1,6 @@ if(MSVC) else() - set ($zstd_platform_sources "ext/decompres/huf_decompress_amd64.S") + set (zstd_platform_sources "ext/decompres/huf_decompress_amd64.S") endif() From a1069dc92906a7c5551aebd24ade854cd86ded83 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 00:21:00 +0200 Subject: [PATCH 11/20] path fix --- lib/zstd/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/zstd/CMakeLists.txt b/lib/zstd/CMakeLists.txt index cab9e71e..51527c4c 100644 --- a/lib/zstd/CMakeLists.txt +++ b/lib/zstd/CMakeLists.txt @@ -1,6 +1,6 @@ if(MSVC) else() - set (zstd_platform_sources "ext/decompres/huf_decompress_amd64.S") + set (zstd_platform_sources "ext/decompress/huf_decompress_amd64.S") endif() From 62dd55aae99416e9be04f5d20d89d5248a56f6af Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 00:28:17 +0200 Subject: [PATCH 12/20] enable assembler --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03430c4c..116c3d21 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,8 @@ endif() project ("longtail") +set(can_use_assembler TRUE) + # Include sub-projects. add_subdirectory("lib") From 933a81fbb8cd2bb4afbfd183a6d69af19cd4ba08 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 00:33:16 +0200 Subject: [PATCH 13/20] disable zstd asm for now --- lib/zstd/ext/common/portability_macros.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/zstd/ext/common/portability_macros.h b/lib/zstd/ext/common/portability_macros.h index 8fd6ea82..02f10a87 100644 --- a/lib/zstd/ext/common/portability_macros.h +++ b/lib/zstd/ext/common/portability_macros.h @@ -81,7 +81,7 @@ && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ && (defined(__x86_64__) || defined(_M_X64)) \ && !defined(__BMI2__) - # define DYNAMIC_BMI2 1 + # define DYNAMIC_BMI2 0 #else # define DYNAMIC_BMI2 0 #endif @@ -128,7 +128,8 @@ #if !defined(ZSTD_DISABLE_ASM) && \ ZSTD_ASM_SUPPORTED && \ defined(__x86_64__) && \ - (DYNAMIC_BMI2 || defined(__BMI2__)) + (DYNAMIC_BMI2 || defined(__BMI2__) && \ + (0)) # define ZSTD_ENABLE_ASM_X86_64_BMI2 1 #else # define ZSTD_ENABLE_ASM_X86_64_BMI2 0 From 3e80e6895f5511ac394d4c80e0d1b1d56a97b36e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 01:14:34 +0200 Subject: [PATCH 14/20] disable brotli in longtail cmd --- cmd/main.c | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/cmd/main.c b/cmd/main.c index 5df10da9..2f705d48 100644 --- a/cmd/main.c +++ b/cmd/main.c @@ -21,7 +21,7 @@ #include "../lib/meowhash/longtail_meowhash.h" #include "../lib/ratelimitedprogress/longtail_ratelimitedprogress.h" #include "../lib/shareblockstore/longtail_shareblockstore.h" -#include "../lib/brotli/longtail_brotli.h" +//#include "../lib/brotli/longtail_brotli.h" #include "../lib/lz4/longtail_lz4.h" #include "../lib/zstd/longtail_zstd.h" #include "../lib/longtail_platform.h" @@ -172,30 +172,30 @@ uint32_t ParseCompressionType(const char* compression_algorithm) { { return 0; } - if (strcmp("brotli", compression_algorithm) == 0) - { - return Longtail_GetBrotliGenericDefaultQuality(); - } - if (strcmp("brotli_min", compression_algorithm) == 0) - { - return Longtail_GetBrotliGenericMinQuality(); - } - if (strcmp("brotli_max", compression_algorithm) == 0) - { - return Longtail_GetBrotliGenericMaxQuality(); - } - if (strcmp("brotli_text", compression_algorithm) == 0) - { - return Longtail_GetBrotliTextDefaultQuality(); - } - if (strcmp("brotli_text_min", compression_algorithm) == 0) - { - return Longtail_GetBrotliTextMinQuality(); - } - if (strcmp("brotli_text_max", compression_algorithm) == 0) - { - return Longtail_GetBrotliTextMaxQuality(); - } +// if (strcmp("brotli", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliGenericDefaultQuality(); +// } +// if (strcmp("brotli_min", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliGenericMinQuality(); +// } +// if (strcmp("brotli_max", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliGenericMaxQuality(); +// } +// if (strcmp("brotli_text", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliTextDefaultQuality(); +// } +// if (strcmp("brotli_text_min", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliTextMinQuality(); +// } +// if (strcmp("brotli_text_max", compression_algorithm) == 0) +// { +// return Longtail_GetBrotliTextMaxQuality(); +// } if (strcmp("lz4", compression_algorithm) == 0) { return Longtail_GetLZ4DefaultQuality(); From d8b4ef86b54fad62610fb18a79a5fa27a6c81377 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 01:17:51 +0200 Subject: [PATCH 15/20] more brotli disabling --- test/test.cpp | 94 +++++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/test/test.cpp b/test/test.cpp index cea8c300..6f4ed466 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -6,7 +6,7 @@ #include "../lib/blake2/longtail_blake2.h" #include "../lib/blake3/longtail_blake3.h" #include "../lib/bikeshed/longtail_bikeshed.h" -#include "../lib/brotli/longtail_brotli.h" +//#include "../lib/brotli/longtail_brotli.h" #include "../lib/archiveblockstore/longtail_archiveblockstore.h" #include "../lib/atomiccancel/longtail_atomiccancel.h" #include "../lib/blockstorestorage/longtail_blockstorestorage.h" @@ -347,49 +347,49 @@ TEST(Longtail, Longtail_LZ4) Longtail_DisposeAPI(&compression_api->m_API); } -TEST(Longtail, Longtail_Brotli) -{ - Longtail_CompressionAPI* compression_api = Longtail_CreateBrotliCompressionAPI(); - ASSERT_NE((Longtail_CompressionAPI*)0, compression_api); - uint32_t compression_settings = Longtail_GetBrotliTextMaxQuality(); - - const char* raw_data = - "A very long file that should be able to be recreated" - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 2 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 3 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 4 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 5 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 6 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 7 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 8 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 9 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 10 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 11 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 12 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 13 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 14 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 15 in a long sequence of stuff." - "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 16 in a long sequence of stuff." - "And in the end it is not the same, it is different, just because why not"; - - size_t data_len = strlen(raw_data) + 1; - size_t compressed_size = 0; - size_t max_compressed_size = compression_api->GetMaxCompressedSize(compression_api, compression_settings, data_len); - char* compressed_buffer = (char*)Longtail_Alloc(0, max_compressed_size); - ASSERT_NE((char*)0, compressed_buffer); - ASSERT_EQ(0, compression_api->Compress(compression_api, compression_settings, raw_data, compressed_buffer, data_len, max_compressed_size, &compressed_size)); - - char* decompressed_buffer = (char*)Longtail_Alloc(0, data_len); - ASSERT_NE((char*)0, decompressed_buffer); - size_t uncompressed_size; - ASSERT_EQ(0, compression_api->Decompress(compression_api, compressed_buffer, decompressed_buffer, compressed_size, data_len, &uncompressed_size)); - ASSERT_EQ(data_len, uncompressed_size); - ASSERT_STREQ(raw_data, decompressed_buffer); - Longtail_Free(decompressed_buffer); - Longtail_Free(compressed_buffer); - - Longtail_DisposeAPI(&compression_api->m_API); -} +//TEST(Longtail, Longtail_Brotli) +//{ +// Longtail_CompressionAPI* compression_api = Longtail_CreateBrotliCompressionAPI(); +// ASSERT_NE((Longtail_CompressionAPI*)0, compression_api); +// uint32_t compression_settings = Longtail_GetBrotliTextMaxQuality(); +// +// const char* raw_data = +// "A very long file that should be able to be recreated" +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 2 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 3 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 4 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 5 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 6 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 7 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 8 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 9 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 10 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 11 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 12 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 13 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 14 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 15 in a long sequence of stuff." +// "Lots of repeating stuff, some good, some bad but still it is repeating. This is the number 16 in a long sequence of stuff." +// "And in the end it is not the same, it is different, just because why not"; +// +// size_t data_len = strlen(raw_data) + 1; +// size_t compressed_size = 0; +// size_t max_compressed_size = compression_api->GetMaxCompressedSize(compression_api, compression_settings, data_len); +// char* compressed_buffer = (char*)Longtail_Alloc(0, max_compressed_size); +// ASSERT_NE((char*)0, compressed_buffer); +// ASSERT_EQ(0, compression_api->Compress(compression_api, compression_settings, raw_data, compressed_buffer, data_len, max_compressed_size, &compressed_size)); +// +// char* decompressed_buffer = (char*)Longtail_Alloc(0, data_len); +// ASSERT_NE((char*)0, decompressed_buffer); +// size_t uncompressed_size; +// ASSERT_EQ(0, compression_api->Decompress(compression_api, compressed_buffer, decompressed_buffer, compressed_size, data_len, &uncompressed_size)); +// ASSERT_EQ(data_len, uncompressed_size); +// ASSERT_STREQ(raw_data, decompressed_buffer); +// Longtail_Free(decompressed_buffer); +// Longtail_Free(compressed_buffer); +// +// Longtail_DisposeAPI(&compression_api->m_API); +//} TEST(Longtail, Longtail_ZStd) { @@ -1212,14 +1212,14 @@ static uint32_t* GetAssetTags(Longtail_StorageAPI* , const Longtail_FileInfos* f { uint32_t count = file_infos->m_Count; uint32_t* result = (uint32_t*)Longtail_Alloc(0, sizeof(uint32_t) * count); - const uint32_t compression_types[4] = { + const uint32_t compression_types[3] = { 0, - Longtail_GetBrotliGenericDefaultQuality(), +// Longtail_GetBrotliGenericDefaultQuality(), Longtail_GetLZ4DefaultQuality(), Longtail_GetZStdDefaultQuality()}; for (uint32_t i = 0; i < count; ++i) { - result[i] = compression_types[i % 4]; + result[i] = compression_types[i % 3]; } return result; } From c4e6a1b2a84f672bc6f436cf65185d6b1a3b2d4a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 01:28:10 +0200 Subject: [PATCH 16/20] enable asm language --- lib/zstd/ext/common/portability_macros.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/zstd/ext/common/portability_macros.h b/lib/zstd/ext/common/portability_macros.h index 02f10a87..8fd6ea82 100644 --- a/lib/zstd/ext/common/portability_macros.h +++ b/lib/zstd/ext/common/portability_macros.h @@ -81,7 +81,7 @@ && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ && (defined(__x86_64__) || defined(_M_X64)) \ && !defined(__BMI2__) - # define DYNAMIC_BMI2 0 + # define DYNAMIC_BMI2 1 #else # define DYNAMIC_BMI2 0 #endif @@ -128,8 +128,7 @@ #if !defined(ZSTD_DISABLE_ASM) && \ ZSTD_ASM_SUPPORTED && \ defined(__x86_64__) && \ - (DYNAMIC_BMI2 || defined(__BMI2__) && \ - (0)) + (DYNAMIC_BMI2 || defined(__BMI2__)) # define ZSTD_ENABLE_ASM_X86_64_BMI2 1 #else # define ZSTD_ENABLE_ASM_X86_64_BMI2 0 From cc432746fbd27f4ff3f6e891408473c528f9bb5b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 01:39:52 +0200 Subject: [PATCH 17/20] enable asm but disable zstd asm --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 116c3d21..0ba80590 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,11 @@ endif() project ("longtail") -set(can_use_assembler TRUE) +enable_language(C) +enable_language(ASM) +enable_language(CXX) + +add_compile_definitions(ZSTD_DISABLE_ASM) # Include sub-projects. add_subdirectory("lib") From 0cae295a02d3741ccba797f63fedb0070ed870cd Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 21:43:33 +0200 Subject: [PATCH 18/20] try re-enable zstd asm --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ba80590..aa2fef06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ enable_language(C) enable_language(ASM) enable_language(CXX) -add_compile_definitions(ZSTD_DISABLE_ASM) +#add_compile_definitions(ZSTD_DISABLE_ASM) # Include sub-projects. add_subdirectory("lib") From fcebbf96cc67dfb483a5201e8df8e8caf3b50a32 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 21:59:12 +0200 Subject: [PATCH 19/20] remove c/c++test analyze --- .github/workflows/pr-build.yml | 51 ---------------------------------- 1 file changed, 51 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 8106d610..629ddc06 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -59,57 +59,6 @@ jobs: name: sarif-file path: ${{ steps.run-analysis.outputs.sarif }} - # A workflow run is made up of one or more jobs that can run sequentially or in parallel. - run-cpptest-make: - name: C/C++test Analyze - - # Specifies the type of runner that the job will run on. - runs-on: ubuntu-latest - - # Specifies required permissions for upload-sarif action - permissions: - # required for all workflows - security-events: write - # only required for workflows in private repositories - actions: read - contents: read - - # Steps represent a sequence of tasks that will be executed as part of the job. - steps: - - # Checks out your repository under $GITHUB_WORKSPACE, so that your job can access it. - - name: Checkout code - uses: actions/checkout@v3 - - # Builds your Make project using 'cpptesttrace' to collect input data for code analysis. - # Be sure 'cpptesttrace' is available on $PATH. - - name: Build project - run: cpptesttrace make clean all - - # Runs code analysis with C/C++test. - - name: Run C/C++test - # Use the 'run-cpptest-action' GitHub action. - uses: parasoft/run-cpptest-action@2.0.1 - # Uncomment if you are using C/C++test 2020.2 to generate a SARIF report: - # with: - # reportFormat: xml,html,custom - # additionalParams: '-property report.custom.extension=sarif -property report.custom.xsl.file=${PARASOFT_SARIF_XSL}' - - # Uploads analysis results in the SARIF format, so that they are displayed as GitHub code scanning alerts. - - name: Upload results (SARIF) - if: always() - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: reports/report.sarif - - # Uploads an archive that includes all report files (.xml, .html, .sarif). - - name: Archive reports - if: always() - uses: actions/upload-artifact@v3 - with: - name: CpptestReports - path: reports/*.* - codeql-analyze: name: CodeQL Analyze runs-on: 'ubuntu-latest' From 620eaad10fb8d93a18e38963c02b5536d6e3466d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 6 Jul 2023 22:12:10 +0200 Subject: [PATCH 20/20] try enable brotli again --- lib/CMakeLists.txt | 2 +- shared_lib/CMakeLists.txt | 2 +- static_lib/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index d007d8ac..617d2c3c 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -6,7 +6,7 @@ add_subdirectory ("bikeshed") add_subdirectory ("blake2") add_subdirectory ("blake3") add_subdirectory ("blockstorestorage") -#add_subdirectory ("brotli") +add_subdirectory ("brotli") add_subdirectory ("cacheblockstore") add_subdirectory ("compressblockstore") add_subdirectory ("compressionregistry") diff --git a/shared_lib/CMakeLists.txt b/shared_lib/CMakeLists.txt index 16c21619..4e6bf7cb 100644 --- a/shared_lib/CMakeLists.txt +++ b/shared_lib/CMakeLists.txt @@ -12,7 +12,7 @@ $ $ $ -# $ + $ $ $ $ diff --git a/static_lib/CMakeLists.txt b/static_lib/CMakeLists.txt index dce4bcc4..7b58854e 100644 --- a/static_lib/CMakeLists.txt +++ b/static_lib/CMakeLists.txt @@ -12,7 +12,7 @@ $ $ $ -# $ + $ $ $ $