First commit: demucs.cpp

Author: sevagh

.clang-format

@@ -0,0 +1,6 @@
+UseTab: Never
+IndentWidth: 4
+BreakBeforeBraces: Allman
+AllowShortIfStatementsOnASingleLine: false
+IndentCaseLabels: false
+ColumnLimit: 80

.gitignore

@@ -0,0 +1,7 @@
+/build/
+*.bin
+*.wav
+!/test/data/*.wav
+*.json
+/__pycache__
+*.pyc

CMakeLists.txt

@@ -0,0 +1,86 @@
+# cmake file to compile src/
+# link against included submodules libnyquist
+
+cmake_minimum_required(VERSION 3.0)
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
+
+set(CMAKE_CXX_FLAGS "-Wall -Wextra")
+set(CMAKE_CXX_FLAGS_DEBUG "-g -DEIGEN_FAST_MATH=0 -O0")
+
+set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -march=native -fno-unsafe-math-optimizations -fassociative-math -freciprocal-math -fno-signed-zeros")
+
+# define a macro NDEBUG for Eigen3 release builds
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG")
+
+# set EIGEN_USE_BLAS to 1 and link to OpenBLAS
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DEIGEN_USE_BLAS -DEIGEN_USE_LAPACKE")
+
+project(demucs.cpp)
+enable_testing()
+
+# set C++ standard to C++17
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+# add openmp support
+find_package(OpenMP REQUIRED)
+if(OPENMP_FOUND)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
+endif()
+
+# compile vendored submodule libnyquist
+set(LIBNYQUIST_BUILD_EXAMPLE OFF CACHE BOOL "Disable libnyquist example")
+add_subdirectory(vendor/libnyquist)
+
+# add library Eigen3
+include_directories(vendor/eigen)
+
+# add OpenBLAS for blas + lapack
+find_package(BLAS REQUIRED)
+find_package(LAPACK REQUIRED)
+
+# include vendor submodules libnyquist
+include_directories(vendor/libnyquist/include)
+
+# include src/ as include directory
+include_directories(src)
+
+# include src/*.cpp and src/*.c as source files
+file(GLOB SOURCES "src/*.cpp")
+
+# compile library, link against libnyquist
+add_library(demucs.cpp.lib SHARED ${SOURCES})
+target_link_libraries(demucs.cpp.lib libnyquist ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES} lapacke)
+if(OPENMP_FOUND)
+    target_link_libraries(demucs.cpp.lib ${OpenMP_CXX_LIBRARIES})
+endif()
+
+file(GLOB SOURCES_TO_LINT "src/*.cpp" "src/*.hpp" "demucs.cpp" "test/*.cpp")
+
+# add target to run standard lints and formatters
+add_custom_target(lint
+    COMMAND clang-format -i ${SOURCES_TO_LINT} --style=file
+    # add clang-tidy command
+    # add include dirs to clang-tidy
+    COMMAND cppcheck --enable=all --suppress=missingIncludeSystem ${SOURCES_TO_LINT} --std=c++17
+    COMMAND scan-build -o ${CMAKE_BINARY_DIR}/scan-build-report make -C ${CMAKE_BINARY_DIR}
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+)
+
+# add target to compile demucs.cpp, the main driver program for demucs.cpp
+add_executable(demucs.cpp.main demucs.cpp)
+# link it against demucs.cpp.lib
+target_link_libraries(demucs.cpp.main demucs.cpp.lib)
+
+# add target to run cpp tests in test/ directory with gtest
+
+# include test/*.cpp as test files
+file(GLOB TEST_SOURCES "test/*.cpp")
+
+add_executable(demucs.cpp.test ${TEST_SOURCES})
+target_link_libraries(demucs.cpp.test demucs.cpp.lib gtest gtest_main libnyquist)
+add_test(NAME tests COMMAND demucs.cpp.test)

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Sevag H
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,50 @@
+# demucs.cpp
+
+Demucs v4 hybrid transformer model reimplemented in C++ with Eigen3
+
+Track 'Zeno - Signs' from MUSDB18-HQ test set
+
+PyTorch CLI inference (output of `demucs /path/to/track` from [this commit of demucs v4](https://github.com/facebookresearch/demucs@2496b8f7f12b01c8dd0187c040000c46e175b44d)):
+```
+vocals          ==> SDR:   8.264  SIR:  18.353  ISR:  15.794  SAR:   8.303
+drums           ==> SDR:  10.111  SIR:  18.503  ISR:  17.089  SAR:  10.746
+bass            ==> SDR:   4.222  SIR:  12.615  ISR:   6.973  SAR:   2.974
+other           ==> SDR:   7.397  SIR:  11.317  ISR:  14.303  SAR:   8.137
+```
+PyTorch custom inference in [my script](./scripts/demucs_pytorch_inference.py):
+```
+vocals          ==> SDR:   8.339  SIR:  18.274  ISR:  15.835  SAR:   8.354
+drums           ==> SDR:  10.058  SIR:  18.598  ISR:  17.023  SAR:  10.812
+bass            ==> SDR:   3.926  SIR:  12.414  ISR:   6.941  SAR:   3.202
+other           ==> SDR:   7.421  SIR:  11.289  ISR:  14.241  SAR:   8.179
+```
+CPP inference (this codebase):
+```
+vocals          ==> SDR:   8.339  SIR:  18.276  ISR:  15.836  SAR:   8.346
+drums           ==> SDR:  10.058  SIR:  18.596  ISR:  17.019  SAR:  10.810
+bass            ==> SDR:   3.919  SIR:  12.436  ISR:   6.931  SAR:   3.182
+other           ==> SDR:   7.421  SIR:  11.286  ISR:  14.252  SAR:   8.183
+```
+
+*n.b.* for testing purposes in this repo, the random shift in the beginning of the song is fixed to 1337 in both PyTorch and C++.
+
+## Build and run
+
+Out-of-source build with CMake:
+```
+$ mkdir -p build && cd build && cmake -DCMAKE_BUILD_TYPE=Release ..
+$ make
+```
+
+The `Release` build type adds optimization flags (Ofast etc.), without which this project is unusably slow.
+
+Run:
+```
+$ ./demucs.cpp.main ../ggml-demucs/ggml-model-htdemucs-f16.bin ../test/data/gspi_stereo.wav  ./demucs-out-cpp/
+```
+
+## Hack
+
+* make lint
+* Valgrind memory error test: `valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose ./demucs.cpp.main ../ggml-demucs/ggml-model-htdemucs-f16.bin ../test/data/gspi_stereo.wav  ./demucs-out-cpp/`
+* 

demucs.cpp

@@ -0,0 +1,111 @@
+#include "dsp.hpp"
+#include "model.hpp"
+#include "tensor.hpp"
+#include <Eigen/Core>
+#include <Eigen/Dense>
+#include <cassert>
+#include <filesystem>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <unsupported/Eigen/FFT>
+#include <vector>
+
+using namespace demucscpp;
+
+int main(int argc, const char **argv)
+{
+    if (argc != 4)
+    {
+        std::cerr << "Usage: " << argv[0]
+                  << " <model file> <wav file> <out dir>" << std::endl;
+        exit(1);
+    }
+
+    // enable openmp parallelization for Eigen
+    // init parallelism for eigen
+    Eigen::initParallel();
+
+    // set eigen nb threads to physical cores minus 1
+    // discover number of physical cores through C++ stdlib
+    // https://stackoverflow.com/questions/150355/programmatically-find-the-number-of-cores-on-a-machine
+    int nb_cores = std::thread::hardware_concurrency();
+    std::cout << "Number of physical cores: " << nb_cores << std::endl;
+    Eigen::setNbThreads(nb_cores - 1);
+
+    std::cout << "demucs.cpp Main driver program" << std::endl;
+
+    // load model passed as argument
+    std::string model_file = argv[1];
+
+    // load audio passed as argument
+    std::string wav_file = argv[2];
+
+    // output dir passed as argument
+    std::string out_dir = argv[3];
+
+    Eigen::MatrixXf audio = load_audio(wav_file);
+    Eigen::Tensor3dXf out_targets;
+
+    std::cout << "Using 4s model" << std::endl;
+
+    // initialize a struct demucs_model
+    struct demucs_model_4s model
+    {
+    };
+
+    auto ret = load_demucs_model_4s(model_file, &model);
+    std::cout << "demucs_model_load returned " << (ret ? "true" : "false")
+              << std::endl;
+    if (!ret)
+    {
+        std::cerr << "Error loading model" << std::endl;
+        exit(1);
+    }
+
+    std::cout << "Starting demucs inference" << std::endl;
+
+    // create 4 audio matrix same size, to hold output
+    Eigen::Tensor3dXf audio_targets =
+        demucscpp::demucs_inference_4s(model, audio);
+    std::cout << "returned!" << std::endl;
+
+    out_targets = audio_targets;
+
+    for (int target = 0; target < 4; ++target)
+    {
+        // now write the 4 audio waveforms to files in the output dir
+        // using libnyquist
+        // join out_dir with "/target_0.wav"
+        // using std::filesystem::path;
+
+        std::filesystem::path p = out_dir;
+        // make sure the directory exists
+        std::filesystem::create_directories(p);
+
+        auto p_target = p / "target_0.wav";
+        // generate p_target = p / "target_{target}.wav"
+        p_target.replace_filename("target_" + std::to_string(target) + ".wav");
+
+        std::cout << "Writing wav file " << p_target << std::endl;
+
+        Eigen::MatrixXf target_waveform(2, audio.cols());
+
+        // copy the input stereo wav file into all 4 targets
+        for (int channel = 0; channel < 2; ++channel)
+        {
+            for (int sample = 0; sample < audio.cols(); ++sample)
+            {
+                target_waveform(channel, sample) =
+                    out_targets(target, channel, sample);
+            }
+        }
+
+        demucscppdebug::debug_matrix_xf(target_waveform,
+                                        "target_waveform for target " +
+                                            std::to_string(target));
+
+        demucscpp::write_audio_file(target_waveform, p_target);
+    }
+}