feat(code): code refactor

.gitignore

@@ -1,14 +1,14 @@
 *.wav
-/test/**/
-!/test/sample1_16k.wav
-!/test/sample2_22k.wav
-!/test/sample3_48k.wav
-!/test/sample4_16k.wav
-!/test/sample5_16k.wav
-!/test/sample6_48k.wav
-!/test/sample7_16k.wav
-!/test/sample8_16k.wav
-!/test/sample9_48k.wav
-!/test/sample10_16k.wav
+/AudCodec/test/**/
+!/AudCodec/test/sample1_16k.wav
+!/AudCodec/test/sample2_22k.wav
+!/AudCodec/test/sample3_48k.wav
+!/AudCodec/test/sample4_16k.wav
+!/AudCodec/test/sample5_16k.wav
+!/AudCodec/test/sample6_48k.wav
+!/AudCodec/test/sample7_16k.wav
+!/AudCodec/test/sample8_16k.wav
+!/AudCodec/test/sample9_48k.wav
+!/AudCodec/test/sample10_16k.wav
 
 __pycache__/

base_codec/academicodec.py → AudCodec/base_codec/academicodec.py

@@ -12,7 +12,7 @@
 from torch.nn.utils import spectral_norm
 from torch.nn.utils import weight_norm
 from librosa.util import normalize
-from base_codec.general import save_audio, ExtractedUnit
+from AudCodec.base_codec.general import save_audio, ExtractedUnit
 
 LRELU_SLOPE = 0.1
 

base_codec/audiodec.py → AudCodec/base_codec/audiodec.py

@@ -1,6 +1,6 @@
 import nlp2
 import torch
-from base_codec.general import save_audio, ExtractedUnit
+from AudCodec.base_codec.general import save_audio, ExtractedUnit
 
 
 class BaseCodec:

base_codec/descript_audio_codec.py → AudCodec/base_codec/descript_audio_codec.py

@@ -1,4 +1,4 @@
-from base_codec.general import save_audio, ExtractedUnit
+from AudCodec.base_codec.general import save_audio, ExtractedUnit
 import torch
 from audiotools import AudioSignal
 

base_codec/encodec.py → AudCodec/base_codec/encodec.py

@@ -1,5 +1,5 @@
 import torch
-from base_codec.general import save_audio, ExtractedUnit
+from AudCodec.base_codec.general import save_audio, ExtractedUnit
 
 
 class BaseCodec:

base_codec/encodec_hf.py → AudCodec/base_codec/encodec_hf.py

@@ -1,6 +1,6 @@
 import torch
 from transformers import AutoModel, AutoProcessor
-from base_codec.general import save_audio, ExtractedUnit
+from AudCodec.base_codec.general import save_audio, ExtractedUnit
 
 
 class BaseCodec:

base_codec/funcodec.py → AudCodec/base_codec/funcodec.py

@@ -2,7 +2,7 @@
 import torch
 import os
 
-from base_codec.general import save_audio, ExtractedUnit
+from AudCodec.base_codec.general import save_audio, ExtractedUnit
 from audiotools import AudioSignal
 
 

base_codec/speech_tokenizer.py → AudCodec/base_codec/speech_tokenizer.py

@@ -1,6 +1,6 @@
 import numpy
 
-from base_codec.general import save_audio, ExtractedUnit
+from AudCodec.base_codec.general import save_audio, ExtractedUnit
 import torchaudio
 import torch
 import nlp2

codec/academicodec_hifi_16k_320d.py → AudCodec/codec/academicodec_hifi_16k_320d.py

@@ -1,6 +1,6 @@
 import json
 import nlp2
-from base_codec.academicodec import BaseCodec
+from AudCodec.base_codec.academicodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/academicodec_hifi_16k_320d_large_uni.py → AudCodec/codec/academicodec_hifi_16k_320d_large_uni.py

@@ -1,6 +1,6 @@
 import json
 import nlp2
-from base_codec.academicodec import BaseCodec
+from AudCodec.base_codec.academicodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/academicodec_hifi_24k_320d.py → AudCodec/codec/academicodec_hifi_24k_320d.py

@@ -1,6 +1,6 @@
 import json
 import nlp2
-from base_codec.academicodec import BaseCodec
+from AudCodec.base_codec.academicodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/audiodec_24k_320d.py → AudCodec/codec/audiodec_24k_320d.py

@@ -1,4 +1,4 @@
-from base_codec.audiodec import BaseCodec
+from AudCodec.base_codec.audiodec import BaseCodec
 import nlp2
 
 

codec/dac_16k.py → AudCodec/codec/dac_16k.py

@@ -1,4 +1,4 @@
-from base_codec.descript_audio_codec import BaseCodec
+from AudCodec.base_codec.descript_audio_codec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/dac_24k.py → AudCodec/codec/dac_24k.py

@@ -1,4 +1,4 @@
-from base_codec.descript_audio_codec import BaseCodec
+from AudCodec.base_codec.descript_audio_codec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/dac_44k.py → AudCodec/codec/dac_44k.py

@@ -1,4 +1,4 @@
-from base_codec.descript_audio_codec import BaseCodec
+from AudCodec.base_codec.descript_audio_codec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/encodec_24k_12bps.py → AudCodec/codec/encodec_24k_12bps.py

@@ -1,4 +1,4 @@
-from base_codec.encodec import BaseCodec
+from AudCodec.base_codec.encodec import BaseCodec
 
 class Codec(BaseCodec):
     def config(self):

codec/encodec_24k_1_5bps.py → AudCodec/codec/encodec_24k_1_5bps.py

@@ -1,4 +1,4 @@
-from base_codec.encodec import BaseCodec
+from AudCodec.base_codec.encodec import BaseCodec
 
 class Codec(BaseCodec):
     def config(self):

codec/encodec_24k_24bps.py → AudCodec/codec/encodec_24k_24bps.py

@@ -1,4 +1,4 @@
-from base_codec.encodec import BaseCodec
+from AudCodec.base_codec.encodec import BaseCodec
 
 class Codec(BaseCodec):
     def config(self):

codec/encodec_24k_3bps.py → AudCodec/codec/encodec_24k_3bps.py

@@ -1,4 +1,4 @@
-from base_codec.encodec import BaseCodec
+from AudCodec.base_codec.encodec import BaseCodec
 
 class Codec(BaseCodec):
     def config(self):

codec/encodec_24k_6bps.py → AudCodec/codec/encodec_24k_6bps.py

@@ -1,4 +1,4 @@
-from base_codec.encodec import BaseCodec
+from AudCodec.base_codec.encodec import BaseCodec
 
 class Codec(BaseCodec):
     def config(self):

codec/funcodec_en_libritts_16k_gr1nq32ds320.py → AudCodec/codec/funcodec_en_libritts_16k_gr1nq32ds320.py

@@ -1,6 +1,6 @@
 import nlp2
 
-from base_codec.funcodec import BaseCodec
+from AudCodec.base_codec.funcodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/funcodec_en_libritts_16k_gr8nq32ds320.py → AudCodec/codec/funcodec_en_libritts_16k_gr8nq32ds320.py

@@ -1,6 +1,6 @@
 import nlp2
 
-from base_codec.funcodec import BaseCodec
+from AudCodec.base_codec.funcodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/funcodec_en_libritts_16k_nq32ds320.py → AudCodec/codec/funcodec_en_libritts_16k_nq32ds320.py

@@ -1,6 +1,6 @@
 import nlp2
 
-from base_codec.funcodec import BaseCodec
+from AudCodec.base_codec.funcodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/funcodec_en_libritts_16k_nq32ds640.py → AudCodec/codec/funcodec_en_libritts_16k_nq32ds640.py

@@ -1,6 +1,6 @@
 import nlp2
 
-from base_codec.funcodec import BaseCodec
+from AudCodec.base_codec.funcodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/funcodec_zh_en_16k_nq32ds320.py → AudCodec/codec/funcodec_zh_en_16k_nq32ds320.py

@@ -1,6 +1,6 @@
 import nlp2
 
-from base_codec.funcodec import BaseCodec
+from AudCodec.base_codec.funcodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/funcodec_zh_en_16k_nq32ds640.py → AudCodec/codec/funcodec_zh_en_16k_nq32ds640.py

@@ -1,6 +1,6 @@
 import nlp2
 
-from base_codec.funcodec import BaseCodec
+from AudCodec.base_codec.funcodec import BaseCodec
 
 
 class Codec(BaseCodec):

codec/speech_tokenizer_16k.py → AudCodec/codec/speech_tokenizer_16k.py

@@ -1,4 +1,4 @@
-from base_codec.speech_tokenizer import BaseCodec
+from AudCodec.base_codec.speech_tokenizer import BaseCodec
 import nlp2
 
 class Codec(BaseCodec):

dataset/__init__.py → AudCodec/dataset/__init__.py

@@ -1,4 +1,3 @@
-import dataset.general
 def load_dataset(dataset_name):
     module = __import__(f"dataset.{dataset_name}", fromlist=[dataset_name])
     return module.load_data()

test/code_extract_unit.py → AudCodec/test/code_extract_unit.py

@@ -1,7 +1,7 @@
 import torch
 import torchaudio
 
-from codec import list_codec, load_codec
+from AudCodec.codec import list_codec, load_codec
 
 if __name__ == '__main__':
     for sample_file in ['sample1_16k.wav', 'sample2_22k.wav', 'sample3_48k.wav', 'sample4_16k.wav',

test/code_synth.py → AudCodec/test/code_synth.py

@@ -1,6 +1,6 @@
 import torchaudio
 
-from codec import list_codec, load_codec
+from AudCodec.codec import list_codec, load_codec
 
 if __name__ == '__main__':
     for sample_file in ['sample1_16k.wav', 'sample2_22k.wav', 'sample3_48k.wav', 'sample4_16k.wav',

test/synth_cli_check.py → AudCodec/test/synth_cli_check.py

@@ -3,7 +3,7 @@
 import torch
 
 from benchmarking import compute_metrics
-from codec import load_codec
+from AudCodec.codec import load_codec
 import torchaudio
 import numpy as np
 

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Contributing to Codec-SUPERB
+
+We welcome contributions to Codec-SUPERB in several areas: models, datasets, and metrics. Here's how you can contribute:
+
+## Contributing Models
+
+1. Fork the Codec-SUPERB repository.
+2. Add your model to the `models` directory. Please ensure your model adheres to the interface defined in `models/README.md`.
+3. Add tests for your model in the `tests` directory.
+4. Submit a pull request with your changes. Please include a detailed description of your model and how it improves Codec-SUPERB.
+
+## Contributing Datasets
+
+1. Fork the Codec-SUPERB repository.
+2. Add your dataset to the `datasets` directory. Please ensure your dataset adheres to the format defined in `datasets/README.md`.
+3. Add tests for your dataset in the `tests` directory.
+4. Submit a pull request with your changes. Please include a detailed description of your dataset and how it improves Codec-SUPERB.
+
+## Contributing Metrics
+
+1. Fork the Codec-SUPERB repository.
+2. Add your metric to the `metrics` directory. Please ensure your metric adheres to the interface defined in `metrics/README.md`.
+3. Add tests for your metric in the `tests` directory.
+4. Submit a pull request with your changes. Please include a detailed description of your metric and how it improves Codec-SUPERB.
+
+We look forward to your contributions!

README.md

@@ -1,36 +1,78 @@
-# Audio Codec Benchmark
+# Codec-SUPERB: Audio Codec Speech Processing Universal Performance Benchmark
 
-## Codec Collection:
+![Overview](AudCodec/img/Overview.png)
 
-- https://github.com/ZhangXInFD/SpeechTokenizer
-- https://github.com/descriptinc/descript-audio-codec
-- https://github.com/facebookresearch/encodec
-- https://github.com/yangdongchao/AcademiCodec
-- https://github.com/facebookresearch/AudioDec
-- https://github.com/alibaba-damo-academy/FunCodec
+Codec-SUPERB is a comprehensive benchmark designed to evaluate audio codec models across a variety of speech tasks. Our
+goal is to facilitate community collaboration and accelerate advancements in the field of speech processing by
+preserving and enhancing speech information quality.
+
+
+## Table of Contents
+
+- [Introduction](#introduction)
+- [Key Features](#key-features)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Benchmarking](#benchmarking)
+- [Contribution](#contribution)
+- [License](#license)
+
+## Introduction
+
+Codec-SUPERB sets a new benchmark in evaluating audio codec models, providing a rigorous and transparent framework for
+assessing performance across a range of speech processing tasks. Our goal is to foster innovation and set new standards
+in audio quality and processing efficiency.
+
+## Key Features
 
-## Criteria
+### Out-of-the-Box Codec Interface
+Codec-SUPERB offers an intuitive, out-of-the-box codec interface that allows for easy integration and testing of various
+codec models, facilitating quick iterations and experiments.
 
-### Waveform (Lower is better)
+### Multi-Perspective Leaderboard
+Codec-SUPERB's unique blend of multi-perspective evaluation and an online leaderboard drives innovation in audio codec research by providing a comprehensive assessment and fostering competitive transparency among developers.
 
-L1Loss in waveform
+### Standardized Environment
+We ensure a standardized testing environment to guarantee fair and consistent comparison across all models. This
+uniformity brings reliability to benchmark results, making them universally interpretable.
 
-### Mel Distance (Lower is better)
+### Unified Datasets
+We provide a collection of unified datasets, curated to test a wide range of speech processing scenarios. This ensures
+that models are evaluated under diverse conditions, reflecting real-world applications.
 
-The Mel Distance is the distance between the log mel spectrograms of the reconstructed and ground truth waveforms.
+## Installation
 
-### STFT Distance (Lower is better)
+```bash
+git clone https://github.com/voidful/Codec-SUPERB.git
+cd Codec-SUPERB
+pip install -r requirements.txt
+```
 
-This metric calculates the distance between the log magnitude spectrograms of the reconstructed and ground truth
-waveforms, using window lengths of [2048, 512], and is better at capturing fidelity in higher frequencies compared to
-the Mel Distance.
+## Usage
 
-### PESQ (Higher is better)
+Detailed instructions on how to use Codec-SUPERB, including preparing your codec model and executing benchmark tests,
+can be found in the `docs` directory.
 
-PESQ is an intrusive perceptual quality metric for automated assessment of the speech quality. We adopt ITU-T P.862.2 (wideband).
+## Benchmarking
 
-### STOI (Higher is better)
+Codec-SUPERB supports a comprehensive suite of speech tasks, from speech recognition to audio quality assessment, each
+designed to rigorously evaluate the capabilities of audio codec models.
 
-STOI is an intrusive perceptual quality metric that assesses audio quality based on the intelligibility of the
-reconstructed speech.
+## Contribution
 
+Contributions are highly encouraged, whether it's through adding new codec models, expanding the dataset collection, or
+enhancing the benchmarking framework. Please see `CONTRIBUTING.md` for more details.
+
+## License
+
+This project is licensed under the MIT License - see the `LICENSE` file for details.
+
+
+## Reference Audio Codec Repositories：
+
+- https://github.com/ZhangXInFD/SpeechTokenizer
+- https://github.com/descriptinc/descript-audio-codec
+- https://github.com/facebookresearch/encodec
+- https://github.com/yangdongchao/AcademiCodec
+- https://github.com/facebookresearch/AudioDec
+- https://github.com/alibaba-damo-academy/FunCodec

benchmarking.py

@@ -9,7 +9,7 @@
 from datasets import load_dataset, load_from_disk
 from collections import defaultdict
 from audiotools import AudioSignal
-from base_codec.general import pad_arrays_to_match
+from AudCodec.base_codec.general import pad_arrays_to_match
 from metrics import get_metrics
 import psutil
 from tqdm.contrib.concurrent import process_map

dataset_checker.py

@@ -2,7 +2,7 @@
 import itertools
 import numpy as np
 from datasets import load_dataset
-from codec import list_codec
+from AudCodec.codec import list_codec
 
 
 def load_datasets(dataset_name, splits):