@inproceedings{f6138ce884404ac8bacd14c3a3e9a6ea,
title = "Unsupervised Audio-Caption Aligning Learns Correspondences between Individual Sound Events and Textual Phrases",
abstract = "We investigate unsupervised learning of correspondences between sound events and textual phrases through aligning audio clips with textual captions describing the content of a whole audio clip. We align originally unaligned and unannotated audio clips and their captions by scoring the similarities between audio frames and words, as encoded by modality-specific encoders and using a ranking-loss criterion to optimize the model. After training, we obtain clip-caption similarity by averaging frame-word similarities and estimate event-phrase correspondences by calculating frame-phrase similarities. We evaluate the method with two cross-modal tasks: audio-caption retrieval, and phrase-based sound event detection (SED). Experimental results show that the proposed method can globally associate audio clips with captions as well as locally learn correspondences between individual sound events and textual phrases in an unsupervised manner.",
keywords = "Cross-modal learning, audio, caption, sound event, unsupervised learning",
author = "Huang Xie and Okko R{\"a}s{\"a}nen and Konstantinos Drossos and Tuomas Virtanen",
note = "jufoid=57409; IEEE International Conference on Acoustics, Speech and Signal Processing ; Conference date: 23-05-2022 Through 27-05-2022",
year = "2022",
month = may,
doi = "10.1109/ICASSP43922.2022.9747336",
language = "English",
isbn = "978-1-6654-0541-6",
series = "Proceedings of the IEEE International Conference on Acoustics, Speech, and Signal Processing",
publisher = "IEEE",
pages = "8867--8871",
booktitle = "ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)",
address = "United States",
}