We describe an automatic natural language processing (NLP)-based image captioning method to describe fetal ultrasound video content by modelling the vocabulary commonly used by sonographers and sonologists. The generated captions are similar to the words spoken by a sonographer when describing the scan experience in terms of visual content and performed scanning actions. Using full-length second-trimester fetal ultrasound videos and text derived from accompanying expert voice-over audio recordings, we train deep learning models consisting of convolutional neural networks and recurrent neural networks in merged configurations to generate captions for ultrasound video frames. We evaluate different model architectures using established general metrics (BLEU, ROUGE-L) and application-specific metrics. Results show that the proposed models can learn joint representations of image and text to generate relevant and descriptive captions for anatomies, such as the spine, the abdomen, the heart, and the head, in clinical fetal ultrasound scans.
@inproceedings{alsharid_captioning_2019,
title = {Captioning Ultrasound Images Automatically},
author = {Alsharid, Mohammad and Sharma, Harshita and Drukker, Lior and Chatelain, Pierre and Papageorghiou, Aris T. and Noble, J. Alison},
booktitle = {Medical Image Computing and Computer Assisted Intervention – MICCAI 2019},
doi = {10.1007/978-3-030-32251-9_37},
editor = {Shen, Dinggang and Liu, Tianming and Peters, Terry M. and Staib, Lawrence H. and Essert, Caroline and Zhou, Sean and Yap, Pew-Thian and Khan, Ali},
isbn = {978-3-030-32251-9},
keywords = {Deep learning, Fetal ultrasound, Image captioning, Image description, Natural language processing, Recurrent neural networks},
language = {en},
pages = {338--346},
address = {Cham},
publisher = {Springer International Publishing},
series = {Lecture Notes in Computer Science},
year = {2019}
}