Despite the recent progress in scaling multilingual machine translation (MT) to several under-resourced African languages, accurately measuring this progress remains challenging. Evaluation is often performed using n-gram matching metrics such as BLEU, which typically show a weaker correlation with human judgments. Learned metrics like COMET have a higher correlation; however, challenges such as the lack of evaluation data with human ratings for under-resourced languages, the complexity of annotation guidelines like Multidimensional Quality Metrics (MQM), and the limited language coverage of multilingual encoders have hampered their applicability to African languages. In this paper, we address these challenges by creating high-quality human evaluation data with simplified MQM guidelines for error detection and direct assessment (DA) scoring for 13 typologically diverse African languages. Furthermore, we develop AFRICOMET: COMET evaluation metrics for African languages by leveraging DA data from well-resourced languages and an African-centric multilingual encoder (AfroXLM-R) to create the state-of-the-art MT evaluation metrics for African languages with respect to Spearman-rank correlation with human judgments (+0.441).<\/p>\n","protected":false},"excerpt":{"rendered":"
Despite the recent progress in scaling multilingual machine translation (MT) to several under-resourced African languages, accurately measuring this progress remains challenging. Evaluation is often performed using n-gram matching metrics such as BLEU, which typically show a weaker correlation with human judgments. Learned metrics like COMET have a higher correlation; however, challenges such as the lack […]<\/p>\n","protected":false},"featured_media":0,"template":"","meta":{"msr-url-field":"","msr-podcast-episode":"","msrModifiedDate":"","msrModifiedDateEnabled":false,"ep_exclude_from_search":false,"_classifai_error":"","msr-author-ordering":null,"msr_publishername":"NAACL 2024","msr_publisher_other":"","msr_booktitle":"","msr_chapter":"","msr_edition":"","msr_editors":"","msr_how_published":"","msr_isbn":"","msr_issue":"","msr_journal":"","msr_number":"","msr_organization":"","msr_pages_string":"","msr_page_range_start":"","msr_page_range_end":"","msr_series":"","msr_volume":"","msr_copyright":"","msr_conference_name":"North American Chapter of the Association for Computational Linguistics","msr_doi":"","msr_arxiv_id":"","msr_s2_paper_id":"","msr_mag_id":"","msr_pubmed_id":"","msr_other_authors":"","msr_other_contributors":"","msr_speaker":"","msr_award":"","msr_affiliation":"","msr_institution":"","msr_host":"","msr_version":"","msr_duration":"","msr_original_fields_of_study":"","msr_release_tracker_id":"","msr_s2_match_type":"","msr_citation_count_updated":"","msr_published_date":"2024-3-13","msr_highlight_text":"","msr_notes":"","msr_longbiography":"","msr_publicationurl":"","msr_external_url":"","msr_secondary_video_url":"","msr_conference_url":"https:\/\/2024.naacl.org\/","msr_journal_url":"","msr_s2_pdf_url":"","msr_year":0,"msr_citation_count":0,"msr_influential_citations":0,"msr_reference_count":0,"msr_s2_match_confidence":0,"msr_microsoftintellectualproperty":true,"msr_s2_open_access":false,"msr_s2_author_ids":[],"msr_pub_ids":[],"msr_hide_image_in_river":0,"footnotes":""},"msr-research-highlight":[],"research-area":[13556,13545],"msr-publication-type":[193716],"msr-publisher":[],"msr-focus-area":[],"msr-locale":[268875],"msr-post-option":[],"msr-field-of-study":[246808],"msr-conference":[],"msr-journal":[],"msr-impact-theme":[],"msr-pillar":[],"class_list":["post-1014642","msr-research-item","type-msr-research-item","status-publish","hentry","msr-research-area-artificial-intelligence","msr-research-area-human-language-technologies","msr-locale-en_us","msr-field-of-study-natural-language-processing"],"msr_publishername":"NAACL 2024","msr_edition":"","msr_affiliation":"","msr_published_date":"2024-3-13","msr_host":"","msr_duration":"","msr_version":"","msr_speaker":"","msr_other_contributors":"","msr_booktitle":"","msr_pages_string":"","msr_chapter":"","msr_isbn":"","msr_journal":"","msr_volume":"","msr_number":"","msr_editors":"","msr_series":"","msr_issue":"","msr_organization":"","msr_how_published":"","msr_notes":"","msr_highlight_text":"","msr_release_tracker_id":"","msr_original_fields_of_study":"","msr_download_urls":"","msr_external_url":"","msr_secondary_video_url":"","msr_longbiography":"","msr_microsoftintellectualproperty":1,"msr_main_download":"","msr_publicationurl":"","msr_doi":"","msr_publication_uploader":[{"type":"url","viewUrl":"false","id":"false","title":"https:\/\/arxiv.org\/abs\/2311.09828","label_id":"243109","label":0}],"msr_related_uploader":"","msr_citation_count":0,"msr_citation_count_updated":"","msr_s2_paper_id":"","msr_influential_citations":0,"msr_reference_count":0,"msr_arxiv_id":"","msr_s2_author_ids":[],"msr_s2_open_access":false,"msr_s2_pdf_url":null,"msr_attachments":[],"msr-author-ordering":[{"type":"text","value":"Jiayi Wang","user_id":0,"rest_url":false},{"type":"text","value":"David Ifeoluwa Adelani","user_id":0,"rest_url":false},{"type":"text","value":"Sweta Agrawal","user_id":0,"rest_url":false},{"type":"text","value":"Marek Masiak","user_id":0,"rest_url":false},{"type":"text","value":"Ricardo Rei","user_id":0,"rest_url":false},{"type":"text","value":"Eleftheria Briakou","user_id":0,"rest_url":false},{"type":"text","value":"Marine Carpuat","user_id":0,"rest_url":false},{"type":"text","value":"Xuanli He","user_id":0,"rest_url":false},{"type":"text","value":"Sofia Bourhim","user_id":0,"rest_url":false},{"type":"text","value":"Andiswa Bukula","user_id":0,"rest_url":false},{"type":"text","value":"Muhidin A. Mohamed","user_id":0,"rest_url":false},{"type":"text","value":"Temitayo Olatoye","user_id":0,"rest_url":false},{"type":"text","value":"Hamam Mokayed","user_id":0,"rest_url":false},{"type":"text","value":"Christine Mwase","user_id":0,"rest_url":false},{"type":"text","value":"Wangui Kimotho","user_id":0,"rest_url":false},{"type":"text","value":"Foutse Yuehgoh","user_id":0,"rest_url":false},{"type":"text","value":"Anuoluwapo Aremu","user_id":0,"rest_url":false},{"type":"text","value":"Jessica Ojo","user_id":0,"rest_url":false},{"type":"text","value":"Shamsuddeen Hassan Muhammad","user_id":0,"rest_url":false},{"type":"text","value":"Salomey Osei","user_id":0,"rest_url":false},{"type":"text","value":"Abdul-Hakeem Omotayo","user_id":0,"rest_url":false},{"type":"text","value":"Chiamaka Ijeoma Chukwuneke","user_id":0,"rest_url":false},{"type":"text","value":"Perez Ogayo","user_id":0,"rest_url":false},{"type":"text","value":"Oumaima Hourrane","user_id":0,"rest_url":false},{"type":"text","value":"Salma EL ANIGRI","user_id":0,"rest_url":false},{"type":"text","value":"Lolwethu Ndolela","user_id":0,"rest_url":false},{"type":"text","value":"Thabiso Mangwana","user_id":0,"rest_url":false},{"type":"text","value":"Shafie Abdi Mohamed","user_id":0,"rest_url":false},{"type":"text","value":"Hassan Ayinde","user_id":0,"rest_url":false},{"type":"text","value":"Oluwabusayo Olufunke Awoyomi","user_id":0,"rest_url":false},{"type":"text","value":"Lama Alkhaled","user_id":0,"rest_url":false},{"type":"text","value":"Sana Sabah al-Azzawi","user_id":0,"rest_url":false},{"type":"text","value":"Naome A Etori","user_id":0,"rest_url":false},{"type":"user_nicename","value":"Millicent Ochieng","user_id":40678,"rest_url":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/microsoft-research\/v1\/researchers?person=Millicent Ochieng"},{"type":"text","value":"Clemencia Siro","user_id":0,"rest_url":false},{"type":"text","value":"Njoroge Kiragu","user_id":0,"rest_url":false},{"type":"text","value":"Eric Muchiri","user_id":0,"rest_url":false},{"type":"text","value":"Wangari Kimotho","user_id":0,"rest_url":false},{"type":"text","value":"Toadoum Sari Sakayo","user_id":0,"rest_url":false},{"type":"text","value":"Lyse Naomi Momo Wamba","user_id":0,"rest_url":false},{"type":"text","value":"Daud Abolade","user_id":0,"rest_url":false},{"type":"text","value":"Simbiat Ajao","user_id":0,"rest_url":false},{"type":"text","value":"Tosin Adewumi","user_id":0,"rest_url":false},{"type":"text","value":"Iyanuoluwa Shode","user_id":0,"rest_url":false},{"type":"text","value":"Ricky Sambo Macharm","user_id":0,"rest_url":false},{"type":"text","value":"Ruqayya Nasir Iro","user_id":0,"rest_url":false},{"type":"text","value":"Saheed Salahudeen Abdullahi","user_id":0,"rest_url":false},{"type":"text","value":"Stephen Edward Moore","user_id":0,"rest_url":false},{"type":"text","value":"Bernard Opoku","user_id":0,"rest_url":false},{"type":"text","value":"Zainab Akinjobi","user_id":0,"rest_url":false},{"type":"text","value":"Abeeb Afolabi","user_id":0,"rest_url":false},{"type":"text","value":"Nnaemeka Casmir Obiefuna","user_id":0,"rest_url":false},{"type":"text","value":"Onyekachi Ogbu","user_id":0,"rest_url":false},{"type":"text","value":"Sam Brian Ochieng'","user_id":0,"rest_url":false},{"type":"text","value":"Verrah Akinyi Otiende","user_id":0,"rest_url":false},{"type":"text","value":"CHINEDU EMMANUEL MBONU","user_id":0,"rest_url":false},{"type":"text","value":"Yao Lu","user_id":0,"rest_url":false},{"type":"text","value":"Pontus Stenetorp","user_id":0,"rest_url":false}],"msr_impact_theme":[],"msr_research_lab":[199561,1021599],"msr_event":[],"msr_group":[],"msr_project":[1068003],"publication":[],"video":[],"msr-tool":[],"msr_publication_type":"inproceedings","related_content":{"projects":[{"ID":1068003,"post_title":"African Languages Research","post_name":"african-languages-research","post_type":"msr-project","post_date":"2024-08-14 06:39:23","post_modified":"2024-08-14 09:29:33","post_status":"publish","permalink":"https:\/\/www.microsoft.com\/en-us\/research\/project\/african-languages-research\/","post_excerpt":"Bridging technological and community divides by enriching Generative AI research with the rich diversity of African languages. Join us in shaping a more inclusive digital future Microsoft Research Africa is pioneering advancements in Generative AI through our African Languages Research initiative, focusing on advancing Large Language Models (LLMs) for African languages. Our goal is to enhance language accessibility and ensure AI tools can effectively engage with Africa's linguistic diversity, supporting both language preservation and the…","_links":{"self":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-project\/1068003"}]}}]},"_links":{"self":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-research-item\/1014642","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-research-item"}],"about":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/types\/msr-research-item"}],"version-history":[{"count":1,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-research-item\/1014642\/revisions"}],"predecessor-version":[{"id":1014648,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-research-item\/1014642\/revisions\/1014648"}],"wp:attachment":[{"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/media?parent=1014642"}],"wp:term":[{"taxonomy":"msr-research-highlight","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-research-highlight?post=1014642"},{"taxonomy":"msr-research-area","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/research-area?post=1014642"},{"taxonomy":"msr-publication-type","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-publication-type?post=1014642"},{"taxonomy":"msr-publisher","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-publisher?post=1014642"},{"taxonomy":"msr-focus-area","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-focus-area?post=1014642"},{"taxonomy":"msr-locale","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-locale?post=1014642"},{"taxonomy":"msr-post-option","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-post-option?post=1014642"},{"taxonomy":"msr-field-of-study","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-field-of-study?post=1014642"},{"taxonomy":"msr-conference","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-conference?post=1014642"},{"taxonomy":"msr-journal","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-journal?post=1014642"},{"taxonomy":"msr-impact-theme","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-impact-theme?post=1014642"},{"taxonomy":"msr-pillar","embeddable":true,"href":"https:\/\/www.microsoft.com\/en-us\/research\/wp-json\/wp\/v2\/msr-pillar?post=1014642"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}