#4 lo->zh 翻译不达标

import sacrebleu


def read_sys(sys_dir, refs_dir):
    ru_sys, zh_sys, ru_refs, zh_refs = [], [], [], []
    with open(sys_dir, 'r', encoding='utf-8') as f1:
        for data in f1.read().split('\n'):
            if data:
                idx, ru_pred, zh_pred = data.split('\t')
                ru_sys.append(ru_pred.strip())
                zh_sys.append(zh_pred.strip())
            else:
                ru_sys.append(" ")
                zh_sys.append(" ")

    with open(refs_dir, 'r', encoding='utf-8') as f2:
        for data in f2.read().split('\n\n'):
            if data:
                zh_test, ru_test = data.split('\t')
                ru_refs.append(ru_test.strip())
                zh_refs.append(zh_test.strip())

    return ru_sys, zh_sys, ru_refs, zh_refs


def compute(sys_dir, refs_dir):
    ru_sys, zh_sys, ru_refs, zh_refs = read_sys(sys_dir, refs_dir)
    zh_refs = [zh_refs[:len(ru_sys)]]
    ru_refs = [ru_refs[:len(ru_sys)]]
    length = len(ru_sys)
    bleu = sacrebleu.corpus_bleu(zh_sys, zh_refs, tokenize='zh')
    # bleu = sacrebleu.corpus_bleu(ru_sys, ru_refs)
    print(bleu)


if __name__ == '__main__':
    sys_path = 'sys/zh-lo.pred'
    ref_path = 'refs/zh_lo.test'
    compute(sys_path, ref_path)

### 背景 使用 mPanGu-Alpha-53 的 ckpt 推理 lo->zh，结果与 leadboard 差距较大 ### 环境 硬件：RTX 3090 GPU 框架：mindspore 1.7, cuda 11.1 ### 测试脚本 ``` import sacrebleu def read_sys(sys_dir, refs_dir): ru_sys, zh_sys, ru_refs, zh_refs = [], [], [], [] with open(sys_dir, 'r', encoding='utf-8') as f1: for data in f1.read().split('\n'): if data: idx, ru_pred, zh_pred = data.split('\t') ru_sys.append(ru_pred.strip()) zh_sys.append(zh_pred.strip()) else: ru_sys.append(" ") zh_sys.append(" ") with open(refs_dir, 'r', encoding='utf-8') as f2: for data in f2.read().split('\n\n'): if data: zh_test, ru_test = data.split('\t') ru_refs.append(ru_test.strip()) zh_refs.append(zh_test.strip()) return ru_sys, zh_sys, ru_refs, zh_refs def compute(sys_dir, refs_dir): ru_sys, zh_sys, ru_refs, zh_refs = read_sys(sys_dir, refs_dir) zh_refs = [zh_refs[:len(ru_sys)]] ru_refs = [ru_refs[:len(ru_sys)]] length = len(ru_sys) bleu = sacrebleu.corpus_bleu(zh_sys, zh_refs, tokenize='zh') # bleu = sacrebleu.corpus_bleu(ru_sys, ru_refs) print(bleu) if __name__ == '__main__': sys_path = 'sys/zh-lo.pred' ref_path = 'refs/zh_lo.test' compute(sys_path, ref_path) ```