PrimeQA: The Prime Repository for QA
  • Installation
  • Development
  • Testing
  • API
  • Tutorials
  • GitHub
  • Slack
  • data_models
    • data_collator
      • primeqa.mrc.data_models.data_collator.FiDDataCollator
    • eval_prediction_with_processing
      • primeqa.mrc.data_models.eval_prediction_with_processing.EvalPredictionWithProcessing
    • model_outputs
      • extractive
        • primeqa.mrc.data_models.model_outputs.extractive.ExtractiveQAModelOutput
        • primeqa.mrc.data_models.model_outputs.extractive.ExtractiveQAWithConfidenceModelOutput
    • subsample_type
      • primeqa.mrc.data_models.subsample_type.SubsampleType
    • target_type
      • primeqa.mrc.data_models.target_type.TargetType
  • metrics
    • mlqa
      • mlqa
        • primeqa.mrc.metrics.mlqa.mlqa.MLQA
      • mlqa_evaluation_v1
        • primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1.evaluate
        • primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1.exact_match_score
        • primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1.f1_score
        • primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1.metric_max_over_ground_truths
        • primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1.mixed_segmentation
        • primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1.normalize_answer
        • primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1.whitespace_tokenize
    • nq_f1
      • eval_utils
        • primeqa.mrc.metrics.nq_f1.eval_utils.gold_has_long_answer
        • primeqa.mrc.metrics.nq_f1.eval_utils.gold_has_short_answer
        • primeqa.mrc.metrics.nq_f1.eval_utils.is_null_span_list
        • primeqa.mrc.metrics.nq_f1.eval_utils.nonnull_span_equal
        • primeqa.mrc.metrics.nq_f1.eval_utils.parse_json_as_predictions
        • primeqa.mrc.metrics.nq_f1.eval_utils.parse_json_as_topk_predictions
        • primeqa.mrc.metrics.nq_f1.eval_utils.read_annotation
        • primeqa.mrc.metrics.nq_f1.eval_utils.read_annotation_from_one_split
        • primeqa.mrc.metrics.nq_f1.eval_utils.read_prediction_json_from_file
        • primeqa.mrc.metrics.nq_f1.eval_utils.span_set_equal
        • primeqa.mrc.metrics.nq_f1.eval_utils.NQLabel
        • primeqa.mrc.metrics.nq_f1.eval_utils.NQSpan
        • primeqa.mrc.metrics.nq_f1.eval_utils.InconsistentSpanError
      • nq_eval
        • primeqa.mrc.metrics.nq_f1.nq_eval.compute_f1
        • primeqa.mrc.metrics.nq_f1.nq_eval.compute_final_f1
        • primeqa.mrc.metrics.nq_f1.nq_eval.compute_optimal_metrics
        • primeqa.mrc.metrics.nq_f1.nq_eval.compute_pr_curves
        • primeqa.mrc.metrics.nq_f1.nq_eval.extract_metrics_at_optimal_threshold
        • primeqa.mrc.metrics.nq_f1.nq_eval.get_metrics_as_dict
        • primeqa.mrc.metrics.nq_f1.nq_eval.get_metrics_with_answer_stats
        • primeqa.mrc.metrics.nq_f1.nq_eval.load_gt_lookup_as_dict
        • primeqa.mrc.metrics.nq_f1.nq_eval.pretty_print
        • primeqa.mrc.metrics.nq_f1.nq_eval.print_r_at_p_table
        • primeqa.mrc.metrics.nq_f1.nq_eval.safe_divide
        • primeqa.mrc.metrics.nq_f1.nq_eval.score_answers
        • primeqa.mrc.metrics.nq_f1.nq_eval.score_long_answer
        • primeqa.mrc.metrics.nq_f1.nq_eval.score_short_answer
      • nq_f1
        • primeqa.mrc.metrics.nq_f1.nq_f1.NQF1
    • squad
      • evaluate
        • primeqa.mrc.metrics.squad.evaluate.evaluate
        • primeqa.mrc.metrics.squad.evaluate.exact_match_score
        • primeqa.mrc.metrics.squad.evaluate.f1_score
        • primeqa.mrc.metrics.squad.evaluate.metric_max_over_ground_truths
        • primeqa.mrc.metrics.squad.evaluate.normalize_answer
      • squad
        • primeqa.mrc.metrics.squad.squad.SQUAD
    • tydi_f1
      • eval_utils
        • primeqa.mrc.metrics.tydi_f1.eval_utils.compute_partial_match_scores
        • primeqa.mrc.metrics.tydi_f1.eval_utils.gold_has_minimal_answer
        • primeqa.mrc.metrics.tydi_f1.eval_utils.gold_has_passage_answer
        • primeqa.mrc.metrics.tydi_f1.eval_utils.nonnull_span_equal
        • primeqa.mrc.metrics.tydi_f1.eval_utils.safe_average
        • primeqa.mrc.metrics.tydi_f1.eval_utils.safe_divide
        • primeqa.mrc.metrics.tydi_f1.eval_utils.Span
        • primeqa.mrc.metrics.tydi_f1.eval_utils.TyDiLabel
      • tydi_eval
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.byte_slice
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.compute_final_f1
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.compute_macro_f1
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.compute_pr_curves
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.get_latex_str
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.get_metrics_with_answer_stats
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.pretty_print
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.print_r_at_p_table
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.score_answers
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.score_minimal_answer
        • primeqa.mrc.metrics.tydi_f1.tydi_eval.score_passage_answer
      • tydi_f1
        • primeqa.mrc.metrics.tydi_f1.tydi_f1.TyDiF1
  • models
    • fid_task_model
      • primeqa.mrc.models.fid_task_model.EncoderWrapper
      • primeqa.mrc.models.fid_task_model.FiDModelForDownstreamTasks
    • heads
      • abstract
        • primeqa.mrc.models.heads.abstract.AbstractTaskHead
      • extractive
        • primeqa.mrc.models.heads.extractive.ExtractiveQAHead
        • primeqa.mrc.models.heads.extractive.ExtractiveQAWithConfidenceHead
      • generative
    • task_model
      • primeqa.mrc.models.task_model.ModelForDownstreamTasks
  • processors
    • postprocessors
      • abstract
        • primeqa.mrc.processors.postprocessors.abstract.AbstractPostProcessor
      • eli5_fid
        • primeqa.mrc.processors.postprocessors.eli5_fid.ELI5FiDPostProcessor
      • extractive
        • primeqa.mrc.processors.postprocessors.extractive.ExtractivePostProcessor
      • natural_questions
        • primeqa.mrc.processors.postprocessors.natural_questions.NaturalQuestionsPostProcessor
      • scorers
        • primeqa.mrc.processors.postprocessors.scorers.compute_score_diff_between_span_and_cls
        • primeqa.mrc.processors.postprocessors.scorers.compute_short_answer_type_weighted_score_diff_between_span_and_cls
        • primeqa.mrc.processors.postprocessors.scorers.compute_weighted_sum_short_answer_type_score_diff_between_span_and_cls
        • primeqa.mrc.processors.postprocessors.scorers.initialize_scorer
        • primeqa.mrc.processors.postprocessors.scorers.SupportedSpanScorers
      • squad
        • primeqa.mrc.processors.postprocessors.squad.SQUADPostProcessor
    • preprocessors
      • abstract
        • primeqa.mrc.processors.preprocessors.abstract.AbstractPreProcessor
      • base
        • primeqa.mrc.processors.preprocessors.base.BasePreProcessor
      • eli5_fid
        • primeqa.mrc.processors.preprocessors.eli5_fid.ELI5FiDPreprocessor
      • mrqa
        • primeqa.mrc.processors.preprocessors.mrqa.MRQAPreprocessor
      • natural_questions
        • primeqa.mrc.processors.preprocessors.natural_questions.NaturalQuestionsPreProcessor
      • squad
        • primeqa.mrc.processors.preprocessors.squad.SQUADPreprocessor
      • tydiboolqa_bpes
        • primeqa.mrc.processors.preprocessors.tydiboolqa_bpes.TyDiBoolQAPreprocessor
      • tydiqa
        • primeqa.mrc.processors.preprocessors.tydiqa.TyDiQAPreprocessor
      • tydiqa_google
        • primeqa.mrc.processors.preprocessors.tydiqa_google.TyDiQAGooglePreprocessor
  • run_mrc_utils
    • primeqa.mrc.run_mrc_utils.get_raw_datasets
    • primeqa.mrc.run_mrc_utils.object_reference
    • primeqa.mrc.run_mrc_utils.process_raw_datasets
  • trainers
    • mrc
      • primeqa.mrc.trainers.mrc.MRCTrainer
    • mrc_mskd
      • primeqa.mrc.trainers.mrc_mskd.IndividualDomainBatchSampler
      • primeqa.mrc.trainers.mrc_mskd.MSKD_MRCTrainer
    • seq2seq_mrc
      • primeqa.mrc.trainers.seq2seq_mrc.MRCSeq2SeqTrainer
primeqa

mlqa#

primeqa.mrc.metrics.mlqa.mlqa

MLQA metric.

primeqa.mrc.metrics.mlqa.mlqa_evaluation_v1

Official evaluation script for the MLQA dataset.