diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/.gitignore b/paper/nature_neuroscience_revision/auxiliary_task_assessment/.gitignore new file mode 100644 index 0000000..41bb267 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/.gitignore @@ -0,0 +1,10 @@ +__pycache__/ +*.py[cod] +.ipynb_checkpoints/ +*.pt +*.pth +*.ckpt +*.npy +*.npz +embeddings/ +raw_data/ diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/PROVENANCE.md b/paper/nature_neuroscience_revision/auxiliary_task_assessment/PROVENANCE.md new file mode 100644 index 0000000..6d33d98 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/PROVENANCE.md @@ -0,0 +1,31 @@ +# Provenance + +## Recovered sources + +- Seed-specific CalMS21 decoder scripts and the all-seed plotting script were + recovered from `task_ablation_code_collection_dell.tar.gz`. +- Annotator/bout summary CSVs, final figures and the plotting notebook were + recovered from `auxiliary_task_final_collection.tar.gz`. + +## Transformations + +- The five seed-specific CalMS21 scripts are byte-for-byte copies of their + recovered sources. +- `plot_calms21_posthoc_allseeds.py` preserves the recovered aggregation and + plotting logic. Its single historical hard-coded absolute root was replaced + with `--root` and optional `--output` arguments. +- `plot_combined_calms21_mars1_annotator_bias.py` was extracted from the final + executed cell of `annotator-bias.ipynb`. Absolute paths were replaced with + package-relative paths, and notebook-only `display()` calls were changed to + terminal printing. The calculations, panel definitions and plotting + parameters were otherwise preserved. +- The original notebook is not included because it contains many duplicated + exploratory cells, embedded outputs and private absolute paths. +- No scientific result was synthesized, recalculated from prose or manually + entered into a new table. + +## Known limitation + +The original metric-generation script for the CalMS21/MARS1 annotator and bout +summary CSVs was not found in the supplied code collections. The preserved +summary tables and final plotting implementation are included transparently. diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/README.md b/paper/nature_neuroscience_revision/auxiliary_task_assessment/README.md new file mode 100644 index 0000000..da8b307 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/README.md @@ -0,0 +1,57 @@ +# Auxiliary-task assessment and annotator/bout analyses + +This directory contains code associated with two additions made during the +LISBET manuscript revision: + +1. **CalMS21 leave-one-task-out decoding**: five matched encoder seeds for the + full auxiliary-task model and the four leave-one-task-out variants, followed + by frozen-embedding k-nearest-neighbor decoding and all-seed aggregation. +2. **CalMS21 Task 2 and MARS1 annotator/bout characterization**: summary-level + agreement, label-prevalence and bout-duration tables together with the code + used to reproduce the final six-panel figure. + +## CalMS21 leave-one-task-out code + +The five `reviewer_2_2_calms21_pipeline_seed*.py` files are exact copies of the +seed-specific analysis scripts recovered from the Dell code archive. Each +script accepts `--lisbet-root`; use that argument rather than relying on its +historical default directory. + +Example for seed 0: + +```bash +python scripts/reviewer_2_2_calms21_pipeline_seed0.py all \ + --lisbet-root /path/to/Lisbet +``` + +The all-seed aggregation script was changed only to replace one private, +hard-coded source path with explicit command-line arguments: + +```bash +python scripts/plot_calms21_posthoc_allseeds.py \ + --root /path/containing/reviewer_2_2_calms21_posthoc_seed0_to_seed4 +``` + +The seed-level and aggregate CalMS21 result CSVs were not present in the +uploaded source archives, so they are not included here. No result table was +reconstructed from manuscript text. + +## Annotator and bout figure + +Run from any directory: + +```bash +python annotator_bout_metrics/scripts/plot_combined_calms21_mars1_annotator_bias.py +``` + +The script reads the included summary CSVs and writes the PNG, PDF, SVG and +combined summary CSV to `annotator_bout_metrics/figures/`. + +The available source material contained plotting code and derived summary +CSVs, but not the upstream script that originally calculated all agreement and +bout tables from raw annotations. Accordingly, this folder supports exact +figure reproduction from the preserved summary tables but does not claim to +recreate those tables from raw CalMS21 or MARS1 annotations. + +Raw annotations, pose data, embeddings, model weights and private machine paths +are intentionally excluded. diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_annotator_bias_summary_shared_legend.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_annotator_bias_summary_shared_legend.csv new file mode 100644 index 0000000..6391f86 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_annotator_bias_summary_shared_legend.csv @@ -0,0 +1,3 @@ +dataset,n_annotators,n_pairwise_comparisons,total_frames_annotated,accuracy_mean,balanced_accuracy_mean,macro_f1_mean,cohen_kappa_mean,mcc_mean +CalMS21 Task 2,5,10,1722361,0.587981380553702,0.36366716663646165,0.32895675619170517,0.10709882698855706,0.11124012746025029 +MARS1,8,28,1818304,0.906520650902092,0.8035393198111029,0.7709196531952476,0.7987126001605034,0.8032185150286512 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.pdf b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.pdf new file mode 100644 index 0000000..995090b Binary files /dev/null and b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.pdf differ diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.png b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.png new file mode 100644 index 0000000..3239768 Binary files /dev/null and b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.png differ diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.svg b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.svg new file mode 100644 index 0000000..c5626a9 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.svg @@ -0,0 +1,2888 @@ + + + + + + + + 2026-07-15T22:36:43.176567 + image/svg+xml + + + Matplotlib v3.10.8, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.0 + + + + + + + + + + + + + 0.2 + + + + + + + + + + + + + 0.4 + + + + + + + + + + + + + 0.6 + + + + + + + + + + + + + 0.8 + + + + + + + + + + + + + 1.0 + + + + Agreement score + + + + + + + + + + + + + + A1–A3 + + + + + + + + + + A2–A4 + + + + + + + + + + A2–A3 + + + + + + + + + + A1–A2 + + + + + + + + + + A3–A5 + + + + + + + + + + A4–A5 + + + + + + + + + + A1–A4 + + + + + + + + + + A2–A5 + + + + + + + + + + A1–A5 + + + + + + + + + + A3–A4 + + + + Annotator pair + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + a + + + CalMS21 Task 2 agreement + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.0 + + + + + + + + + + + + + 0.2 + + + + + + + + + + + + + 0.4 + + + + + + + + + + + + + 0.6 + + + + + + + + + + + + + 0.8 + + + + Agreement score + + + + + + + + + + + A5–A7 + + + + + + + + + + A5–A8 + + + + + + + + + + A5–A6 + + + + + + + + + + A6–A7 + + + + + + + + + + A4–A5 + + + + + + + + + + A2–A5 + + + + + + + + + + A6–A8 + + + + + + + + + + A3–A5 + + + + + + + + + + A1–A5 + + + + + + + + + + A1–A7 + + + + + + + + + + A2–A7 + + + + + + + + + + A1–A8 + + + + + + + + + + A4–A7 + + + + + + + + + + A3–A7 + + + + + + + + + + A4–A8 + + + + + + + + + + A7–A8 + + + + + + + + + + A2–A8 + + + + + + + + + + A1–A6 + + + + + + + + + + A3–A6 + + + + + + + + + + A3–A8 + + + + + + + + + + A2–A6 + + + + + + + + + + A4–A6 + + + + + + + + + + A1–A3 + + + + + + + + + + A1–A4 + + + + + + + + + + A2–A3 + + + + + + + + + + A1–A2 + + + + + + + + + + A2–A4 + + + + + + + + + + A3–A4 + + + + Annotator pair + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + b + + + MARS1 agreement + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A1 + + + + + + + + + + A2 + + + + + + + + + + A3 + + + + + + + + + + A4 + + + + + + + + + + A5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Annotator + + + + + + + + + + + other + + + + + + + + + + investigation + + + + + + + + + + mount + + + + + + + + + + attack + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Behavior + + + + + + + + + + + + + + + + c + + + CalMS21 label distribution + + + + + + + + + + + + + + + + + + A1 + + + + + + + + + + A2 + + + + + + + + + + A3 + + + + + + + + + + A4 + + + + + + + + + + A5 + + + + + + + + + + A6 + + + + + + + + + + A7 + + + + + + + + + + A8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Annotator + + + + + + + + + + + sniff + + + + + + + + + + mount + + + + + + + + + + aggressivemount + + + + + + + + + + attack + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Behavior + + + + + + + + + + + + + + + + d + + + MARS1 positive-frame fraction + + + + + + + + + + + + + + + + + + A1 + + + + + + + + + + A2 + + + + + + + + + + A3 + + + + + + + + + + A4 + + + + + + + + + + A5 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Annotator + + + + + + + + + + + other + + + + + + + + + + mount + + + + + + + + + + attack + + + + + + + + + + investigation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Behavior + + + + + + + + + + + + + + + + e + + + CalMS21 bout duration + + + + + + + + + + + + + + + + + + A1 + + + + + + + + + + A2 + + + + + + + + + + A3 + + + + + + + + + + A4 + + + + + + + + + + A5 + + + + + + + + + + A6 + + + + + + + + + + A7 + + + + + + + + + + A8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Annotator + + + + + + + + + + + sniff + + + + + + + + + + mount + + + + + + + + + + aggressivemount + + + + + + + + + + attack + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Behavior + + + + + + + + + + + + + + + + f + + + MARS1 bout duration + + + + + + + + + + + + + + + + + + A1 + + + + + + + + + + A2 + + + + + + + + + + A3 + + + + + + + + + + A4 + + + + + + + + + + A5 + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 200 + + + + + + + + + + + + + 300 + + + + + + + + + + + + + 400 + + + + frames + ×10³ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + coverage + + + + + + + + + + + + + + + A1 + + + + + + + + + + A2 + + + + + + + + + + A3 + + + + + + + + + + A4 + + + + + + + + + + A5 + + + + + + + + + + A6 + + + + + + + + + + A7 + + + + + + + + + + A8 + + + + + + + + + + + + + + + 0 + + + + + + + + + + + + + 100 + + + + + + + + + + + + + 200 + + + + frames + ×10³ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + coverage + + + + + + + + + + + + + + + + + + + + 0.0 + + + + + + + + + + 0.1 + + + + + + + + + + 0.2 + + + + + + + + + + 0.3 + + + + + + + + + + 0.4 + + + + + + + + + + 0.5 + + + + + + + + + + 0.6 + + + + Fraction + + + + + + + + + + + + + + + + + + + + + + 0.00 + + + + + + + + + + 0.05 + + + + + + + + + + 0.10 + + + + + + + + + + 0.15 + + + + + + + + + + 0.20 + + + + Positive fraction + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 2 + + + + + + + + + + 4 + + + + + + + + + + 6 + + + + + + + + + + 8 + + + + + + + + + + 10 + + + + + + + + + + 12 + + + + + + + + + + 14 + + + + Mean duration (s) + + + + + + + + + + + + + + + + + + + + + + 0 + + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 3 + + + + + + + + + + 4 + + + + + + + + + + 5 + + + + + + + + + + 6 + + + + + + + + + + 7 + + + + Mean duration (s) + + + + + + + + + + + + + + + Cohen's κ + + + + + + + + Macro-F1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_annotator_file_summary.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_annotator_file_summary.csv new file mode 100644 index 0000000..4b7f013 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_annotator_file_summary.csv @@ -0,0 +1,6 @@ +annotator,n_files,total_frames,mean_frames_per_file +annotator1,19,425932,22417.473684210527 +annotator2,12,286542,23878.5 +annotator3,11,191140,17376.363636363636 +annotator4,19,362357,19071.42105263158 +annotator5,26,456390,17553.46153846154 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_by_annotator.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_by_annotator.csv new file mode 100644 index 0000000..6d1267d --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_by_annotator.csv @@ -0,0 +1,21 @@ +annotator,behavior,n_bouts,mean_bout_duration_s,median_bout_duration_s,mean_bout_rate_per_min +annotator1,attack,231,2.3428101293350574,1.5166666666666666,2.3482358768510565 +annotator1,investigation,1603,2.0041389483416436,0.95,7.03240299337685 +annotator1,mount,234,7.584541226747109,3.8666666666666667,1.3979955228037393 +annotator1,other,1390,6.380059594332485,3.15,6.1864732967350164 +annotator2,attack,230,1.6317873251169976,1.1666666666666667,2.7294798818436825 +annotator2,investigation,913,2.644359236517415,1.2166666666666668,5.797733335629368 +annotator2,mount,102,1.5936928104575163,1.25,1.1285370788325968 +annotator2,other,985,6.917779833641067,2.5166666666666666,6.233999878448947 +annotator3,attack,191,2.285598296440402,1.5333333333333334,3.5022042751365445 +annotator3,investigation,301,3.49542912401197,3.0,3.062829967091486 +annotator3,mount,101,8.723133195307108,1.2666666666666666,2.0489168810328118 +annotator3,other,478,9.562765920675396,3.466666666666667,4.786278051772098 +annotator4,attack,164,7.075741758241759,5.533333333333333,1.5063684525469951 +annotator4,investigation,432,4.238955965357694,2.4166666666666665,2.1854846022483922 +annotator4,mount,99,15.041912369886226,3.9,1.0569075344992482 +annotator4,other,627,14.203506868556866,5.733333333333333,3.1570792607557725 +annotator5,attack,469,1.1718586772528983,0.7333333333333334,3.319703732296314 +annotator5,investigation,1714,2.093718584797418,0.9333333333333333,6.726368710561998 +annotator5,mount,31,1.2140972222222222,1.2416666666666667,0.46327377273484516 +annotator5,other,1794,6.551177837700304,2.666666666666667,7.018491349308483 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_long.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_long.csv new file mode 100644 index 0000000..81ae15b --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_long.csv @@ -0,0 +1,263 @@ +behavior,n_bouts,mean_bout_duration_s,median_bout_duration_s,total_duration_s,bout_rate_per_min,sex_group,split,mouse,record_key,annotator,file +investigation,127,2.094488188976378,0.7666666666666667,266.0,8.436358268443001,Female_likely,test,013,Female_likely__test__mouse013,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse013/Manual_Scoring_annotator1.csv +mount,20,4.49,1.1666666666666665,89.8,1.3285603572351183,Female_likely,test,013,Female_likely__test__mouse013,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse013/Manual_Scoring_annotator1.csv +other,110,4.9766666666666675,3.583333333333333,547.4333333333334,7.30708196479315,Female_likely,test,013,Female_likely__test__mouse013,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse013/Manual_Scoring_annotator1.csv +investigation,99,1.7454545454545456,0.8,172.8,6.103784894673745,Female_likely,test,015,Female_likely__test__mouse015,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse015/Manual_Scoring_annotator1.csv +mount,40,8.449166666666667,5.283333333333333,337.9666666666667,2.4661757150196952,Female_likely,test,015,Female_likely__test__mouse015,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse015/Manual_Scoring_annotator1.csv +other,76,6.084210526315789,4.016666666666667,462.4,4.685733858537421,Female_likely,test,015,Female_likely__test__mouse015,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse015/Manual_Scoring_annotator1.csv +investigation,75,3.132888888888889,1.3333333333333333,234.96666666666667,5.0423934560938255,Female_likely,test,016,Female_likely__test__mouse016,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse016/Manual_Scoring_annotator1.csv +mount,21,4.2555555555555555,3.8666666666666667,89.36666666666666,1.4118701677062713,Female_likely,test,016,Female_likely__test__mouse016,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse016/Manual_Scoring_annotator1.csv +other,59,9.628813559322035,5.766666666666667,568.1,3.966682852127143,Female_likely,test,016,Female_likely__test__mouse016,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse016/Manual_Scoring_annotator1.csv +investigation,90,1.7133333333333332,0.8,154.2,6.1056043417630885,Female_likely,test,017,Female_likely__test__mouse017,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse017/Manual_Scoring_annotator1.csv +mount,34,6.326470588235294,2.3666666666666667,215.1,2.306561640221611,Female_likely,test,017,Female_likely__test__mouse017,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse017/Manual_Scoring_annotator1.csv +other,67,7.688557213930348,3.466666666666667,515.1333333333333,4.54528323220141,Female_likely,test,017,Female_likely__test__mouse017,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse017/Manual_Scoring_annotator1.csv +investigation,93,1.6383512544802867,0.9666666666666667,152.36666666666667,6.253268584236085,Female_likely,test,018,Female_likely__test__mouse018,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse018/Manual_Scoring_annotator1.csv +mount,26,9.17051282051282,7.4,238.43333333333334,1.7482256257004107,Female_likely,test,018,Female_likely__test__mouse018,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse018/Manual_Scoring_annotator1.csv +other,71,7.063849765258216,2.2333333333333334,501.53333333333336,4.774000747104968,Female_likely,test,018,Female_likely__test__mouse018,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse018/Manual_Scoring_annotator1.csv +investigation,83,1.7012048192771083,1.1666666666666667,141.2,5.695985359716345,Female_likely,test,019,Female_likely__test__mouse019,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse019/Manual_Scoring_annotator1.csv +mount,24,9.963888888888889,6.833333333333334,239.13333333333333,1.6470319112432805,Female_likely,test,019,Female_likely__test__mouse019,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse019/Manual_Scoring_annotator1.csv +other,67,7.372636815920398,5.033333333333333,493.96666666666664,4.597964085554158,Female_likely,test,019,Female_likely__test__mouse019,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse019/Manual_Scoring_annotator1.csv +investigation,67,2.2318407960199003,1.0666666666666667,149.53333333333333,4.391682750081934,Female_likely,train,003,Female_likely__train__mouse003,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse003/Manual_Scoring_annotator1.csv +mount,24,10.191666666666666,2.4166666666666665,244.6,1.5731400895815884,Female_likely,train,003,Female_likely__train__mouse003,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse003/Manual_Scoring_annotator1.csv +other,55,9.476969696969697,4.8,521.2333333333333,3.60511270529114,Female_likely,train,003,Female_likely__train__mouse003,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse003/Manual_Scoring_annotator1.csv +investigation,95,1.8305263157894738,0.9,173.9,5.470599526521211,Female_likely,train,004,Female_likely__train__mouse004,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse004/Manual_Scoring_annotator1.csv +mount,16,20.275000000000002,22.083333333333336,324.40000000000003,0.9213641307825197,Female_likely,train,004,Female_likely__train__mouse004,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse004/Manual_Scoring_annotator1.csv +other,88,6.177651515151515,3.4833333333333334,543.6333333333333,5.067502719303858,Female_likely,train,004,Female_likely__train__mouse004,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse004/Manual_Scoring_annotator1.csv +investigation,83,1.3751004016064259,0.9666666666666667,114.13333333333334,5.294117647058823,Female_likely,train,014,Female_likely__train__mouse014,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse014/Manual_Scoring_annotator1.csv +mount,26,7.007692307692308,3.9166666666666665,182.2,1.6583982990786676,Female_likely,train,014,Female_likely__train__mouse014,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse014/Manual_Scoring_annotator1.csv +other,67,9.616915422885572,7.5,644.3333333333334,4.2735648476257975,Female_likely,train,014,Female_likely__train__mouse014,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse014/Manual_Scoring_annotator1.csv +investigation,40,1.9125,1.2,76.5,5.301133853629804,Female_likely,test,012,Female_likely__test__mouse012,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_test_mouse012/Manual_Scoring_annotator2.csv +mount,6,3.6777777777777776,3.4833333333333334,22.066666666666666,0.7951700780444706,Female_likely,test,012,Female_likely__test__mouse012,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_test_mouse012/Manual_Scoring_annotator2.csv +other,42,8.432539682539684,3.3166666666666664,354.1666666666667,5.566190546311295,Female_likely,test,012,Female_likely__test__mouse012,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_test_mouse012/Manual_Scoring_annotator2.csv +investigation,57,2.610526315789474,1.3333333333333333,148.8,5.442682085830991,Female_likely,train,002,Female_likely__train__mouse002,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse002/Manual_Scoring_annotator2.csv +mount,10,1.6866666666666668,2.033333333333333,16.866666666666667,0.9548565062861387,Female_likely,train,002,Female_likely__train__mouse002,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse002/Manual_Scoring_annotator2.csv +other,58,7.977586206896552,5.183333333333334,462.7,5.538167736459605,Female_likely,train,002,Female_likely__train__mouse002,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse002/Manual_Scoring_annotator2.csv +investigation,69,1.921256038647343,1.2333333333333334,132.56666666666666,7.530010913059296,Female_likely,train,003,Female_likely__train__mouse003,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse003/Manual_Scoring_annotator2.csv +mount,29,2.4298850574712643,1.6333333333333333,70.46666666666667,3.1647871953437616,Female_likely,train,003,Female_likely__train__mouse003,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse003/Manual_Scoring_annotator2.csv +other,76,4.5627192982456135,1.5333333333333334,346.76666666666665,8.293925063659513,Female_likely,train,003,Female_likely__train__mouse003,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse003/Manual_Scoring_annotator2.csv +investigation,81,2.863374485596708,1.3,231.93333333333334,6.094298612272195,Female_likely,train,011,Female_likely__train__mouse011,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse011/Manual_Scoring_annotator2.csv +mount,29,1.3701149425287356,1.3333333333333333,39.733333333333334,2.1819093797023905,Female_likely,train,011,Female_likely__train__mouse011,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse011/Manual_Scoring_annotator2.csv +other,86,6.1139534883720925,2.8833333333333333,525.8,6.470489884634676,Female_likely,train,011,Female_likely__train__mouse011,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse011/Manual_Scoring_annotator2.csv +investigation,49,2.1006802721088436,1.1333333333333333,102.93333333333334,4.5682912933133055,Female_likely,test,008,Female_likely__test__mouse008,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator3.csv +mount,30,2.3144444444444447,1.1,69.43333333333334,2.7969130367224317,Female_likely,test,008,Female_likely__test__mouse008,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator3.csv +other,49,9.616326530612245,8.833333333333334,471.2,4.5682912933133055,Female_likely,test,008,Female_likely__test__mouse008,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator3.csv +investigation,45,2.4896296296296296,1.6,112.03333333333333,4.195369555083648,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator4.csv +mount,16,3.5833333333333335,2.1666666666666665,57.333333333333336,1.4916869529186303,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator4.csv +other,46,10.308695652173913,8.066666666666666,474.2,4.288599989641062,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator4.csv +investigation,46,4.780434782608696,3.0,219.9,4.216315307057745,Female_likely,test,010,Female_likely__test__mouse010,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator3.csv +mount,6,28.616666666666664,5.566666666666666,171.7,0.5499541704857929,Female_likely,test,010,Female_likely__test__mouse010,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator3.csv +other,47,5.595744680851064,3.466666666666667,263.0,4.307974335472044,Female_likely,test,010,Female_likely__test__mouse010,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator3.csv +investigation,38,6.010526315789474,3.666666666666667,228.4,3.4830430797433545,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator4.csv +mount,5,33.49333333333333,8.133333333333333,167.46666666666667,0.458295142071494,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator4.csv +other,38,6.8087719298245615,5.233333333333333,258.73333333333335,3.4830430797433545,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator4.csv +investigation,37,4.003603603603604,1.7333333333333334,148.13333333333333,2.7416433393709863,Female_likely,train,001,Female_likely__train__mouse001,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator3.csv +mount,23,4.179710144927536,3.3666666666666667,96.13333333333333,1.7042647785279104,Female_likely,train,001,Female_likely__train__mouse001,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator3.csv +other,42,13.463492063492064,9.149999999999999,565.4666666666667,3.1121356825292277,Female_likely,train,001,Female_likely__train__mouse001,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator3.csv +investigation,31,4.596774193548387,1.9,142.5,2.2970525275810965,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator4.csv +mount,17,5.354901960784313,3.9,91.03333333333333,1.2596739667380206,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator4.csv +other,41,14.053658536585367,9.9,576.2,3.0380372138975793,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator4.csv +investigation,17,4.519607843137256,3.2,76.83333333333334,4.607739798223159,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator3.csv +mount,15,1.728888888888889,1.2666666666666666,25.933333333333334,4.065652763138082,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator3.csv +other,25,4.744,3.033333333333333,118.6,6.776087938563469,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator3.csv +investigation,10,9.389999999999999,4.75,93.89999999999999,2.7104351754253875,Female_likely,train,005,Female_likely__train__mouse005,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator4.csv +mount,5,2.506666666666667,2.4,12.533333333333333,1.3552175877126937,Female_likely,train,005,Female_likely__train__mouse005,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator4.csv +other,15,7.662222222222223,4.233333333333333,114.93333333333334,4.065652763138082,Female_likely,train,005,Female_likely__train__mouse005,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator4.csv +investigation,35,4.058095238095238,3.066666666666667,142.03333333333333,3.3787407486860457,Female_likely,train,007,Female_likely__train__mouse007,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator3.csv +mount,10,21.66,21.9,216.6,0.9653544996245844,Female_likely,train,007,Female_likely__train__mouse007,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator3.csv +other,38,6.918421052631579,5.516666666666667,262.9,3.668347098573421,Female_likely,train,007,Female_likely__train__mouse007,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator3.csv +investigation,28,5.251190476190476,4.1,147.03333333333333,2.7029925989488364,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator4.csv +mount,9,24.003703703703703,22.0,216.03333333333333,0.868819049662126,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator4.csv +other,33,7.8323232323232315,5.733333333333333,258.46666666666664,3.185669848761129,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator4.csv +investigation,45,2.4896296296296296,1.6,112.03333333333333,4.195369555083648,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse008/Manual_Scoring_annotator4.csv +mount,16,3.5833333333333335,2.1666666666666665,57.333333333333336,1.4916869529186303,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse008/Manual_Scoring_annotator4.csv +other,46,10.308695652173913,8.066666666666666,474.2,4.288599989641062,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse008/Manual_Scoring_annotator4.csv +investigation,38,6.010526315789474,3.666666666666667,228.4,3.4830430797433545,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse010/Manual_Scoring_annotator4.csv +mount,5,33.49333333333333,8.133333333333333,167.46666666666667,0.458295142071494,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse010/Manual_Scoring_annotator4.csv +other,38,6.8087719298245615,5.233333333333333,258.73333333333335,3.4830430797433545,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse010/Manual_Scoring_annotator4.csv +investigation,31,4.596774193548387,1.9,142.5,2.2970525275810965,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse001/Manual_Scoring_annotator4.csv +mount,17,5.354901960784313,3.9,91.03333333333333,1.2596739667380206,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse001/Manual_Scoring_annotator4.csv +other,41,14.053658536585367,9.9,576.2,3.0380372138975793,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse001/Manual_Scoring_annotator4.csv +investigation,17,4.519607843137256,3.2,76.83333333333334,4.607739798223159,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse005/Manual_Scoring_annotator3.csv +mount,15,1.728888888888889,1.2666666666666666,25.933333333333334,4.065652763138082,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse005/Manual_Scoring_annotator3.csv +other,25,4.744,3.033333333333333,118.6,6.776087938563469,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse005/Manual_Scoring_annotator3.csv +investigation,28,5.251190476190476,4.1,147.03333333333333,2.7029925989488364,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse007/Manual_Scoring_annotator4.csv +mount,9,24.003703703703703,22.0,216.03333333333333,0.868819049662126,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse007/Manual_Scoring_annotator4.csv +other,33,7.8323232323232315,5.733333333333333,258.46666666666664,3.185669848761129,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse007/Manual_Scoring_annotator4.csv +investigation,59,3.749717514124294,2.0,221.23333333333335,5.7915689589354855,Female_likely,test,021,Female_likely__test__mouse021,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_test_mouse021/Manual_Scoring_annotator5.csv +mount,6,2.2555555555555555,1.5,13.533333333333333,0.5889731144680155,Female_likely,test,021,Female_likely__test__mouse021,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_test_mouse021/Manual_Scoring_annotator5.csv +other,56,6.722619047619048,4.916666666666666,376.4666666666667,5.497082401701477,Female_likely,test,021,Female_likely__test__mouse021,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_test_mouse021/Manual_Scoring_annotator5.csv +investigation,66,3.1297979797979796,1.2666666666666666,206.56666666666666,6.619121907733453,Female_likely,train,002,Female_likely__train__mouse002,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse002/Manual_Scoring_annotator5.csv +mount,12,1.4527777777777777,1.15,17.433333333333334,1.2034767104969915,Female_likely,train,002,Female_likely__train__mouse002,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse002/Manual_Scoring_annotator5.csv +other,56,6.683333333333333,4.833333333333334,374.26666666666665,5.61622464898596,Female_likely,train,002,Female_likely__train__mouse002,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse002/Manual_Scoring_annotator5.csv +investigation,34,4.245098039215685,1.8666666666666667,144.33333333333331,6.334747955698167,Female_likely,train,024,Female_likely__train__mouse024,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse024/Manual_Scoring_annotator5.csv +mount,3,1.2444444444444445,1.3333333333333333,3.7333333333333334,0.5589483490321913,Female_likely,train,024,Female_likely__train__mouse024,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse024/Manual_Scoring_annotator5.csv +other,32,5.436458333333333,4.366666666666667,173.96666666666667,5.96211572301004,Female_likely,train,024,Female_likely__train__mouse024,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse024/Manual_Scoring_annotator5.csv +investigation,29,2.2816091954022992,1.6333333333333333,66.16666666666667,4.959148774463234,Female_likely,train,025,Female_likely__train__mouse025,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse025/Manual_Scoring_annotator5.csv +mount,5,2.0933333333333333,1.9,10.466666666666667,0.8550256507695231,Female_likely,train,025,Female_likely__train__mouse025,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse025/Manual_Scoring_annotator5.csv +other,27,10.15679012345679,6.3,274.23333333333335,4.617138514155425,Female_likely,train,025,Female_likely__train__mouse025,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse025/Manual_Scoring_annotator5.csv +attack,34,1.9431372549019608,1.4833333333333334,66.06666666666666,3.1750972762645913,Male_likely,test,005,Male_likely__test__mouse005,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse005/Manual_Scoring_annotator1.csv +investigation,87,2.011877394636015,0.8666666666666667,175.03333333333333,8.124513618677042,Male_likely,test,005,Male_likely__test__mouse005,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse005/Manual_Scoring_annotator1.csv +other,66,6.081818181818181,2.6333333333333333,401.4,6.163424124513618,Male_likely,test,005,Male_likely__test__mouse005,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse005/Manual_Scoring_annotator1.csv +attack,20,2.128333333333333,1.2833333333333332,42.56666666666666,2.0573779860555494,Male_likely,test,006,Male_likely__test__mouse006,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse006/Manual_Scoring_annotator1.csv +investigation,93,1.5129032258064514,0.7333333333333333,140.7,9.566807635158305,Male_likely,test,006,Male_likely__test__mouse006,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse006/Manual_Scoring_annotator1.csv +other,78,5.128205128205129,3.15,400.0,8.023774145616642,Male_likely,test,006,Male_likely__test__mouse006,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse006/Manual_Scoring_annotator1.csv +attack,17,2.7333333333333334,2.2333333333333334,46.46666666666667,1.7030276046304542,Male_likely,test,007,Male_likely__test__mouse007,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse007/Manual_Scoring_annotator1.csv +investigation,72,1.7259259259259259,0.95,124.26666666666667,7.212822796081924,Male_likely,test,007,Male_likely__test__mouse007,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse007/Manual_Scoring_annotator1.csv +other,65,6.587692307692308,2.466666666666667,428.2,6.511576135351737,Male_likely,test,007,Male_likely__test__mouse007,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse007/Manual_Scoring_annotator1.csv +attack,20,2.46,2.25,49.2,2.073613271124935,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +investigation,85,2.2737254901960786,1.1,193.26666666666668,8.812856402280973,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +mount,1,2.1,2.1,2.1,0.10368066355624675,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +other,81,4.125102880658436,1.9333333333333333,334.1333333333333,8.398133748055987,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +attack,12,3.3277777777777775,3.216666666666667,39.93333333333333,1.2776529042943334,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +investigation,83,3.4614457831325303,1.5333333333333334,287.3,8.837099254702473,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +mount,2,1.2,1.2,2.4,0.21294215071572226,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +other,73,3.2041095890410958,1.7333333333333334,233.9,7.772388501123862,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +attack,18,2.4444444444444446,1.5499999999999998,44.0,1.8179777802715744,Male_likely,test,010,Male_likely__test__mouse010,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse010/Manual_Scoring_annotator1.csv +investigation,52,1.0108974358974359,0.55,52.56666666666666,5.251935809673437,Male_likely,test,010,Male_likely__test__mouse010,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse010/Manual_Scoring_annotator1.csv +other,54,9.212962962962964,4.05,497.5,5.453933340814723,Male_likely,test,010,Male_likely__test__mouse010,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse010/Manual_Scoring_annotator1.csv +attack,29,1.4781609195402299,0.9333333333333333,42.86666666666667,3.002933900937698,Male_likely,test,011,Male_likely__test__mouse011,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse011/Manual_Scoring_annotator1.csv +investigation,87,1.8988505747126436,0.8666666666666667,165.2,9.008801702813093,Male_likely,test,011,Male_likely__test__mouse011,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse011/Manual_Scoring_annotator1.csv +other,97,3.8285223367697596,2.1333333333333333,371.3666666666667,10.044296151412299,Male_likely,test,011,Male_likely__test__mouse011,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse011/Manual_Scoring_annotator1.csv +attack,7,3.5,2.7,24.5,0.7260992335619202,Male_likely,train,001,Male_likely__train__mouse001,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse001/Manual_Scoring_annotator1.csv +investigation,75,3.1466666666666665,1.6666666666666667,236.0,7.779634645306288,Male_likely,train,001,Male_likely__train__mouse001,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse001/Manual_Scoring_annotator1.csv +other,73,4.355251141552511,1.9333333333333333,317.93333333333334,7.572177721431454,Male_likely,train,001,Male_likely__train__mouse001,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse001/Manual_Scoring_annotator1.csv +attack,38,1.6008771929824561,1.1,60.833333333333336,3.927648578811369,Male_likely,train,002,Male_likely__train__mouse002,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse002/Manual_Scoring_annotator1.csv +investigation,76,1.745175438596491,0.8500000000000001,132.63333333333333,7.855297157622738,Male_likely,train,002,Male_likely__train__mouse002,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse002/Manual_Scoring_annotator1.csv +other,78,4.961965811965811,2.4000000000000004,387.0333333333333,8.062015503875969,Male_likely,train,002,Male_likely__train__mouse002,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse002/Manual_Scoring_annotator1.csv +attack,36,1.8120370370370371,1.2833333333333332,65.23333333333333,3.7209302325581395,Male_likely,train,012,Male_likely__train__mouse012,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse012/Manual_Scoring_annotator1.csv +investigation,81,1.8279835390946502,1.0333333333333334,148.06666666666666,8.372093023255813,Male_likely,train,012,Male_likely__train__mouse012,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse012/Manual_Scoring_annotator1.csv +other,65,5.64923076923077,1.3666666666666667,367.20000000000005,6.718346253229973,Male_likely,train,012,Male_likely__train__mouse012,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse012/Manual_Scoring_annotator1.csv +attack,40,1.4158333333333333,0.7333333333333334,56.63333333333333,2.665482007996446,Male_likely,test,006,Male_likely__test__mouse006,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse006/Manual_Scoring_annotator2.csv +investigation,29,0.6448275862068965,0.5,18.7,1.9324744557974234,Male_likely,test,006,Male_likely__test__mouse006,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse006/Manual_Scoring_annotator2.csv +other,63,13.096296296296297,2.2666666666666666,825.0666666666667,4.198134162594402,Male_likely,test,006,Male_likely__test__mouse006,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse006/Manual_Scoring_annotator2.csv +attack,16,1.1770833333333333,0.7833333333333334,18.833333333333332,0.9823986901350796,Male_likely,test,008,Male_likely__test__mouse008,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse008/Manual_Scoring_annotator2.csv +investigation,109,3.686238532110092,1.4333333333333333,401.8,6.692591076545231,Male_likely,test,008,Male_likely__test__mouse008,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse008/Manual_Scoring_annotator2.csv +other,116,4.797988505747127,2.0166666666666666,556.5666666666667,7.122390503479328,Male_likely,test,008,Male_likely__test__mouse008,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse008/Manual_Scoring_annotator2.csv +attack,31,2.052688172043011,1.3,63.63333333333333,2.042235479266552,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +investigation,87,0.7555555555555555,0.36666666666666664,65.73333333333333,5.731435054715807,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +mount,5,0.38,0.4666666666666667,1.9,0.3293928192365406,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +other,94,8.292553191489361,4.5,779.5,6.192585001646965,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +attack,73,1.5360730593607306,1.1666666666666667,112.13333333333334,6.085869112130053,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +investigation,93,0.9336917562724014,0.6333333333333333,86.83333333333333,7.75323051271363,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +mount,5,0.9933333333333334,0.9333333333333333,4.966666666666667,0.4168403501458941,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +other,92,5.606159420289855,1.1,515.7666666666667,7.669862442684451,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +attack,38,1.8017543859649123,1.5666666666666669,68.46666666666667,2.4931656642974303,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +investigation,92,1.6677536231884058,1.0666666666666667,153.43333333333334,6.036085292509568,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +mount,1,1.0,1.0,1.0,0.06560962274466922,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +other,89,7.770786516853933,2.7666666666666666,691.6,5.83925642427556,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +attack,11,1.9303030303030304,1.4,21.233333333333334,1.10398661834402,Male_likely,test,003,Male_likely__test__mouse003,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator3.csv +investigation,42,4.563492063492063,3.0666666666666664,191.66666666666666,4.215221633677167,Male_likely,test,003,Male_likely__test__mouse003,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator3.csv +other,48,8.019444444444444,5.966666666666667,384.93333333333334,4.817396152773905,Male_likely,test,003,Male_likely__test__mouse003,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator3.csv +attack,4,5.2,3.533333333333333,20.8,0.40144967939782544,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator4.csv +investigation,43,5.242635658914729,3.2,225.43333333333334,4.315584053526623,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator4.csv +other,45,7.8133333333333335,6.033333333333333,351.6,4.516308893225536,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator4.csv +attack,40,2.853333333333333,2.1166666666666667,114.13333333333333,3.564532897668202,Male_likely,test,009,Male_likely__test__mouse009,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator3.csv +investigation,10,2.263333333333333,2.333333333333333,22.633333333333333,0.8911332244170505,Male_likely,test,009,Male_likely__test__mouse009,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator3.csv +other,47,11.415602836879431,3.3666666666666667,536.5333333333333,4.188326154760137,Male_likely,test,009,Male_likely__test__mouse009,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator3.csv +attack,21,7.480952380952381,5.533333333333333,157.1,1.8713797712758058,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator4.csv +investigation,3,1.511111111111111,0.9333333333333333,4.533333333333333,0.2673399673251151,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator4.csv +other,24,21.319444444444443,8.683333333333334,511.66666666666663,2.138719738600921,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator4.csv +attack,19,2.501754385964912,1.5333333333333334,47.53333333333333,1.8489484781315888,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +investigation,13,3.8846153846153846,3.2333333333333334,50.5,1.2650700113531923,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +mount,2,0.8333333333333333,0.8333333333333333,1.6666666666666665,0.19462615559279883,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +other,25,20.674666666666667,1.9,516.8666666666667,2.4328269449099853,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +attack,8,8.270833333333334,8.35,66.16666666666667,0.7785046223711953,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator4.csv +investigation,7,6.685714285714286,5.666666666666667,46.800000000000004,0.6811915445747959,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator4.csv +other,15,33.57333333333334,2.7666666666666666,503.6,1.4596961669459911,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator4.csv +attack,30,2.21,1.65,66.3,2.78508432616432,Male_likely,train,004,Male_likely__train__mouse004,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator3.csv +investigation,16,1.8895833333333334,1.65,30.233333333333334,1.4853783072876374,Male_likely,train,004,Male_likely__train__mouse004,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator3.csv +other,36,15.271296296296295,3.783333333333333,549.7666666666667,3.342101191397184,Male_likely,train,004,Male_likely__train__mouse004,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator3.csv +attack,23,3.9333333333333336,3.6,90.46666666666667,2.135231316725979,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator4.csv +investigation,8,2.4916666666666667,2.4166666666666665,19.933333333333334,0.7426891536438187,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator4.csv +other,29,18.479310344827585,4.266666666666667,535.9,2.692248181958843,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator4.csv +attack,91,1.9326007326007326,1.5,175.86666666666667,8.208469055374593,Male_likely,train,006,Male_likely__train__mouse006,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator3.csv +investigation,19,1.8666666666666667,1.8,35.46666666666667,1.7138561763968931,Male_likely,train,006,Male_likely__train__mouse006,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator3.csv +other,96,4.727430555555555,1.8166666666666667,453.8333333333333,8.659483838636934,Male_likely,train,006,Male_likely__train__mouse006,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator3.csv +attack,26,10.493589743589743,6.0,272.8333333333333,2.3452768729641695,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator4.csv +investigation,8,1.2958333333333334,1.1,10.366666666666667,0.7216236532197444,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator4.csv +other,35,10.913333333333332,5.1,381.96666666666664,3.157103482836382,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator4.csv +attack,4,5.2,3.533333333333333,20.8,0.40144967939782544,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse003/Manual_Scoring_annotator4.csv +investigation,43,5.242635658914729,3.2,225.43333333333334,4.315584053526623,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse003/Manual_Scoring_annotator4.csv +other,45,7.8133333333333335,6.033333333333333,351.6,4.516308893225536,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse003/Manual_Scoring_annotator4.csv +attack,21,7.480952380952381,5.533333333333333,157.1,1.8713797712758058,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse009/Manual_Scoring_annotator4.csv +investigation,3,1.511111111111111,0.9333333333333333,4.533333333333333,0.2673399673251151,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse009/Manual_Scoring_annotator4.csv +other,24,21.319444444444443,8.683333333333334,511.66666666666663,2.138719738600921,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse009/Manual_Scoring_annotator4.csv +attack,8,8.270833333333334,8.35,66.16666666666667,0.7785046223711953,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse002/Manual_Scoring_annotator4.csv +investigation,7,6.685714285714286,5.666666666666667,46.800000000000004,0.6811915445747959,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse002/Manual_Scoring_annotator4.csv +other,15,33.57333333333334,2.7666666666666666,503.6,1.4596961669459911,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse002/Manual_Scoring_annotator4.csv +attack,23,3.9333333333333336,3.6,90.46666666666667,2.135231316725979,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse004/Manual_Scoring_annotator4.csv +investigation,8,2.4916666666666667,2.4166666666666665,19.933333333333334,0.7426891536438187,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse004/Manual_Scoring_annotator4.csv +other,29,18.479310344827585,4.266666666666667,535.9,2.692248181958843,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse004/Manual_Scoring_annotator4.csv +attack,26,10.493589743589743,6.0,272.8333333333333,2.3452768729641695,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse006/Manual_Scoring_annotator4.csv +investigation,8,1.2958333333333334,1.1,10.366666666666667,0.7216236532197444,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse006/Manual_Scoring_annotator4.csv +other,35,10.913333333333332,5.1,381.96666666666664,3.157103482836382,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse006/Manual_Scoring_annotator4.csv +attack,20,1.1099999999999999,0.8666666666666667,22.2,1.86799501867995,Male_likely,test,003,Male_likely__test__mouse003,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse003/Manual_Scoring_annotator5.csv +investigation,81,1.1930041152263375,0.6333333333333333,96.63333333333334,7.565379825653798,Male_likely,test,003,Male_likely__test__mouse003,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse003/Manual_Scoring_annotator5.csv +other,82,6.384959349593497,1.9833333333333334,523.5666666666667,7.658779576587795,Male_likely,test,003,Male_likely__test__mouse003,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse003/Manual_Scoring_annotator5.csv +attack,16,0.6083333333333334,0.48333333333333334,9.733333333333334,1.5640273704789835,Male_likely,test,010,Male_likely__test__mouse010,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse010/Manual_Scoring_annotator5.csv +investigation,79,2.1118143459915615,1.0666666666666667,166.83333333333334,7.722385141739982,Male_likely,test,010,Male_likely__test__mouse010,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse010/Manual_Scoring_annotator5.csv +other,78,5.605555555555556,3.3833333333333333,437.23333333333335,7.624633431085045,Male_likely,test,010,Male_likely__test__mouse010,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse010/Manual_Scoring_annotator5.csv +attack,31,0.7258064516129032,0.5,22.5,2.9710878015015174,Male_likely,test,011,Male_likely__test__mouse011,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse011/Manual_Scoring_annotator5.csv +investigation,35,0.9495238095238095,0.5333333333333333,33.233333333333334,3.354453969437197,Male_likely,test,011,Male_likely__test__mouse011,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse011/Manual_Scoring_annotator5.csv +other,48,11.88125,2.216666666666667,570.3,4.600394015228156,Male_likely,test,011,Male_likely__test__mouse011,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse011/Manual_Scoring_annotator5.csv +attack,29,0.6114942528735633,0.4666666666666667,17.733333333333334,2.877618522601985,Male_likely,test,012,Male_likely__test__mouse012,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse012/Manual_Scoring_annotator5.csv +investigation,97,1.1670103092783506,0.6333333333333333,113.2,9.625137816979052,Male_likely,test,012,Male_likely__test__mouse012,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse012/Manual_Scoring_annotator5.csv +other,98,4.834013605442177,2.283333333333333,473.73333333333335,9.724366041896362,Male_likely,test,012,Male_likely__test__mouse012,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse012/Manual_Scoring_annotator5.csv +attack,44,0.9924242424242423,0.7,43.666666666666664,4.3892706716914205,Male_likely,test,014,Male_likely__test__mouse014,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse014/Manual_Scoring_annotator5.csv +investigation,51,0.9196078431372549,0.4,46.9,5.087563733096874,Male_likely,test,014,Male_likely__test__mouse014,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse014/Manual_Scoring_annotator5.csv +other,78,6.55,1.1,510.9,7.780979827089337,Male_likely,test,014,Male_likely__test__mouse014,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse014/Manual_Scoring_annotator5.csv +attack,75,0.9422222222222223,0.7666666666666667,70.66666666666667,7.312713287470884,Male_likely,test,016,Male_likely__test__mouse016,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse016/Manual_Scoring_annotator5.csv +investigation,29,0.8689655172413793,0.6,25.2,2.8275824711554085,Male_likely,test,016,Male_likely__test__mouse016,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse016/Manual_Scoring_annotator5.csv +other,73,7.116438356164384,0.7333333333333333,519.5,7.1177075998049935,Male_likely,test,016,Male_likely__test__mouse016,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse016/Manual_Scoring_annotator5.csv +attack,74,1.5945945945945945,0.8,118.0,7.524148449415352,Male_likely,test,019,Male_likely__test__mouse019,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse019/Manual_Scoring_annotator5.csv +investigation,70,1.5042857142857142,0.9,105.3,7.117437722419928,Male_likely,test,019,Male_likely__test__mouse019,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse019/Manual_Scoring_annotator5.csv +other,96,3.8208333333333333,1.0666666666666667,366.8,9.761057447890188,Male_likely,test,019,Male_likely__test__mouse019,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse019/Manual_Scoring_annotator5.csv +attack,66,0.6691919191919191,0.5166666666666666,44.166666666666664,6.618384401114206,Male_likely,test,022,Male_likely__test__mouse022,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse022/Manual_Scoring_annotator5.csv +investigation,47,0.827659574468085,0.5333333333333333,38.9,4.713091922005571,Male_likely,test,022,Male_likely__test__mouse022,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse022/Manual_Scoring_annotator5.csv +other,80,6.440833333333333,1.1333333333333333,515.2666666666667,8.022284122562674,Male_likely,test,022,Male_likely__test__mouse022,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse022/Manual_Scoring_annotator5.csv +attack,5,2.52,2.3,12.6,0.5027090431771212,Male_likely,test,023,Male_likely__test__mouse023,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse023/Manual_Scoring_annotator5.csv +investigation,63,3.0825396825396822,1.1,194.2,6.334133944031726,Male_likely,test,023,Male_likely__test__mouse023,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse023/Manual_Scoring_annotator5.csv +other,59,6.6096045197740105,4.6,389.96666666666664,5.9319667094900295,Male_likely,test,023,Male_likely__test__mouse023,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse023/Manual_Scoring_annotator5.csv +attack,7,2.219047619047619,1.3333333333333333,15.533333333333333,0.6790256520801897,Male_likely,test,026,Male_likely__test__mouse026,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse026/Manual_Scoring_annotator5.csv +investigation,69,1.817391304347826,0.7666666666666667,125.39999999999999,6.693252856219012,Male_likely,test,026,Male_likely__test__mouse026,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse026/Manual_Scoring_annotator5.csv +other,65,7.3476923076923075,5.733333333333333,477.59999999999997,6.305238197887475,Male_likely,test,026,Male_likely__test__mouse026,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse026/Manual_Scoring_annotator5.csv +attack,55,0.8696969696969697,0.6,47.833333333333336,5.4799070076386585,Male_likely,train,001,Male_likely__train__mouse001,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse001/Manual_Scoring_annotator5.csv +investigation,104,1.5432692307692308,0.8833333333333333,160.5,10.362005978080372,Male_likely,train,001,Male_likely__train__mouse001,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse001/Manual_Scoring_annotator5.csv +other,68,5.792156862745098,3.2333333333333334,393.8666666666667,6.7751577548987045,Male_likely,train,001,Male_likely__train__mouse001,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse001/Manual_Scoring_annotator5.csv +attack,15,1.4644444444444444,0.9666666666666667,21.966666666666665,1.4787228216222137,Male_likely,train,007,Male_likely__train__mouse007,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse007/Manual_Scoring_annotator5.csv +investigation,58,1.700574712643678,0.8500000000000001,98.63333333333333,5.717728243605893,Male_likely,train,007,Male_likely__train__mouse007,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse007/Manual_Scoring_annotator5.csv +other,54,9.037654320987654,4.433333333333334,488.0333333333333,5.323402157839969,Male_likely,train,007,Male_likely__train__mouse007,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse007/Manual_Scoring_annotator5.csv +attack,27,0.8654320987654321,0.7666666666666667,23.366666666666667,2.7115996205992303,Male_likely,train,008,Male_likely__train__mouse008,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse008/Manual_Scoring_annotator5.csv +investigation,54,2.5413580246913576,1.1333333333333333,137.23333333333332,5.423199241198461,Male_likely,train,008,Male_likely__train__mouse008,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse008/Manual_Scoring_annotator5.csv +other,51,8.565359477124185,1.2333333333333334,436.83333333333337,5.121910394465213,Male_likely,train,008,Male_likely__train__mouse008,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse008/Manual_Scoring_annotator5.csv +investigation,69,9.084057971014492,5.033333333333333,626.8,4.5806594379287455,Undetermined,test,005,Undetermined__test__mouse005,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse005/Manual_Scoring_annotator2.csv +other,69,4.0144927536231885,2.2333333333333334,277.0,4.5806594379287455,Undetermined,test,005,Undetermined__test__mouse005,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse005/Manual_Scoring_annotator2.csv +investigation,86,1.558139534883721,0.7833333333333334,134.0,5.825681168146922,Undetermined,test,010,Undetermined__test__mouse010,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse010/Manual_Scoring_annotator2.csv +other,87,8.640613026819924,6.833333333333333,751.7333333333333,5.8934216468463045,Undetermined,test,010,Undetermined__test__mouse010,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse010/Manual_Scoring_annotator2.csv +attack,32,1.8072916666666667,1.1666666666666667,57.833333333333336,2.107728337236534,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +investigation,101,4.094389438943895,1.8333333333333333,413.53333333333336,6.652517564402811,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +mount,17,1.211764705882353,1.1666666666666667,20.6,1.1197306791569088,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +other,113,3.7076696165191736,1.4666666666666666,418.96666666666664,7.442915690866512,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +investigation,69,2.8043478260869565,0.9333333333333333,193.5,6.986948694869487,Undetermined,test,004,Undetermined__test__mouse004,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse004/Manual_Scoring_annotator5.csv +other,70,5.7004761904761905,3.3166666666666664,399.0333333333333,7.088208820882088,Undetermined,test,004,Undetermined__test__mouse004,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse004/Manual_Scoring_annotator5.csv +investigation,75,2.481777777777778,1.1666666666666667,186.13333333333333,7.515448421755831,Undetermined,test,005,Undetermined__test__mouse005,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse005/Manual_Scoring_annotator5.csv +mount,1,0.13333333333333333,0.13333333333333333,0.13333333333333333,0.10020597895674442,Undetermined,test,005,Undetermined__test__mouse005,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse005/Manual_Scoring_annotator5.csv +other,75,5.5,2.1666666666666665,412.5,7.515448421755831,Undetermined,test,005,Undetermined__test__mouse005,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse005/Manual_Scoring_annotator5.csv +investigation,74,2.523873873873874,0.8500000000000001,186.76666666666668,7.3741903338315895,Undetermined,test,006,Undetermined__test__mouse006,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse006/Manual_Scoring_annotator5.csv +other,75,5.5377777777777775,2.7666666666666666,415.3333333333333,7.473841554559043,Undetermined,test,006,Undetermined__test__mouse006,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse006/Manual_Scoring_annotator5.csv +investigation,83,1.630120481927711,1.0333333333333334,135.3,8.243668266843239,Undetermined,test,009,Undetermined__test__mouse009,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse009/Manual_Scoring_annotator5.csv +other,83,5.648192771084338,2.5,468.8,8.243668266843239,Undetermined,test,009,Undetermined__test__mouse009,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse009/Manual_Scoring_annotator5.csv +investigation,73,1.3543378995433788,0.9333333333333333,98.86666666666666,7.455742169768497,Undetermined,test,013,Undetermined__test__mouse013,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse013/Manual_Scoring_annotator5.csv +other,74,6.602702702702703,4.466666666666667,488.6,7.557875624148887,Undetermined,test,013,Undetermined__test__mouse013,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse013/Manual_Scoring_annotator5.csv +attack,5,1.2133333333333334,0.6666666666666666,6.066666666666666,0.4986425840766801,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +investigation,65,2.276923076923077,1.2333333333333334,148.0,6.482353592996842,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +mount,2,1.4666666666666666,1.4666666666666666,2.933333333333333,0.19945703363067205,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +other,65,6.840512820512821,2.566666666666667,444.6333333333333,6.482353592996842,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +investigation,97,2.2628865979381443,1.1,219.5,9.744391115079807,Undetermined,test,017,Undetermined__test__mouse017,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse017/Manual_Scoring_annotator5.csv +mount,1,0.4,0.4,0.4,0.1004576403616475,Undetermined,test,017,Undetermined__test__mouse017,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse017/Manual_Scoring_annotator5.csv +other,97,3.890378006872852,2.5,377.3666666666667,9.744391115079807,Undetermined,test,017,Undetermined__test__mouse017,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse017/Manual_Scoring_annotator5.csv +investigation,56,3.3636904761904765,1.9666666666666668,188.36666666666667,5.580159433126661,Undetermined,test,018,Undetermined__test__mouse018,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse018/Manual_Scoring_annotator5.csv +mount,1,0.6666666666666666,0.6666666666666666,0.6666666666666666,0.09964570416297608,Undetermined,test,018,Undetermined__test__mouse018,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse018/Manual_Scoring_annotator5.csv +other,57,7.247368421052632,5.366666666666666,413.1,5.6798051372896365,Undetermined,test,018,Undetermined__test__mouse018,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse018/Manual_Scoring_annotator5.csv +investigation,97,2.1054982817869417,0.9,204.23333333333332,9.254743983886355,Undetermined,test,020,Undetermined__test__mouse020,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse020/Manual_Scoring_annotator5.csv +other,97,4.377663230240549,2.566666666666667,424.6333333333333,9.254743983886355,Undetermined,test,020,Undetermined__test__mouse020,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse020/Manual_Scoring_annotator5.csv diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_by_annotator.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_by_annotator.csv new file mode 100644 index 0000000..d770300 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_by_annotator.csv @@ -0,0 +1,21 @@ +annotator,behavior,n_frames,fraction_within_annotator +annotator1,attack,14450,0.03392560314792033 +annotator1,investigation,96424,0.22638355418235775 +annotator1,mount,58965,0.13843759097696345 +annotator1,other,256093,0.6012532516927585 +annotator2,attack,11326,0.03952649175339043 +annotator2,investigation,74719,0.26076107516524627 +annotator2,mount,5328,0.01859413279728626 +annotator2,other,195169,0.681118300284077 +annotator3,attack,12752,0.06671549649471592 +annotator3,investigation,32915,0.17220362038296536 +annotator3,mount,18222,0.09533326357643612 +annotator3,other,127251,0.6657476195458826 +annotator4,attack,36442,0.100569328038371 +annotator4,investigation,59039,0.16293048016182934 +annotator4,mount,32288,0.0891054954092235 +annotator4,other,234588,0.6473946963905761 +annotator5,attack,14281,0.0312912202283135 +annotator5,investigation,106234,0.23277021845351564 +annotator5,mount,1479,0.003240649444554 +annotator5,other,334396,0.7326979118736169 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_long.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_long.csv new file mode 100644 index 0000000..3f47b5e --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_long.csv @@ -0,0 +1,263 @@ +behavior,n_frames,fraction,sex_group,split,mouse,record_key,annotator,file +other,16423,0.6060818540797874,Female_likely,test,013,Female_likely__test__mouse013,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse013/Manual_Scoring_annotator1.csv +investigation,7980,0.29449754585378457,Female_likely,test,013,Female_likely__test__mouse013,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse013/Manual_Scoring_annotator1.csv +mount,2694,0.09942060006642801,Female_likely,test,013,Female_likely__test__mouse013,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse013/Manual_Scoring_annotator1.csv +other,13872,0.4751498544271279,Female_likely,test,015,Female_likely__test__mouse015,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse015/Manual_Scoring_annotator1.csv +mount,10139,0.34728549409145404,Female_likely,test,015,Female_likely__test__mouse015,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse015/Manual_Scoring_annotator1.csv +investigation,5184,0.17756465148141806,Female_likely,test,015,Female_likely__test__mouse015,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse015/Manual_Scoring_annotator1.csv +other,17043,0.6365741605348673,Female_likely,test,016,Female_likely__test__mouse016,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse016/Manual_Scoring_annotator1.csv +investigation,7049,0.26328764053337317,Female_likely,test,016,Female_likely__test__mouse016,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse016/Manual_Scoring_annotator1.csv +mount,2681,0.10013819893175961,Female_likely,test,016,Female_likely__test__mouse016,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse016/Manual_Scoring_annotator1.csv +other,15454,0.5824445030716466,Female_likely,test,017,Female_likely__test__mouse017,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse017/Manual_Scoring_annotator1.csv +mount,6453,0.24320657294689632,Female_likely,test,017,Female_likely__test__mouse017,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse017/Manual_Scoring_annotator1.csv +investigation,4626,0.17434892398145704,Female_likely,test,017,Female_likely__test__mouse017,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse017/Manual_Scoring_annotator1.csv +other,15046,0.5620470676129996,Female_likely,test,018,Female_likely__test__mouse018,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse018/Manual_Scoring_annotator1.csv +mount,7153,0.2672020918939111,Female_likely,test,018,Female_likely__test__mouse018,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse018/Manual_Scoring_annotator1.csv +investigation,4571,0.17075084049308928,Female_likely,test,018,Female_likely__test__mouse018,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse018/Manual_Scoring_annotator1.csv +other,14819,0.5649853215906058,Female_likely,test,019,Female_likely__test__mouse019,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse019/Manual_Scoring_annotator1.csv +mount,7174,0.27351404933470586,Female_likely,test,019,Female_likely__test__mouse019,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse019/Manual_Scoring_annotator1.csv +investigation,4236,0.16150062907468832,Female_likely,test,019,Female_likely__test__mouse019,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse019/Manual_Scoring_annotator1.csv +other,15637,0.569425731036743,Female_likely,train,003,Female_likely__train__mouse003,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse003/Manual_Scoring_annotator1.csv +mount,7338,0.26721532354976146,Female_likely,train,003,Female_likely__train__mouse003,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse003/Manual_Scoring_annotator1.csv +investigation,4486,0.1633589454134955,Female_likely,train,003,Female_likely__train__mouse003,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse003/Manual_Scoring_annotator1.csv +other,16309,0.5217544308656984,Female_likely,train,004,Female_likely__train__mouse004,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse004/Manual_Scoring_annotator1.csv +mount,9732,0.3113442958602598,Female_likely,train,004,Female_likely__train__mouse004,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse004/Manual_Scoring_annotator1.csv +investigation,5217,0.16690127327404183,Female_likely,train,004,Female_likely__train__mouse004,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse004/Manual_Scoring_annotator1.csv +other,19330,0.684975194897236,Female_likely,train,014,Female_likely__train__mouse014,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse014/Manual_Scoring_annotator1.csv +mount,5466,0.19369241672572643,Female_likely,train,014,Female_likely__train__mouse014,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse014/Manual_Scoring_annotator1.csv +investigation,3424,0.12133238837703757,Female_likely,train,014,Female_likely__train__mouse014,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse014/Manual_Scoring_annotator1.csv +other,10625,0.7822853777057871,Female_likely,test,012,Female_likely__test__mouse012,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_test_mouse012/Manual_Scoring_annotator2.csv +investigation,2295,0.16897364158445,Female_likely,test,012,Female_likely__test__mouse012,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_test_mouse012/Manual_Scoring_annotator2.csv +mount,662,0.04874098070976292,Female_likely,test,012,Female_likely__test__mouse012,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_test_mouse012/Manual_Scoring_annotator2.csv +other,13881,0.7363535090976606,Female_likely,train,002,Female_likely__train__mouse002,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse002/Manual_Scoring_annotator2.csv +investigation,4464,0.2368044135589624,Female_likely,train,002,Female_likely__train__mouse002,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse002/Manual_Scoring_annotator2.csv +mount,506,0.02684207734337701,Female_likely,train,002,Female_likely__train__mouse002,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse002/Manual_Scoring_annotator2.csv +other,10403,0.630714199102704,Female_likely,train,003,Female_likely__train__mouse003,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse003/Manual_Scoring_annotator2.csv +investigation,3977,0.2411179822965927,Female_likely,train,003,Female_likely__train__mouse003,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse003/Manual_Scoring_annotator2.csv +mount,2114,0.12816781860070328,Female_likely,train,003,Female_likely__train__mouse003,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse003/Manual_Scoring_annotator2.csv +other,15774,0.659337903360642,Female_likely,train,011,Female_likely__train__mouse011,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse011/Manual_Scoring_annotator2.csv +investigation,6958,0.29083765256646044,Female_likely,train,011,Female_likely__train__mouse011,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse011/Manual_Scoring_annotator2.csv +mount,1192,0.04982444407289751,Female_likely,train,011,Female_likely__train__mouse011,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse011/Manual_Scoring_annotator2.csv +other,14136,0.7321696793908945,Female_likely,test,008,Female_likely__test__mouse008,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator3.csv +investigation,3088,0.15994198995183095,Female_likely,test,008,Female_likely__test__mouse008,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator3.csv +mount,2083,0.10788833065727456,Female_likely,test,008,Female_likely__test__mouse008,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator3.csv +other,14226,0.7368312011187652,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator4.csv +investigation,3361,0.1740819391930388,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator4.csv +mount,1720,0.08908685968819599,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator4.csv +other,7890,0.4017720745493431,Female_likely,test,010,Female_likely__test__mouse010,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator3.csv +investigation,6597,0.33593033913840514,Female_likely,test,010,Female_likely__test__mouse010,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator3.csv +mount,5151,0.26229758631225175,Female_likely,test,010,Female_likely__test__mouse010,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator3.csv +other,7762,0.3952540991954374,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator4.csv +investigation,6852,0.3489153681637641,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator4.csv +mount,5024,0.25583053264079847,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator4.csv +other,16964,0.6983369010373786,Female_likely,train,001,Female_likely__train__mouse001,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator3.csv +investigation,4444,0.1829408858883583,Female_likely,train,001,Female_likely__train__mouse001,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator3.csv +mount,2884,0.11872221307426313,Female_likely,train,001,Female_likely__train__mouse001,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator3.csv +other,17286,0.7115922937592624,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator4.csv +investigation,4275,0.17598386300016466,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator4.csv +mount,2731,0.11242384324057303,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator4.csv +other,3558,0.5357626863424183,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator3.csv +investigation,2305,0.3470862821864177,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator3.csv +mount,778,0.11715103147116399,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator3.csv +other,3448,0.5191989158259298,Female_likely,train,005,Female_likely__train__mouse005,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator4.csv +investigation,2817,0.4241831049540732,Female_likely,train,005,Female_likely__train__mouse005,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator4.csv +mount,376,0.05661797921999699,Female_likely,train,005,Female_likely__train__mouse005,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator4.csv +other,7887,0.42298616325217203,Female_likely,train,007,Female_likely__train__mouse007,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator3.csv +mount,6498,0.34849297436447496,Female_likely,train,007,Female_likely__train__mouse007,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator3.csv +investigation,4261,0.228520862383353,Female_likely,train,007,Female_likely__train__mouse007,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator3.csv +other,7754,0.41585326611605705,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator4.csv +mount,6481,0.3475812506703851,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator4.csv +investigation,4411,0.23656548321355786,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator4.csv +other,14226,0.7368312011187652,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse008/Manual_Scoring_annotator4.csv +investigation,3361,0.1740819391930388,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse008/Manual_Scoring_annotator4.csv +mount,1720,0.08908685968819599,Female_likely,test,008,Female_likely__test__mouse008,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse008/Manual_Scoring_annotator4.csv +other,7762,0.3952540991954374,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse010/Manual_Scoring_annotator4.csv +investigation,6852,0.3489153681637641,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse010/Manual_Scoring_annotator4.csv +mount,5024,0.25583053264079847,Female_likely,test,010,Female_likely__test__mouse010,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse010/Manual_Scoring_annotator4.csv +other,17286,0.7115922937592624,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse001/Manual_Scoring_annotator4.csv +investigation,4275,0.17598386300016466,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse001/Manual_Scoring_annotator4.csv +mount,2731,0.11242384324057303,Female_likely,train,001,Female_likely__train__mouse001,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse001/Manual_Scoring_annotator4.csv +other,3558,0.5357626863424183,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse005/Manual_Scoring_annotator3.csv +investigation,2305,0.3470862821864177,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse005/Manual_Scoring_annotator3.csv +mount,778,0.11715103147116399,Female_likely,train,005,Female_likely__train__mouse005,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse005/Manual_Scoring_annotator3.csv +other,7754,0.41585326611605705,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse007/Manual_Scoring_annotator4.csv +mount,6481,0.3475812506703851,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse007/Manual_Scoring_annotator4.csv +investigation,4411,0.23656548321355786,Female_likely,train,007,Female_likely__train__mouse007,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse007/Manual_Scoring_annotator4.csv +other,11294,0.6159131810001636,Female_likely,test,021,Female_likely__test__mouse021,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_test_mouse021/Manual_Scoring_annotator5.csv +investigation,6637,0.3619457926596499,Female_likely,test,021,Female_likely__test__mouse021,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_test_mouse021/Manual_Scoring_annotator5.csv +mount,406,0.02214102634018651,Female_likely,test,021,Female_likely__test__mouse021,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_test_mouse021/Manual_Scoring_annotator5.csv +other,11228,0.625585023400936,Female_likely,train,002,Female_likely__train__mouse002,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse002/Manual_Scoring_annotator5.csv +investigation,6197,0.3452752395810118,Female_likely,train,002,Female_likely__train__mouse002,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse002/Manual_Scoring_annotator5.csv +mount,523,0.02913973701805215,Female_likely,train,002,Female_likely__train__mouse002,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse002/Manual_Scoring_annotator5.csv +other,5219,0.5402132284442605,Female_likely,train,024,Female_likely__train__mouse024,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse024/Manual_Scoring_annotator5.csv +investigation,4330,0.4481937687609978,Female_likely,train,024,Female_likely__train__mouse024,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse024/Manual_Scoring_annotator5.csv +mount,112,0.011593002794741744,Female_likely,train,024,Female_likely__train__mouse024,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse024/Manual_Scoring_annotator5.csv +other,8227,0.7815884476534296,Female_likely,train,025,Female_likely__train__mouse025,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse025/Manual_Scoring_annotator5.csv +investigation,1985,0.1885806574197226,Female_likely,train,025,Female_likely__train__mouse025,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse025/Manual_Scoring_annotator5.csv +mount,314,0.029830894926847807,Female_likely,train,025,Female_likely__train__mouse025,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse025/Manual_Scoring_annotator5.csv +other,12042,0.6247470817120623,Male_likely,test,005,Male_likely__test__mouse005,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse005/Manual_Scoring_annotator1.csv +investigation,5251,0.2724254215304799,Male_likely,test,005,Male_likely__test__mouse005,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse005/Manual_Scoring_annotator1.csv +attack,1982,0.10282749675745785,Male_likely,test,005,Male_likely__test__mouse005,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse005/Manual_Scoring_annotator1.csv +other,12000,0.6857926620185164,Male_likely,test,006,Male_likely__test__mouse006,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse006/Manual_Scoring_annotator1.csv +investigation,4221,0.24122756886501315,Male_likely,test,006,Male_likely__test__mouse006,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse006/Manual_Scoring_annotator1.csv +attack,1277,0.07297976911647046,Male_likely,test,006,Male_likely__test__mouse006,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse006/Manual_Scoring_annotator1.csv +other,12846,0.7149376669634907,Male_likely,test,007,Male_likely__test__mouse007,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse007/Manual_Scoring_annotator1.csv +investigation,3728,0.20747996438112198,Male_likely,test,007,Male_likely__test__mouse007,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse007/Manual_Scoring_annotator1.csv +attack,1394,0.07758236865538735,Male_likely,test,007,Male_likely__test__mouse007,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse007/Manual_Scoring_annotator1.csv +other,10024,0.5773860952710097,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +investigation,5798,0.3339669373883993,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +attack,1476,0.08501814411612234,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +mount,63,0.0036288232244686366,Male_likely,test,008,Male_likely__test__mouse008,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv +investigation,8619,0.5098189991718917,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +other,7017,0.41505974210339525,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +attack,1198,0.07086241571039867,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +mount,72,0.004258843014314445,Male_likely,test,009,Male_likely__test__mouse009,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv +other,14925,0.8374480978565818,Male_likely,test,010,Male_likely__test__mouse010,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse010/Manual_Scoring_annotator1.csv +investigation,1577,0.08848614072494669,Male_likely,test,010,Male_likely__test__mouse010,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse010/Manual_Scoring_annotator1.csv +attack,1320,0.07406576141847156,Male_likely,test,010,Male_likely__test__mouse010,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse010/Manual_Scoring_annotator1.csv +other,11141,0.640913536213542,Male_likely,test,011,Male_likely__test__mouse011,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse011/Manual_Scoring_annotator1.csv +investigation,4956,0.2851061381809814,Male_likely,test,011,Male_likely__test__mouse011,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse011/Manual_Scoring_annotator1.csv +attack,1286,0.07398032560547661,Male_likely,test,011,Male_likely__test__mouse011,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse011/Manual_Scoring_annotator1.csv +other,9538,0.5496455944217138,Male_likely,train,001,Male_likely__train__mouse001,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse001/Manual_Scoring_annotator1.csv +investigation,7080,0.4079986169538408,Male_likely,train,001,Male_likely__train__mouse001,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse001/Manual_Scoring_annotator1.csv +attack,735,0.04235578862444534,Male_likely,train,001,Male_likely__train__mouse001,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse001/Manual_Scoring_annotator1.csv +other,11611,0.6667240884295148,Male_likely,train,002,Male_likely__train__mouse002,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse002/Manual_Scoring_annotator1.csv +investigation,3979,0.22848119437266723,Male_likely,train,002,Male_likely__train__mouse002,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse002/Manual_Scoring_annotator1.csv +attack,1825,0.10479471719781798,Male_likely,train,002,Male_likely__train__mouse002,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse002/Manual_Scoring_annotator1.csv +other,11016,0.6325581395348837,Male_likely,train,012,Male_likely__train__mouse012,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse012/Manual_Scoring_annotator1.csv +investigation,4442,0.25506747057134654,Male_likely,train,012,Male_likely__train__mouse012,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse012/Manual_Scoring_annotator1.csv +attack,1957,0.11237438989376974,Male_likely,train,012,Male_likely__train__mouse012,annotator1,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse012/Manual_Scoring_annotator1.csv +other,24752,0.9163334814156671,Male_likely,test,006,Male_likely__test__mouse006,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse006/Manual_Scoring_annotator2.csv +attack,1699,0.06289797127202724,Male_likely,test,006,Male_likely__test__mouse006,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse006/Manual_Scoring_annotator2.csv +investigation,561,0.02076854731230564,Male_likely,test,006,Male_likely__test__mouse006,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse006/Manual_Scoring_annotator2.csv +other,16697,0.5695524628189385,Male_likely,test,008,Male_likely__test__mouse008,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse008/Manual_Scoring_annotator2.csv +investigation,12054,0.41117478510028654,Male_likely,test,008,Male_likely__test__mouse008,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse008/Manual_Scoring_annotator2.csv +attack,565,0.019272752080775002,Male_likely,test,008,Male_likely__test__mouse008,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse008/Manual_Scoring_annotator2.csv +other,23385,0.8558723419829448,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +investigation,1972,0.0721736266149398,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +attack,1909,0.06986787688028401,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +mount,57,0.002086154521831424,Male_likely,test,009,Male_likely__test__mouse009,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv +other,15473,0.7166411930897133,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +attack,3364,0.1558056597656431,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +investigation,2605,0.12065212357000601,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +mount,149,0.006901023574637581,Male_likely,train,004,Male_likely__train__mouse004,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv +other,20748,0.7562602515035538,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +investigation,4603,0.16777838527428468,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +attack,2054,0.07486786950975031,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +mount,30,0.0010934937124111536,Male_likely,train,007,Male_likely__train__mouse007,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv +other,11548,0.643880680234179,Male_likely,test,003,Male_likely__test__mouse003,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator3.csv +investigation,5750,0.3206021745190967,Male_likely,test,003,Male_likely__test__mouse003,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator3.csv +attack,637,0.03551714524672428,Male_likely,test,003,Male_likely__test__mouse003,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator3.csv +other,10548,0.5881237803178143,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator4.csv +investigation,6763,0.37708391413437414,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator4.csv +attack,624,0.03479230554781154,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator4.csv +other,16096,0.796871132234269,Male_likely,test,009,Male_likely__test__mouse009,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator3.csv +attack,3424,0.16951334224466558,Male_likely,test,009,Male_likely__test__mouse009,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator3.csv +investigation,679,0.0336155255210654,Male_likely,test,009,Male_likely__test__mouse009,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator3.csv +other,15350,0.759938610822318,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator4.csv +attack,4713,0.2333283825931977,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator4.csv +investigation,136,0.006733006584484381,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator4.csv +other,15506,0.8382981023949829,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +investigation,1515,0.0819051738119695,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +attack,1426,0.07709358274314754,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +mount,50,0.002703141049899984,Male_likely,train,002,Male_likely__train__mouse002,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv +other,15108,0.8167810996377791,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator4.csv +attack,1985,0.10731469968102936,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator4.csv +investigation,1404,0.07590420068119154,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator4.csv +other,16493,0.850636959100521,Male_likely,train,004,Male_likely__train__mouse004,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator3.csv +attack,1989,0.10258393934705246,Male_likely,train,004,Male_likely__train__mouse004,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator3.csv +investigation,907,0.046779101552426636,Male_likely,train,004,Male_likely__train__mouse004,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator3.csv +other,16077,0.8291814946619217,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator4.csv +attack,2714,0.13997627520759193,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator4.csv +investigation,598,0.03084223013048636,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator4.csv +other,13615,0.6822851415685292,Male_likely,train,006,Male_likely__train__mouse006,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator3.csv +attack,5276,0.264394888499123,Male_likely,train,006,Male_likely__train__mouse006,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator3.csv +investigation,1064,0.053319969932347784,Male_likely,train,006,Male_likely__train__mouse006,annotator3,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator3.csv +other,11459,0.5742420446003508,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator4.csv +attack,8185,0.41017288900025056,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator4.csv +investigation,311,0.015585066399398648,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator4.csv +other,10548,0.5881237803178143,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse003/Manual_Scoring_annotator4.csv +investigation,6763,0.37708391413437414,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse003/Manual_Scoring_annotator4.csv +attack,624,0.03479230554781154,Male_likely,test,003,Male_likely__test__mouse003,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse003/Manual_Scoring_annotator4.csv +other,15350,0.759938610822318,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse009/Manual_Scoring_annotator4.csv +attack,4713,0.2333283825931977,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse009/Manual_Scoring_annotator4.csv +investigation,136,0.006733006584484381,Male_likely,test,009,Male_likely__test__mouse009,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse009/Manual_Scoring_annotator4.csv +other,15108,0.8167810996377791,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse002/Manual_Scoring_annotator4.csv +attack,1985,0.10731469968102936,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse002/Manual_Scoring_annotator4.csv +investigation,1404,0.07590420068119154,Male_likely,train,002,Male_likely__train__mouse002,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse002/Manual_Scoring_annotator4.csv +other,16077,0.8291814946619217,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse004/Manual_Scoring_annotator4.csv +attack,2714,0.13997627520759193,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse004/Manual_Scoring_annotator4.csv +investigation,598,0.03084223013048636,Male_likely,train,004,Male_likely__train__mouse004,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse004/Manual_Scoring_annotator4.csv +other,11459,0.5742420446003508,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse006/Manual_Scoring_annotator4.csv +attack,8185,0.41017288900025056,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse006/Manual_Scoring_annotator4.csv +investigation,311,0.015585066399398648,Male_likely,train,006,Male_likely__train__mouse006,annotator4,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse006/Manual_Scoring_annotator4.csv +other,15707,0.815016604400166,Male_likely,test,003,Male_likely__test__mouse003,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse003/Manual_Scoring_annotator5.csv +investigation,2899,0.15042548775425488,Male_likely,test,003,Male_likely__test__mouse003,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse003/Manual_Scoring_annotator5.csv +attack,666,0.03455790784557908,Male_likely,test,003,Male_likely__test__mouse003,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse003/Manual_Scoring_annotator5.csv +other,13117,0.7123384381448897,Male_likely,test,010,Male_likely__test__mouse010,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse010/Manual_Scoring_annotator5.csv +investigation,5005,0.27180406212664276,Male_likely,test,010,Male_likely__test__mouse010,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse010/Manual_Scoring_annotator5.csv +attack,292,0.01585749972846747,Male_likely,test,010,Male_likely__test__mouse010,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse010/Manual_Scoring_annotator5.csv +other,17109,0.9109738565571588,Male_likely,test,011,Male_likely__test__mouse011,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse011/Manual_Scoring_annotator5.csv +investigation,997,0.0530855651988712,Male_likely,test,011,Male_likely__test__mouse011,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse011/Manual_Scoring_annotator5.csv +attack,675,0.03594057824396997,Male_likely,test,011,Male_likely__test__mouse011,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse011/Manual_Scoring_annotator5.csv +other,14212,0.7834619625137818,Male_likely,test,012,Male_likely__test__mouse012,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse012/Manual_Scoring_annotator5.csv +investigation,3396,0.18721058434399118,Male_likely,test,012,Male_likely__test__mouse012,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse012/Manual_Scoring_annotator5.csv +attack,532,0.029327453142227122,Male_likely,test,012,Male_likely__test__mouse012,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse012/Manual_Scoring_annotator5.csv +other,15327,0.8494236311239193,Male_likely,test,014,Male_likely__test__mouse014,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse014/Manual_Scoring_annotator5.csv +investigation,1407,0.07797605852360896,Male_likely,test,014,Male_likely__test__mouse014,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse014/Manual_Scoring_annotator5.csv +attack,1310,0.07260031035247173,Male_likely,test,014,Male_likely__test__mouse014,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse014/Manual_Scoring_annotator5.csv +other,15585,0.8442121228535833,Male_likely,test,016,Male_likely__test__mouse016,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse016/Manual_Scoring_annotator5.csv +attack,2120,0.11483668273657982,Male_likely,test,016,Male_likely__test__mouse016,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse016/Manual_Scoring_annotator5.csv +investigation,756,0.04095119440983695,Male_likely,test,016,Male_likely__test__mouse016,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse016/Manual_Scoring_annotator5.csv +other,11004,0.6215895610913404,Male_likely,test,019,Male_likely__test__mouse019,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse019/Manual_Scoring_annotator5.csv +attack,3540,0.19996610743941703,Male_likely,test,019,Male_likely__test__mouse019,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse019/Manual_Scoring_annotator5.csv +investigation,3159,0.1784443314692425,Male_likely,test,019,Male_likely__test__mouse019,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse019/Manual_Scoring_annotator5.csv +other,15458,0.8611699164345404,Male_likely,test,022,Male_likely__test__mouse022,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse022/Manual_Scoring_annotator5.csv +attack,1325,0.07381615598885793,Male_likely,test,022,Male_likely__test__mouse022,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse022/Manual_Scoring_annotator5.csv +investigation,1167,0.06501392757660167,Male_likely,test,022,Male_likely__test__mouse022,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse022/Manual_Scoring_annotator5.csv +other,11699,0.6534658995699045,Male_likely,test,023,Male_likely__test__mouse023,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse023/Manual_Scoring_annotator5.csv +investigation,5826,0.32542032061665643,Male_likely,test,023,Male_likely__test__mouse023,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse023/Manual_Scoring_annotator5.csv +attack,378,0.02111377981343909,Male_likely,test,023,Male_likely__test__mouse023,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse023/Manual_Scoring_annotator5.csv +other,14328,0.7721491700797586,Male_likely,test,026,Male_likely__test__mouse026,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse026/Manual_Scoring_annotator5.csv +investigation,3762,0.20273765897822807,Male_likely,test,026,Male_likely__test__mouse026,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse026/Manual_Scoring_annotator5.csv +attack,466,0.025113170942013364,Male_likely,test,026,Male_likely__test__mouse026,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse026/Manual_Scoring_annotator5.csv +other,11816,0.6540462747702868,Male_likely,train,001,Male_likely__train__mouse001,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse001/Manual_Scoring_annotator5.csv +investigation,4815,0.2665227499169711,Male_likely,train,001,Male_likely__train__mouse001,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse001/Manual_Scoring_annotator5.csv +attack,1435,0.07943097531274217,Male_likely,train,001,Male_likely__train__mouse001,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse001/Manual_Scoring_annotator5.csv +other,14641,0.8018511419026234,Male_likely,train,007,Male_likely__train__mouse007,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse007/Manual_Scoring_annotator5.csv +investigation,2959,0.16205706774741224,Male_likely,train,007,Male_likely__train__mouse007,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse007/Manual_Scoring_annotator5.csv +attack,659,0.0360917903499644,Male_likely,train,007,Male_likely__train__mouse007,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse007/Manual_Scoring_annotator5.csv +other,13105,0.7311833956368912,Male_likely,train,008,Male_likely__train__mouse008,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse008/Manual_Scoring_annotator5.csv +investigation,4117,0.22970484851866316,Male_likely,train,008,Male_likely__train__mouse008,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse008/Manual_Scoring_annotator5.csv +attack,701,0.039111755844445685,Male_likely,train,008,Male_likely__train__mouse008,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse008/Manual_Scoring_annotator5.csv +investigation,18804,0.6935162646603231,Undetermined,test,005,Undetermined__test__mouse005,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse005/Manual_Scoring_annotator2.csv +other,8310,0.30648373533967693,Undetermined,test,005,Undetermined__test__mouse005,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse005/Manual_Scoring_annotator2.csv +other,22552,0.8487129309047118,Undetermined,test,010,Undetermined__test__mouse010,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse010/Manual_Scoring_annotator2.csv +investigation,4020,0.15128706909528827,Undetermined,test,010,Undetermined__test__mouse010,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse010/Manual_Scoring_annotator2.csv +other,12569,0.45993120608899296,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +investigation,12406,0.45396662763466045,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +attack,1735,0.06348799765807962,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +mount,618,0.02261416861826698,Undetermined,train,001,Undetermined__train__mouse001,annotator2,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv +other,11971,0.673436093609361,Undetermined,test,004,Undetermined__test__mouse004,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse004/Manual_Scoring_annotator5.csv +investigation,5805,0.3265639063906391,Undetermined,test,004,Undetermined__test__mouse004,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse004/Manual_Scoring_annotator5.csv +other,12375,0.6889161053276179,Undetermined,test,005,Undetermined__test__mouse005,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse005/Manual_Scoring_annotator5.csv +investigation,5584,0.3108612147191449,Undetermined,test,005,Undetermined__test__mouse005,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse005/Manual_Scoring_annotator5.csv +mount,4,0.00022267995323720983,Undetermined,test,005,Undetermined__test__mouse005,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse005/Manual_Scoring_annotator5.csv +other,12460,0.6898078945911532,Undetermined,test,006,Undetermined__test__mouse006,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse006/Manual_Scoring_annotator5.csv +investigation,5603,0.31019210540884684,Undetermined,test,006,Undetermined__test__mouse006,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse006/Manual_Scoring_annotator5.csv +other,14064,0.7760304585333554,Undetermined,test,009,Undetermined__test__mouse009,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse009/Manual_Scoring_annotator5.csv +investigation,4059,0.2239695414666446,Undetermined,test,009,Undetermined__test__mouse009,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse009/Manual_Scoring_annotator5.csv +other,14658,0.8317067635043123,Undetermined,test,013,Undetermined__test__mouse013,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse013/Manual_Scoring_annotator5.csv +investigation,2966,0.1682932364956877,Undetermined,test,013,Undetermined__test__mouse013,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse013/Manual_Scoring_annotator5.csv +other,13339,0.739043714333204,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +investigation,4440,0.24599700814449554,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +attack,182,0.010083661144661754,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +mount,88,0.0048756163776386505,Undetermined,test,015,Undetermined__test__mouse015,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv +other,11321,0.6318227480745618,Undetermined,test,017,Undetermined__test__mouse017,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse017/Manual_Scoring_annotator5.csv +investigation,6585,0.36750753432302713,Undetermined,test,017,Undetermined__test__mouse017,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse017/Manual_Scoring_annotator5.csv +mount,12,0.0006697176024109834,Undetermined,test,017,Undetermined__test__mouse017,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse017/Manual_Scoring_annotator5.csv +other,12393,0.6860606731620903,Undetermined,test,018,Undetermined__test__mouse018,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse018/Manual_Scoring_annotator5.csv +investigation,5651,0.3128321523472099,Undetermined,test,018,Undetermined__test__mouse018,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse018/Manual_Scoring_annotator5.csv +mount,20,0.0011071744906997344,Undetermined,test,018,Undetermined__test__mouse018,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse018/Manual_Scoring_annotator5.csv +other,12739,0.6752358740591541,Undetermined,test,020,Undetermined__test__mouse020,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse020/Manual_Scoring_annotator5.csv +investigation,6127,0.32476412594084597,Undetermined,test,020,Undetermined__test__mouse020,annotator5,public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse020/Manual_Scoring_annotator5.csv diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_manifest.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_manifest.csv new file mode 100644 index 0000000..5e2fbd0 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_manifest.csv @@ -0,0 +1,88 @@ +path,sex_group,split,mouse,record_key,folder_annotator,file_annotator,annotator,n_frames,n_behaviors,behaviors,onehot_min_sum,onehot_max_sum,onehot_mean_sum,parse_format +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse013/Manual_Scoring_annotator1.csv,Female_likely,test,013,Female_likely__test__mouse013,1,1,annotator1,27097,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse015/Manual_Scoring_annotator1.csv,Female_likely,test,015,Female_likely__test__mouse015,1,1,annotator1,29195,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse016/Manual_Scoring_annotator1.csv,Female_likely,test,016,Female_likely__test__mouse016,1,1,annotator1,26773,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse017/Manual_Scoring_annotator1.csv,Female_likely,test,017,Female_likely__test__mouse017,1,1,annotator1,26533,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse018/Manual_Scoring_annotator1.csv,Female_likely,test,018,Female_likely__test__mouse018,1,1,annotator1,26770,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_test_mouse019/Manual_Scoring_annotator1.csv,Female_likely,test,019,Female_likely__test__mouse019,1,1,annotator1,26229,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse003/Manual_Scoring_annotator1.csv,Female_likely,train,003,Female_likely__train__mouse003,1,1,annotator1,27461,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse004/Manual_Scoring_annotator1.csv,Female_likely,train,004,Female_likely__train__mouse004,1,1,annotator1,31258,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator1_train_mouse014/Manual_Scoring_annotator1.csv,Female_likely,train,014,Female_likely__train__mouse014,1,1,annotator1,28220,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_test_mouse012/Manual_Scoring_annotator2.csv,Female_likely,test,012,Female_likely__test__mouse012,2,2,annotator2,13582,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse002/Manual_Scoring_annotator2.csv,Female_likely,train,002,Female_likely__train__mouse002,2,2,annotator2,18851,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse003/Manual_Scoring_annotator2.csv,Female_likely,train,003,Female_likely__train__mouse003,2,2,annotator2,16494,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator2_train_mouse011/Manual_Scoring_annotator2.csv,Female_likely,train,011,Female_likely__train__mouse011,2,2,annotator2,23924,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator3.csv,Female_likely,test,008,Female_likely__test__mouse008,3,3,annotator3,19307,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse008/Manual_Scoring_annotator4.csv,Female_likely,test,008,Female_likely__test__mouse008,3,4,annotator4,19307,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator3.csv,Female_likely,test,010,Female_likely__test__mouse010,3,3,annotator3,19638,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_test_mouse010/Manual_Scoring_annotator4.csv,Female_likely,test,010,Female_likely__test__mouse010,3,4,annotator4,19638,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator3.csv,Female_likely,train,001,Female_likely__train__mouse001,3,3,annotator3,24292,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse001/Manual_Scoring_annotator4.csv,Female_likely,train,001,Female_likely__train__mouse001,3,4,annotator4,24292,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator3.csv,Female_likely,train,005,Female_likely__train__mouse005,3,3,annotator3,6641,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse005/Manual_Scoring_annotator4.csv,Female_likely,train,005,Female_likely__train__mouse005,3,4,annotator4,6641,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator3.csv,Female_likely,train,007,Female_likely__train__mouse007,3,3,annotator3,18646,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator3_train_mouse007/Manual_Scoring_annotator4.csv,Female_likely,train,007,Female_likely__train__mouse007,3,4,annotator4,18646,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse008/Manual_Scoring_annotator4.csv,Female_likely,test,008,Female_likely__test__mouse008,4,4,annotator4,19307,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_test_mouse010/Manual_Scoring_annotator4.csv,Female_likely,test,010,Female_likely__test__mouse010,4,4,annotator4,19638,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse001/Manual_Scoring_annotator4.csv,Female_likely,train,001,Female_likely__train__mouse001,4,4,annotator4,24292,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse005/Manual_Scoring_annotator3.csv,Female_likely,train,005,Female_likely__train__mouse005,4,3,annotator3,6641,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator4_train_mouse007/Manual_Scoring_annotator4.csv,Female_likely,train,007,Female_likely__train__mouse007,4,4,annotator4,18646,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_test_mouse021/Manual_Scoring_annotator5.csv,Female_likely,test,021,Female_likely__test__mouse021,5,5,annotator5,18337,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse002/Manual_Scoring_annotator5.csv,Female_likely,train,002,Female_likely__train__mouse002,5,5,annotator5,17948,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse024/Manual_Scoring_annotator5.csv,Female_likely,train,024,Female_likely__train__mouse024,5,5,annotator5,9661,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Female_likely/task2_annotator5_train_mouse025/Manual_Scoring_annotator5.csv,Female_likely,train,025,Female_likely__train__mouse025,5,5,annotator5,10526,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse005/Manual_Scoring_annotator1.csv,Male_likely,test,005,Male_likely__test__mouse005,1,1,annotator1,19275,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse006/Manual_Scoring_annotator1.csv,Male_likely,test,006,Male_likely__test__mouse006,1,1,annotator1,17498,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse007/Manual_Scoring_annotator1.csv,Male_likely,test,007,Male_likely__test__mouse007,1,1,annotator1,17968,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse008/Manual_Scoring_annotator1.csv,Male_likely,test,008,Male_likely__test__mouse008,1,1,annotator1,17361,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse009/Manual_Scoring_annotator1.csv,Male_likely,test,009,Male_likely__test__mouse009,1,1,annotator1,16906,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse010/Manual_Scoring_annotator1.csv,Male_likely,test,010,Male_likely__test__mouse010,1,1,annotator1,17822,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_test_mouse011/Manual_Scoring_annotator1.csv,Male_likely,test,011,Male_likely__test__mouse011,1,1,annotator1,17383,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse001/Manual_Scoring_annotator1.csv,Male_likely,train,001,Male_likely__train__mouse001,1,1,annotator1,17353,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse002/Manual_Scoring_annotator1.csv,Male_likely,train,002,Male_likely__train__mouse002,1,1,annotator1,17415,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator1_train_mouse012/Manual_Scoring_annotator1.csv,Male_likely,train,012,Male_likely__train__mouse012,1,1,annotator1,17415,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse006/Manual_Scoring_annotator2.csv,Male_likely,test,006,Male_likely__test__mouse006,2,2,annotator2,27012,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse008/Manual_Scoring_annotator2.csv,Male_likely,test,008,Male_likely__test__mouse008,2,2,annotator2,29316,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_test_mouse009/Manual_Scoring_annotator2.csv,Male_likely,test,009,Male_likely__test__mouse009,2,2,annotator2,27323,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse004/Manual_Scoring_annotator2.csv,Male_likely,train,004,Male_likely__train__mouse004,2,2,annotator2,21591,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator2_train_mouse007/Manual_Scoring_annotator2.csv,Male_likely,train,007,Male_likely__train__mouse007,2,2,annotator2,27435,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator3.csv,Male_likely,test,003,Male_likely__test__mouse003,3,3,annotator3,17935,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse003/Manual_Scoring_annotator4.csv,Male_likely,test,003,Male_likely__test__mouse003,3,4,annotator4,17935,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator3.csv,Male_likely,test,009,Male_likely__test__mouse009,3,3,annotator3,20199,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_test_mouse009/Manual_Scoring_annotator4.csv,Male_likely,test,009,Male_likely__test__mouse009,3,4,annotator4,20199,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator3.csv,Male_likely,train,002,Male_likely__train__mouse002,3,3,annotator3,18497,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse002/Manual_Scoring_annotator4.csv,Male_likely,train,002,Male_likely__train__mouse002,3,4,annotator4,18497,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator3.csv,Male_likely,train,004,Male_likely__train__mouse004,3,3,annotator3,19389,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse004/Manual_Scoring_annotator4.csv,Male_likely,train,004,Male_likely__train__mouse004,3,4,annotator4,19389,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator3.csv,Male_likely,train,006,Male_likely__train__mouse006,3,3,annotator3,19955,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator3_train_mouse006/Manual_Scoring_annotator4.csv,Male_likely,train,006,Male_likely__train__mouse006,3,4,annotator4,19955,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse003/Manual_Scoring_annotator4.csv,Male_likely,test,003,Male_likely__test__mouse003,4,4,annotator4,17935,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_test_mouse009/Manual_Scoring_annotator4.csv,Male_likely,test,009,Male_likely__test__mouse009,4,4,annotator4,20199,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse002/Manual_Scoring_annotator4.csv,Male_likely,train,002,Male_likely__train__mouse002,4,4,annotator4,18497,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse004/Manual_Scoring_annotator4.csv,Male_likely,train,004,Male_likely__train__mouse004,4,4,annotator4,19389,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator4_train_mouse006/Manual_Scoring_annotator4.csv,Male_likely,train,006,Male_likely__train__mouse006,4,4,annotator4,19955,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse003/Manual_Scoring_annotator5.csv,Male_likely,test,003,Male_likely__test__mouse003,5,5,annotator5,19272,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse010/Manual_Scoring_annotator5.csv,Male_likely,test,010,Male_likely__test__mouse010,5,5,annotator5,18414,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse011/Manual_Scoring_annotator5.csv,Male_likely,test,011,Male_likely__test__mouse011,5,5,annotator5,18781,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse012/Manual_Scoring_annotator5.csv,Male_likely,test,012,Male_likely__test__mouse012,5,5,annotator5,18140,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse014/Manual_Scoring_annotator5.csv,Male_likely,test,014,Male_likely__test__mouse014,5,5,annotator5,18044,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse016/Manual_Scoring_annotator5.csv,Male_likely,test,016,Male_likely__test__mouse016,5,5,annotator5,18461,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse019/Manual_Scoring_annotator5.csv,Male_likely,test,019,Male_likely__test__mouse019,5,5,annotator5,17703,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse022/Manual_Scoring_annotator5.csv,Male_likely,test,022,Male_likely__test__mouse022,5,5,annotator5,17950,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse023/Manual_Scoring_annotator5.csv,Male_likely,test,023,Male_likely__test__mouse023,5,5,annotator5,17903,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_test_mouse026/Manual_Scoring_annotator5.csv,Male_likely,test,026,Male_likely__test__mouse026,5,5,annotator5,18556,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse001/Manual_Scoring_annotator5.csv,Male_likely,train,001,Male_likely__train__mouse001,5,5,annotator5,18066,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse007/Manual_Scoring_annotator5.csv,Male_likely,train,007,Male_likely__train__mouse007,5,5,annotator5,18259,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Male_likely/task2_annotator5_train_mouse008/Manual_Scoring_annotator5.csv,Male_likely,train,008,Male_likely__train__mouse008,5,5,annotator5,17923,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse005/Manual_Scoring_annotator2.csv,Undetermined,test,005,Undetermined__test__mouse005,2,2,annotator2,27114,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_test_mouse010/Manual_Scoring_annotator2.csv,Undetermined,test,010,Undetermined__test__mouse010,2,2,annotator2,26572,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator2_train_mouse001/Manual_Scoring_annotator2.csv,Undetermined,train,001,Undetermined__train__mouse001,2,2,annotator2,27328,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse004/Manual_Scoring_annotator5.csv,Undetermined,test,004,Undetermined__test__mouse004,5,5,annotator5,17776,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse005/Manual_Scoring_annotator5.csv,Undetermined,test,005,Undetermined__test__mouse005,5,5,annotator5,17963,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse006/Manual_Scoring_annotator5.csv,Undetermined,test,006,Undetermined__test__mouse006,5,5,annotator5,18063,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse009/Manual_Scoring_annotator5.csv,Undetermined,test,009,Undetermined__test__mouse009,5,5,annotator5,18123,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse013/Manual_Scoring_annotator5.csv,Undetermined,test,013,Undetermined__test__mouse013,5,5,annotator5,17624,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse015/Manual_Scoring_annotator5.csv,Undetermined,test,015,Undetermined__test__mouse015,5,5,annotator5,18049,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse017/Manual_Scoring_annotator5.csv,Undetermined,test,017,Undetermined__test__mouse017,5,5,annotator5,17918,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse018/Manual_Scoring_annotator5.csv,Undetermined,test,018,Undetermined__test__mouse018,5,5,annotator5,18064,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" +public_data/calms21_task2_converted_nas/CalMS21/Undetermined/task2_annotator5_test_mouse020/Manual_Scoring_annotator5.csv,Undetermined,test,020,Undetermined__test__mouse020,5,5,annotator5,18866,4,attack|investigation|mount|other,1.0,1.0,1.0,"{'header': [0, 1]}" diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_agreement.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_agreement.csv new file mode 100644 index 0000000..16ac360 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_agreement.csv @@ -0,0 +1,31 @@ +record_key,sex_group,split,mouse,annotator_a,annotator_b,n_frames_compared,accuracy,balanced_accuracy,macro_f1,weighted_f1,cohen_kappa,mcc,adjusted_rand,normalized_mutual_info +Female_likely__train__mouse003,Female_likely,train,mouse003,annotator1,annotator2,16494,0.4039044501030678,0.36370697363392424,0.33802458534792285,0.36444119121130614,0.04193096288459075,0.04464791973783397,0.02276819194555807,0.014850608014396607 +Female_likely__train__mouse002,Female_likely,train,mouse002,annotator2,annotator5,17948,0.6485959438377535,0.46320573399027837,0.44812535273511706,0.6644548529847005,0.23448578043486457,0.24110240466235147,0.11181267870738362,0.04853631089294546 +Female_likely__test__mouse008,Female_likely,test,mouse008,annotator3,annotator4,19307,0.9218936137152328,0.8554400063036098,0.86701586422364,0.9218891482084861,0.8153775601183475,0.8159675246436876,0.7812561092828942,0.6216484732111585 +Female_likely__test__mouse010,Female_likely,test,mouse010,annotator3,annotator4,19638,0.9395050412465628,0.9432460674751687,0.9439153932743669,0.9397799943929638,0.9079060301034081,0.9080808796737051,0.818360832217306,0.7883136259795048 +Female_likely__train__mouse001,Female_likely,train,mouse001,annotator3,annotator4,24292,0.9055656183105549,0.8502663061191004,0.859721772209907,0.9047376175334692,0.7935982282067418,0.7939300761139455,0.7235080573359697,0.5660896946301407 +Female_likely__train__mouse005,Female_likely,train,mouse005,annotator3,annotator4,6641,0.8601114290016564,0.759471277243417,0.782293537442412,0.8520281416845269,0.7537049909490066,0.7606167360168221,0.6320024568014917,0.5210765684680227 +Female_likely__train__mouse007,Female_likely,train,mouse007,annotator3,annotator4,18646,0.9312453073045157,0.9267862169530033,0.9247171849118767,0.9314988785507446,0.8940458810036451,0.8941282882847944,0.8170611518129011,0.7616072795455856 +Male_likely__test__mouse006,Male_likely,test,mouse006,annotator1,annotator2,17498,0.6587610012572865,0.3338242716542457,0.2953753687465264,0.5648084965716536,0.024698291682487716,0.036509544214701324,0.022513640260264475,0.010687621851189478 +Male_likely__test__mouse008,Male_likely,test,mouse008,annotator1,annotator2,17361,0.48804792350671045,0.27617755847504455,0.25609905852847725,0.474704253036334,0.08304890417753097,0.08731990074915223,0.004447900552417427,0.018637717874306518 +Male_likely__test__mouse009,Male_likely,test,mouse009,annotator1,annotator2,16906,0.4000354903584526,0.27613723770658244,0.21654659274872562,0.3139099015266101,0.014933460645479846,0.022100223833915553,0.015872691364128694,0.00637047901196171 +Male_likely__test__mouse009,Male_likely,test,mouse009,annotator1,annotator3,16906,0.3923459126937182,0.28002854732663535,0.20028734449644958,0.29851536741278933,0.059531504696466064,0.08589653314398554,0.0077721502492616315,0.020546492314253924 +Male_likely__test__mouse009,Male_likely,test,mouse009,annotator1,annotator4,16906,0.3764344019874601,0.29339636955802934,0.19098508926963795,0.2715591037673826,0.07672229657163365,0.10943039211408608,0.022389064348247934,0.03145470134681595 +Male_likely__test__mouse009,Male_likely,test,mouse009,annotator2,annotator3,20199,0.6901826823110055,0.25958923446896515,0.2489716070698084,0.7020548594059618,0.02265227725230534,0.023184200330324502,0.02103013854892277,0.0032354678855123537 +Male_likely__test__mouse009,Male_likely,test,mouse009,annotator2,annotator4,20199,0.6687459775236397,0.2720058604783582,0.24201271872175084,0.6878181985688958,0.04048868207093359,0.043067406150068294,0.02865008365228025,0.005758795998655996 +Male_likely__test__mouse009,Male_likely,test,mouse009,annotator3,annotator4,20199,0.901529778701916,0.6844580357585811,0.6864362049371927,0.8984331691305756,0.7223439818054838,0.729468968606389,0.6664161373564781,0.4945550188457223 +Male_likely__test__mouse010,Male_likely,test,mouse010,annotator1,annotator5,17822,0.6435304679609472,0.36370997522665816,0.32224316844684814,0.6754890724855915,0.05167736429896774,0.05638352024514488,0.03730490682092814,0.00732354537756817 +Male_likely__test__mouse011,Male_likely,test,mouse011,annotator1,annotator5,17383,0.6283150204222516,0.34690669387483264,0.3181914337161452,0.5466236563347342,0.07105808807738878,0.09680496936725837,0.03304271776312204,0.030398619093810263 +Male_likely__train__mouse001,Male_likely,train,mouse001,annotator1,annotator5,17353,0.5291880366507232,0.3947902089163982,0.38021151804952735,0.5206832285604163,0.11156574948153697,0.11512643200315303,0.04052043310185562,0.015817009423956037 +Male_likely__train__mouse002,Male_likely,train,mouse002,annotator1,annotator3,17415,0.6111398219925351,0.3610657056511463,0.2633049983262741,0.5618087205752977,0.07265720809527532,0.07931643798862953,0.07642922385139278,0.021201553986210885 +Male_likely__train__mouse002,Male_likely,train,mouse002,annotator1,annotator4,17415,0.6138960666092449,0.3877184080841684,0.3791008908097458,0.573988318543896,0.1075982280920218,0.11517826464687995,0.09281088120213646,0.02778494985988579 +Male_likely__train__mouse002,Male_likely,train,mouse002,annotator3,annotator4,18497,0.9458831161810023,0.670101786350425,0.641432222601625,0.9467443682962342,0.8200923793789678,0.8231303771815267,0.8399373872016986,0.6691970813516401 +Male_likely__train__mouse004,Male_likely,train,mouse004,annotator2,annotator3,19389,0.6272113053793388,0.26723221124437513,0.2602572801724441,0.5730120304003532,0.053089477031663335,0.05854555774327753,0.047498038996621444,0.01118695080744534 +Male_likely__train__mouse004,Male_likely,train,mouse004,annotator2,annotator4,19389,0.6254577337665687,0.2712977636074583,0.26322612141894086,0.5766569674611886,0.07298053125647364,0.0785023851870585,0.07071075408261353,0.014353864115451859 +Male_likely__train__mouse004,Male_likely,train,mouse004,annotator3,annotator4,19389,0.9030893805766156,0.6751614091879355,0.6538282644537937,0.9027157000485604,0.6524825604712206,0.6558090842366178,0.689883547153328,0.4468863236433701 +Male_likely__train__mouse007,Male_likely,train,mouse007,annotator2,annotator5,18259,0.6190371871405882,0.2607654390991093,0.25750167164310617,0.5872931150554282,0.04758528033924647,0.04917042698971403,0.0471274485475786,0.007506599641240076 +Male_likely__test__mouse003,Male_likely,test,mouse003,annotator3,annotator4,17935,0.8478394201282409,0.7937663473242283,0.7853268448118956,0.8501574762591944,0.6951843158557286,0.6999047200229856,0.527055632323398,0.4193709315772175 +Male_likely__test__mouse003,Male_likely,test,mouse003,annotator3,annotator5,17935,0.5403401170894898,0.2960594368558706,0.27771169109888555,0.49404115768469625,-0.07079309797097944,-0.07697747968192092,-0.0007328657397576199,0.020972180132778597 +Male_likely__test__mouse003,Male_likely,test,mouse003,annotator4,annotator5,17935,0.512684694730973,0.30490225401088517,0.27993085058302447,0.45816988585106544,-0.04702187372175537,-0.05382071898628711,0.01324049368178369,0.02189183262023253 +Male_likely__train__mouse006,Male_likely,train,mouse006,annotator3,annotator4,19955,0.801904284640441,0.6381730003690005,0.6300795408857404,0.8022292295019943,0.6029545839738002,0.6249891192960384,0.4448759062162791,0.35977267151232045 +Undetermined__test__mouse005,Undetermined,test,mouse005,annotator2,annotator5,17963,0.35294772588097756,0.41280505772991505,0.23531858773263217,0.35393711895952024,-0.1293087135997184,-0.1734120230366092,0.06674964536602013,0.02402781417430315 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_summary.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_summary.csv new file mode 100644 index 0000000..4506277 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_summary.csv @@ -0,0 +1,11 @@ +annotator_a,annotator_b,n_shared_records,total_frames_compared,accuracy_mean,balanced_accuracy_mean,macro_f1_mean,cohen_kappa_mean,mcc_mean,ari_mean,nmi_mean +annotator1,annotator2,4,68259,0.48768721630637935,0.31246151036744924,0.276511401342913,0.04115290484752232,0.04764439713390077,0.016400606030592166,0.012636606687963578 +annotator1,annotator3,2,34321,0.5017428673431267,0.32054712648889083,0.23179617141136183,0.06609435639587069,0.08260648556630754,0.042100687050327205,0.020874023150232405 +annotator1,annotator4,2,34321,0.49516523429835246,0.34055738882109887,0.28504299003969186,0.09216026233182772,0.11230432838048302,0.057599972775192194,0.02961982560335087 +annotator1,annotator5,3,52558,0.6003445083446407,0.3684689593392963,0.34021537340417357,0.07810040061929784,0.08943830720518542,0.03695601922863526,0.017846391298444824 +annotator2,annotator3,2,39588,0.6586969938451721,0.2634107228566701,0.25461444362112623,0.03787087714198434,0.04086487903680101,0.03426408877277211,0.007211209346478847 +annotator2,annotator4,2,39588,0.6471018556451043,0.2716518120429082,0.2526194200703459,0.05673460666370361,0.0607848956685634,0.04968041886744689,0.010056330057053928 +annotator2,annotator5,3,54170,0.5401936189531065,0.37892541027310095,0.31364853737028514,0.05092078239146421,0.03895360287181877,0.07522992420699412,0.02669024156949623 +annotator3,annotator4,10,184499,0.8958566989806739,0.779687045308447,0.7774766829752451,0.765769051186635,0.7706025774076513,0.6940357217701745,0.5648517668764683 +annotator3,annotator5,1,17935,0.5403401170894898,0.2960594368558706,0.27771169109888555,-0.07079309797097944,-0.07697747968192092,-0.0007328657397576199,0.020972180132778597 +annotator4,annotator5,1,17935,0.512684694730973,0.30490225401088517,0.27993085058302447,-0.04702187372175537,-0.05382071898628711,0.01324049368178369,0.02189183262023253 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_annotator_file_summary.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_annotator_file_summary.csv new file mode 100644 index 0000000..1ec14b4 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_annotator_file_summary.csv @@ -0,0 +1,9 @@ +annotator,n_files,total_frames,mean_frames_per_file +annotator1,12,227288,18940.666666666668 +annotator2,12,227288,18940.666666666668 +annotator3,12,227288,18940.666666666668 +annotator4,12,227288,18940.666666666668 +annotator5,12,227288,18940.666666666668 +annotator6,12,227288,18940.666666666668 +annotator7,12,227288,18940.666666666668 +annotator8,12,227288,18940.666666666668 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_behavior_positive_fraction_by_annotator.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_behavior_positive_fraction_by_annotator.csv new file mode 100644 index 0000000..beff544 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_behavior_positive_fraction_by_annotator.csv @@ -0,0 +1,26 @@ +annotator,behavior,n_positive_frames,total_frames,fraction_within_annotator +annotator1,aggressivemount,14,227288,6.159586075815705e-05 +annotator1,attack,15145,227288,0.06663352222730633 +annotator1,mount,20346,227288,0.08951638449896167 +annotator1,sniff,49990,227288,0.21994121995001936 +annotator2,attack,17015,227288,0.07486096934286016 +annotator2,mount,20894,227288,0.09192742247720953 +annotator2,sniff,47833,227288,0.2104510576889233 +annotator3,attack,14597,227288,0.06422248424905846 +annotator3,mount,19489,227288,0.08574583787969449 +annotator3,sniff,42148,227288,0.18543873851677167 +annotator4,attack,17422,227288,0.07665164900918658 +annotator4,mount,21108,227288,0.09286895920594136 +annotator4,sniff,44154,227288,0.1942645454225476 +annotator5,attack,13816,227288,0.060786315159621275 +annotator5,mount,20181,227288,0.08879043328288339 +annotator5,sniff,45286,227288,0.1992450107352786 +annotator6,attack,14865,227288,0.06540160501214319 +annotator6,mount,20535,227288,0.09034792861919679 +annotator6,sniff,50969,227288,0.2242485304987505 +annotator7,attack,21208,227288,0.09330892963992819 +annotator7,mount,19103,227288,0.0840475520045053 +annotator7,sniff,38217,227288,0.16814350075674914 +annotator8,attack,14547,227288,0.06400249903206505 +annotator8,mount,20122,227288,0.08853085072683116 +annotator8,sniff,38535,227288,0.16954260673682728 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_bout_stats_by_annotator.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_bout_stats_by_annotator.csv new file mode 100644 index 0000000..06eddfb --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_bout_stats_by_annotator.csv @@ -0,0 +1,26 @@ +annotator,behavior,n_bouts,mean_bout_duration_frames,median_bout_duration_frames,mean_bout_duration_s,median_bout_duration_s +annotator1,aggressivemount,1,14.0,14.0,0.4666666666666667,0.4666666666666667 +annotator1,attack,280,54.089285714285715,39.5,1.8029761904761905,1.3166666666666667 +annotator1,mount,127,160.20472440944883,61.0,5.340157480314961,2.033333333333333 +annotator1,sniff,924,54.1017316017316,24.0,1.8033910533910533,0.8 +annotator2,attack,252,67.51984126984127,47.5,2.2506613756613754,1.5833333333333335 +annotator2,mount,128,163.234375,62.5,5.441145833333334,2.0833333333333335 +annotator2,sniff,869,55.04372842347526,24.0,1.8347909474491753,0.8 +annotator3,attack,281,51.94661921708185,38.0,1.7315539739027284,1.2666666666666666 +annotator3,mount,142,137.24647887323943,43.0,4.574882629107981,1.4333333333333333 +annotator3,sniff,939,44.88604898828541,18.0,1.4962016329428471,0.6 +annotator4,attack,273,63.81684981684982,45.0,2.127228327228327,1.5 +annotator4,mount,154,137.06493506493507,42.5,4.568831168831169,1.4166666666666665 +annotator4,sniff,864,51.104166666666664,20.0,1.7034722222222223,0.6666666666666666 +annotator5,attack,365,37.85205479452055,26.0,1.2617351598173516,0.8666666666666667 +annotator5,mount,126,160.16666666666666,55.5,5.3388888888888895,1.85 +annotator5,sniff,1083,41.81532779316713,19.0,1.3938442597722376,0.6333333333333333 +annotator6,attack,331,44.909365558912384,27.0,1.4969788519637461,0.9 +annotator6,mount,129,159.1860465116279,61.0,5.3062015503875966,2.033333333333333 +annotator6,sniff,940,54.22234042553192,22.0,1.8074113475177305,0.7333333333333333 +annotator7,attack,87,243.77011494252875,162.0,8.125670498084292,5.4 +annotator7,mount,73,261.6849315068493,110.0,8.72283105022831,3.6666666666666665 +annotator7,sniff,270,141.54444444444445,76.5,4.718148148148148,2.55 +annotator8,attack,208,69.9375,50.0,2.33125,1.6666666666666667 +annotator8,mount,108,186.3148148148148,69.0,6.210493827160494,2.3 +annotator8,sniff,376,102.48670212765957,60.0,3.416223404255319,2.0 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_label_distribution_by_annotator.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_label_distribution_by_annotator.csv new file mode 100644 index 0000000..3026747 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_label_distribution_by_annotator.csv @@ -0,0 +1,38 @@ +annotator,behavior,n_frames,fraction_within_annotator +annotator1,aggressivemount,14,6.159586075815705e-05 +annotator1,attack,15145,0.06663352222730633 +annotator1,mount,20346,0.08951638449896167 +annotator1,other,141793,0.6238472774629545 +annotator1,sniff,49990,0.21994121995001936 +annotator2,attack,17014,0.07485656963852029 +annotator2,attack+sniff,1,4.3997043398683606e-06 +annotator2,mount,20894,0.09192742247720953 +annotator2,other,141547,0.6227649501953468 +annotator2,sniff,47832,0.21044665798458342 +annotator3,attack,14597,0.06422248424905846 +annotator3,mount,19489,0.08574583787969449 +annotator3,other,151054,0.6645929393544754 +annotator3,sniff,42148,0.18543873851677167 +annotator4,attack,17422,0.07665164900918658 +annotator4,mount,21108,0.09286895920594136 +annotator4,other,144604,0.6362148463623245 +annotator4,sniff,44154,0.1942645454225476 +annotator5,attack,13808,0.06075111752490232 +annotator5,attack+sniff,8,3.5197634718946884e-05 +annotator5,mount,20181,0.08879043328288339 +annotator5,other,148013,0.6512134384569357 +annotator5,sniff,45278,0.19920981310055963 +annotator6,attack,14859,0.06537520678610398 +annotator6,attack+sniff,6,2.6398226039210167e-05 +annotator6,mount,20525,0.09030393157579811 +annotator6,mount+sniff,10,4.3997043398683606e-05 +annotator6,other,140935,0.6200723311393475 +annotator6,sniff,50953,0.2241781352293126 +annotator7,attack,21208,0.09330892963992819 +annotator7,mount,19103,0.0840475520045053 +annotator7,other,148760,0.6545000175988174 +annotator7,sniff,38217,0.16814350075674914 +annotator8,attack,14547,0.06400249903206505 +annotator8,mount,20122,0.08853085072683116 +annotator8,other,154084,0.6779240435042765 +annotator8,sniff,38535,0.16954260673682728 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_manifest.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_manifest.csv new file mode 100644 index 0000000..3da9a15 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_manifest.csv @@ -0,0 +1,97 @@ +relative_path,record_key,sex_group,mouse_record,annotator,version,n_frames,behavior_columns +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi1_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator1,v1,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi1_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator1,v2,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi2_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator2,v1,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi2_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator2,v2,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi3_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator3,v1,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi3_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator3,v2,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi4_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator4,v1,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi4_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator4,v2,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi5_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator5,v1,24292,attack|mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi5_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator5,v2,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi6_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator6,v1,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi6_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator6,v2,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi7_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator7,v1,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi7_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator7,v2,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi8_v1.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator8,v1,24292,mount|sniff +Female_likely/Mouse060_20160526_18-16-27/Manual_Scoring_annotator_multi8_v2.csv,Female_likely/Mouse060_20160526_18-16-27,Female_likely,Mouse060_20160526_18-16-27,annotator8,v2,24292,mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi1_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator1,v1,18646,mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi2_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator2,v1,18646,mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi3_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator3,v1,18646,mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi4_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator4,v1,18646,mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi5_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator5,v1,18646,attack|mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi6_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator6,v1,18646,mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi7_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator7,v1,18646,mount|sniff +Female_likely/Mouse061_20160526_18-38-49/Manual_Scoring_annotator_multi8_v1.csv,Female_likely/Mouse061_20160526_18-38-49,Female_likely,Mouse061_20160526_18-38-49,annotator8,v1,18646,mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi1_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator1,v1,19638,aggressivemount|mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi2_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator2,v1,19638,mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi3_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator3,v1,19638,mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi4_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator4,v1,19638,mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi5_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator5,v1,19638,attack|mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi6_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator6,v1,19638,mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi7_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator7,v1,19638,mount|sniff +Female_likely/Mouse062_20160526_18-56-26/Manual_Scoring_annotator_multi8_v1.csv,Female_likely/Mouse062_20160526_18-56-26,Female_likely,Mouse062_20160526_18-56-26,annotator8,v1,19638,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi1_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator1,v1,19307,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi2_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator2,v1,19307,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi3_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator3,v1,19307,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi4_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator4,v1,19307,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi5_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator5,v1,19307,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi6_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator6,v1,19307,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi7_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator7,v1,19307,mount|sniff +Female_likely/Mouse063_20160526_19-14-46/Manual_Scoring_annotator_multi8_v1.csv,Female_likely/Mouse063_20160526_19-14-46,Female_likely,Mouse063_20160526_19-14-46,annotator8,v1,19307,mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi1_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator1,v1,6641,mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi2_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator2,v1,6641,mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi3_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator3,v1,6641,mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi4_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator4,v1,6641,mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi5_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator5,v1,6641,attack|mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi6_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator6,v1,6641,mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi7_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator7,v1,6641,mount|sniff +Female_likely/Mouse163_20161018_18-22-45/Manual_Scoring_annotator_multi8_v1.csv,Female_likely/Mouse163_20161018_18-22-45,Female_likely,Mouse163_20161018_18-22-45,annotator8,v1,6641,mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi1_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator1,v1,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi1_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator1,v2,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi2_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator2,v1,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi2_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator2,v2,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi3_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator3,v1,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi3_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator3,v2,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi4_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator4,v1,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi4_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator4,v2,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi5_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator5,v1,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi5_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator5,v2,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi6_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator6,v1,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi6_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator6,v2,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi7_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator7,v1,18497,attack|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi7_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator7,v2,18497,attack|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi8_v1.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator8,v1,18497,attack|mount|sniff +Male_likely/Mouse069_20160709_16-02-03/Manual_Scoring_annotator_multi8_v2.csv,Male_likely/Mouse069_20160709_16-02-03,Male_likely,Mouse069_20160709_16-02-03,annotator8,v2,18497,attack|mount|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi1_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator1,v1,20199,attack|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi2_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator2,v1,20199,attack|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi3_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator3,v1,20199,attack|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi4_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator4,v1,20199,attack|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi5_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator5,v1,20199,attack|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi6_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator6,v1,20199,attack|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi7_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator7,v1,20199,attack|sniff +Male_likely/Mouse070_20160709_16-58-36/Manual_Scoring_annotator_multi8_v1.csv,Male_likely/Mouse070_20160709_16-58-36,Male_likely,Mouse070_20160709_16-58-36,annotator8,v1,20199,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi1_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator1,v1,19389,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi2_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator2,v1,19389,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi3_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator3,v1,19389,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi4_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator4,v1,19389,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi5_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator5,v1,19389,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi6_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator6,v1,19389,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi7_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator7,v1,19389,attack|sniff +Male_likely/Mouse074_20160709_19-15-23/Manual_Scoring_annotator_multi8_v1.csv,Male_likely/Mouse074_20160709_19-15-23,Male_likely,Mouse074_20160709_19-15-23,annotator8,v1,19389,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi1_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator1,v1,19955,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi2_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator2,v1,19955,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi3_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator3,v1,19955,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi4_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator4,v1,19955,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi5_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator5,v1,19955,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi6_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator6,v1,19955,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi7_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator7,v1,19955,attack|sniff +Male_likely/Mouse077_20160709_18-29-34/Manual_Scoring_annotator_multi8_v1.csv,Male_likely/Mouse077_20160709_18-29-34,Male_likely,Mouse077_20160709_18-29-34,annotator8,v1,19955,attack|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi1_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator1,v1,17935,attack|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi2_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator2,v1,17935,attack|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi3_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator3,v1,17935,attack|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi4_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator4,v1,17935,attack|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi5_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator5,v1,17935,attack|mount|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi6_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator6,v1,17935,attack|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi7_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator7,v1,17935,attack|sniff +Undetermined/Mouse162_20161017_19-58-28/Manual_Scoring_annotator_multi8_v1.csv,Undetermined/Mouse162_20161017_19-58-28,Undetermined,Mouse162_20161017_19-58-28,annotator8,v1,17935,attack|sniff diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_agreement_by_behavior.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_agreement_by_behavior.csv new file mode 100644 index 0000000..adc316a --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_agreement_by_behavior.csv @@ -0,0 +1,1345 @@ +behavior,n_true_bouts,n_pred_bouts,tp,fp,fn,bout_precision,bout_recall,bout_f1,mean_matched_iou,record_key,mouse_record,sex_group,version,annotator_a,annotator_b +0,26,26,25,1,1,0.9615384615384616,0.9615384615384616,0.9615384615384616,0.9726730911072553,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator2 +1,25,25,24,1,1,0.96,0.96,0.96,0.9103139407048119,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator2 +0,26,33,26,7,0,0.7878787878787878,1.0,0.8813559322033898,0.9736085887487569,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator3 +1,25,32,22,10,3,0.6875,0.88,0.7719298245614036,0.8734604454543967,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator3 +0,26,46,25,21,1,0.5434782608695652,0.9615384615384616,0.6944444444444445,0.9302220561063481,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator4 +1,25,45,23,22,2,0.5111111111111111,0.92,0.6571428571428571,0.8786865814828184,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator4 +0,26,5,2,3,24,0.4,0.07692307692307693,0.12903225806451613,0.5809152322636087,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator5 +1,25,4,0,4,25,0.0,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator5 +0,26,30,26,4,0,0.8666666666666667,1.0,0.9285714285714286,0.9482969750904366,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator6 +1,25,29,25,4,0,0.8620689655172413,1.0,0.9259259259259259,0.9201683586588213,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator6 +0,26,18,17,1,9,0.9444444444444444,0.6538461538461539,0.7727272727272727,0.8960012011173928,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator7 +1,25,17,16,1,9,0.9411764705882353,0.64,0.7619047619047621,0.9199784676310078,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator7 +0,26,23,22,1,4,0.9565217391304348,0.8461538461538461,0.8979591836734695,0.8785465257492384,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator8 +1,25,22,18,4,7,0.8181818181818182,0.72,0.7659574468085107,0.7430509661862991,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator8 +0,26,33,26,7,0,0.7878787878787878,1.0,0.8813559322033898,0.9491108045366216,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator3 +1,25,32,22,10,3,0.6875,0.88,0.7719298245614036,0.8526640449523175,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator3 +0,26,46,25,21,1,0.5434782608695652,0.9615384615384616,0.6944444444444445,0.9202323250280883,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator4 +1,25,45,24,21,1,0.5333333333333333,0.96,0.6857142857142858,0.870771149824067,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator4 +0,26,5,2,3,24,0.4,0.07692307692307693,0.12903225806451613,0.5779702548414711,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator5 +1,25,4,0,4,25,0.0,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator5 +0,26,30,25,5,1,0.8333333333333334,0.9615384615384616,0.8928571428571429,0.9312450089400512,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator6 +1,25,29,23,6,2,0.7931034482758621,0.92,0.851851851851852,0.9005440786664513,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator6 +0,26,18,17,1,9,0.9444444444444444,0.6538461538461539,0.7727272727272727,0.8879086050595454,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator7 +1,25,17,16,1,9,0.9411764705882353,0.64,0.7619047619047621,0.915698874519639,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator7 +0,26,23,22,1,4,0.9565217391304348,0.8461538461538461,0.8979591836734695,0.8746725420549168,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator8 +1,25,22,17,5,8,0.7727272727272727,0.68,0.7234042553191491,0.7505425164175544,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator8 +0,33,46,26,20,7,0.5652173913043478,0.7878787878787878,0.6582278481012658,0.9228403060570016,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator4 +1,32,45,24,21,8,0.5333333333333333,0.75,0.6233766233766235,0.8540122577104513,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator4 +0,33,5,2,3,31,0.4,0.06060606060606061,0.10526315789473685,0.5789778465892941,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator5 +1,32,4,1,3,31,0.25,0.03125,0.05555555555555555,0.6666666666666666,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator5 +0,33,30,26,4,7,0.8666666666666667,0.7878787878787878,0.8253968253968254,0.935825027225785,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator6 +1,32,29,21,8,11,0.7241379310344828,0.65625,0.6885245901639345,0.8472079577030242,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator6 +0,33,18,17,1,16,0.9444444444444444,0.5151515151515151,0.6666666666666666,0.8999309215816753,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator7 +1,32,17,16,1,16,0.9411764705882353,0.5,0.6530612244897959,0.9197436297154112,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator7 +0,33,23,22,1,11,0.9565217391304348,0.6666666666666666,0.7857142857142856,0.8721468121863161,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator8 +1,32,22,17,5,15,0.7727272727272727,0.53125,0.6296296296296297,0.7289628684590134,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator8 +0,46,5,2,3,44,0.4,0.043478260869565216,0.0784313725490196,0.5305935121565282,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator5 +1,45,4,0,4,45,0.0,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator5 +0,46,30,28,2,18,0.9333333333333333,0.6086956521739131,0.7368421052631579,0.9296653849036345,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator6 +1,45,29,24,5,21,0.8275862068965517,0.5333333333333333,0.6486486486486487,0.8736305189039522,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator6 +0,46,18,15,3,31,0.8333333333333334,0.32608695652173914,0.46875000000000006,0.8490538110127849,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator7 +1,45,17,16,1,29,0.9411764705882353,0.35555555555555557,0.5161290322580645,0.9120175898686429,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator7 +0,46,23,21,2,25,0.9130434782608695,0.45652173913043476,0.608695652173913,0.8127375003843058,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator8 +1,45,22,17,5,28,0.7727272727272727,0.37777777777777777,0.5074626865671642,0.7349985970568285,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator8 +0,5,30,2,28,3,0.06666666666666667,0.4,0.1142857142857143,0.528417716020908,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator6 +1,4,29,0,29,4,0.0,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator6 +0,5,18,3,15,2,0.16666666666666666,0.6,0.2608695652173913,0.555218773240763,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator7 +1,4,17,0,17,4,0.0,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator7 +0,5,23,2,21,3,0.08695652173913043,0.4,0.14285714285714285,0.5821323695566953,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator8 +1,4,22,0,22,4,0.0,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator8 +0,30,18,17,1,13,0.9444444444444444,0.5666666666666667,0.7083333333333334,0.8542456789156114,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator7 +1,29,17,16,1,13,0.9411764705882353,0.5517241379310345,0.6956521739130435,0.901032757167084,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator7 +0,30,23,22,1,8,0.9565217391304348,0.7333333333333333,0.8301886792452831,0.8560037272424291,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator8 +1,29,22,18,4,11,0.8181818181818182,0.6206896551724138,0.7058823529411765,0.7626661207656874,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator8 +0,18,23,18,5,0,0.782608695652174,1.0,0.878048780487805,0.8503254091053732,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator7,annotator8 +1,17,22,14,8,3,0.6363636363636364,0.8235294117647058,0.717948717948718,0.7722072299375358,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator7,annotator8 +0,26,31,25,6,1,0.8064516129032258,0.9615384615384616,0.8771929824561403,0.9421990827471676,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator2 +1,25,30,25,5,0,0.8333333333333334,1.0,0.9090909090909091,0.8908984598356581,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator2 +0,26,30,26,4,0,0.8666666666666667,1.0,0.9285714285714286,0.977882529671183,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator3 +1,25,29,22,7,3,0.7586206896551724,0.88,0.8148148148148148,0.8909456699788884,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator3 +0,26,29,24,5,2,0.8275862068965517,0.9230769230769231,0.8727272727272727,0.9529296956175566,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator4 +1,25,28,21,7,4,0.75,0.84,0.7924528301886793,0.9175706999104711,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator4 +0,26,27,25,2,1,0.9259259259259259,0.9615384615384616,0.9433962264150944,0.9547171302349092,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator5 +1,25,26,24,2,1,0.9230769230769231,0.96,0.9411764705882353,0.9253311589146515,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator5 +0,26,25,23,2,3,0.92,0.8846153846153846,0.9019607843137256,0.9556151723868446,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator6 +1,25,24,23,1,2,0.9583333333333334,0.92,0.9387755102040817,0.8637835914485111,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator6 +0,26,24,24,0,2,1.0,0.9230769230769231,0.9600000000000001,0.959582677397551,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator7 +1,25,23,22,1,3,0.9565217391304348,0.88,0.9166666666666666,0.9267163750258635,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator7 +0,26,24,23,1,3,0.9583333333333334,0.8846153846153846,0.9199999999999999,0.9021445360617725,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator8 +1,25,23,20,3,5,0.8695652173913043,0.8,0.8333333333333333,0.8392222733200887,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator8 +0,31,30,25,5,6,0.8333333333333334,0.8064516129032258,0.819672131147541,0.9283837386900204,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator3 +1,30,29,21,8,9,0.7241379310344828,0.7,0.711864406779661,0.8393214575860368,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator3 +0,31,29,25,4,6,0.8620689655172413,0.8064516129032258,0.8333333333333334,0.9136476643259428,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator4 +1,30,28,23,5,7,0.8214285714285714,0.7666666666666667,0.793103448275862,0.834004341918102,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator4 +0,31,27,26,1,5,0.9629629629629629,0.8387096774193549,0.896551724137931,0.925882956841871,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator5 +1,30,26,26,0,4,1.0,0.8666666666666667,0.9285714285714286,0.8445197164355119,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator5 +0,31,25,23,2,8,0.92,0.7419354838709677,0.8214285714285714,0.9175744989869763,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator6 +1,30,24,22,2,8,0.9166666666666666,0.7333333333333333,0.8148148148148148,0.8178055538848443,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator6 +0,31,24,23,1,8,0.9583333333333334,0.7419354838709677,0.8363636363636364,0.9062019395481097,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator7 +1,30,23,22,1,8,0.9565217391304348,0.7333333333333333,0.8301886792452831,0.8584954520005996,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator7 +0,31,24,23,1,8,0.9583333333333334,0.7419354838709677,0.8363636363636364,0.8845805178640961,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator8 +1,30,23,21,2,9,0.9130434782608695,0.7,0.7924528301886793,0.7904925927182763,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator8 +0,30,29,25,4,5,0.8620689655172413,0.8333333333333334,0.847457627118644,0.9404619399500795,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator4 +1,29,28,23,5,6,0.8214285714285714,0.7931034482758621,0.8070175438596492,0.885532149792219,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator4 +0,30,27,25,2,5,0.9259259259259259,0.8333333333333334,0.8771929824561403,0.9551546935553504,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator5 +1,29,26,22,4,7,0.8461538461538461,0.7586206896551724,0.8,0.9134716190540373,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator5 +0,30,25,23,2,7,0.92,0.7666666666666667,0.8363636363636363,0.961287805306342,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator6 +1,29,24,21,3,8,0.875,0.7241379310344828,0.7924528301886793,0.8877749309585289,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator6 +0,30,24,24,0,6,1.0,0.8,0.888888888888889,0.9604256224434629,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator7 +1,29,23,21,2,8,0.9130434782608695,0.7241379310344828,0.8076923076923076,0.9185745067898103,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator7 +0,30,24,23,1,7,0.9583333333333334,0.7666666666666667,0.8518518518518519,0.9150994354785018,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator8 +1,29,23,19,4,10,0.8260869565217391,0.6551724137931034,0.7307692307692308,0.8339809156150628,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator8 +0,29,27,25,2,4,0.9259259259259259,0.8620689655172413,0.8928571428571429,0.9522005728206482,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator5 +1,28,26,22,4,6,0.8461538461538461,0.7857142857142857,0.8148148148148148,0.9044245000641673,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator5 +0,29,25,24,1,5,0.96,0.8275862068965517,0.888888888888889,0.9378514712857612,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator6 +1,28,24,21,3,7,0.875,0.75,0.8076923076923077,0.8774262460264158,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator6 +0,29,24,24,0,5,1.0,0.8275862068965517,0.9056603773584906,0.9614869699718609,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator7 +1,28,23,21,2,7,0.9130434782608695,0.75,0.8235294117647057,0.8974765702574666,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator7 +0,29,24,24,0,5,1.0,0.8275862068965517,0.9056603773584906,0.9188661930778087,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator8 +1,28,23,20,3,8,0.8695652173913043,0.7142857142857143,0.7843137254901961,0.8124881152132456,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator8 +0,27,25,25,0,2,1.0,0.9259259259259259,0.9615384615384615,0.9440718645678,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator6 +1,26,24,23,1,3,0.9583333333333334,0.8846153846153846,0.9199999999999999,0.8754448772114375,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator6 +0,27,24,24,0,3,1.0,0.8888888888888888,0.9411764705882353,0.9528520313881114,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator7 +1,26,23,22,1,4,0.9565217391304348,0.8461538461538461,0.8979591836734695,0.9330469843466431,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator7 +0,27,24,24,0,3,1.0,0.8888888888888888,0.9411764705882353,0.9275612240471826,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator8 +1,26,23,22,1,4,0.9565217391304348,0.8461538461538461,0.8979591836734695,0.8250318385215255,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator8 +0,25,24,23,1,2,0.9583333333333334,0.92,0.9387755102040817,0.9363162149353698,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator7 +1,24,23,22,1,2,0.9565217391304348,0.9166666666666666,0.9361702127659574,0.9077689274353961,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator7 +0,25,24,23,1,2,0.9583333333333334,0.92,0.9387755102040817,0.8996419564110354,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator8 +1,24,23,19,4,5,0.8260869565217391,0.7916666666666666,0.8085106382978724,0.8374249976625746,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator8 +0,24,24,23,1,1,0.9583333333333334,0.9583333333333334,0.9583333333333334,0.9197579382356681,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator7,annotator8 +1,23,23,20,3,3,0.8695652173913043,0.8695652173913043,0.8695652173913043,0.8358815888380858,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator7,annotator8 +0,10,10,10,0,0,1.0,1.0,1.0,0.9536614039418785,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator2 +1,10,10,10,0,0,1.0,1.0,1.0,0.9413937869082349,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator2 +0,10,12,10,2,0,0.8333333333333334,1.0,0.9090909090909091,0.9567206983563812,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator3 +1,10,12,10,2,0,0.8333333333333334,1.0,0.9090909090909091,0.9028056740393946,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator3 +0,10,15,9,6,1,0.6,0.9,0.7200000000000001,0.8940662263602681,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator4 +1,10,15,10,5,0,0.6666666666666666,1.0,0.8,0.9108129509460443,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator4 +0,10,2,0,2,10,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator5 +1,10,1,0,1,10,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator5 +0,10,10,9,1,1,0.9,0.9,0.9,0.9892403664326577,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator6 +1,10,10,10,0,0,1.0,1.0,1.0,0.9387207989181151,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator6 +0,10,9,9,0,1,1.0,0.9,0.9473684210526316,0.9163255352270944,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator7 +1,10,9,9,0,1,1.0,0.9,0.9473684210526316,0.9423529896509419,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator7 +0,10,10,10,0,0,1.0,1.0,1.0,0.926781137234947,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator8 +1,10,10,9,1,1,0.9,0.9,0.9,0.8796482835645562,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator8 +0,10,12,10,2,0,0.8333333333333334,1.0,0.9090909090909091,0.9176418102482288,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator3 +1,10,12,10,2,0,0.8333333333333334,1.0,0.9090909090909091,0.8739295280413423,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator3 +0,10,15,10,5,0,0.6666666666666666,1.0,0.8,0.886744734223012,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator4 +1,10,15,10,5,0,0.6666666666666666,1.0,0.8,0.9200122055497439,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator4 +0,10,2,0,2,10,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator5 +1,10,1,0,1,10,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator5 +0,10,10,10,0,0,1.0,1.0,1.0,0.9606147795938815,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator6 +1,10,10,10,0,0,1.0,1.0,1.0,0.9374854263831504,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator6 +0,10,9,9,0,1,1.0,0.9,0.9473684210526316,0.9489321261806926,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator7 +1,10,9,9,0,1,1.0,0.9,0.9473684210526316,0.953947676951755,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator7 +0,10,10,10,0,0,1.0,1.0,1.0,0.950643262120116,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator8 +1,10,10,9,1,1,0.9,0.9,0.9,0.8808095999064309,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator8 +0,12,15,10,5,2,0.6666666666666666,0.8333333333333334,0.7407407407407408,0.9090078606576588,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator4 +1,12,15,10,5,2,0.6666666666666666,0.8333333333333334,0.7407407407407408,0.8387297124078291,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator4 +0,12,2,0,2,12,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator5 +1,12,1,0,1,12,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator5 +0,12,10,9,1,3,0.9,0.75,0.8181818181818182,0.9592143329789962,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator6 +1,12,10,10,0,2,1.0,0.8333333333333334,0.9090909090909091,0.8975762143362669,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator6 +0,12,9,9,0,3,1.0,0.75,0.8571428571428571,0.8850940764507308,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator7 +1,12,9,9,0,3,1.0,0.75,0.8571428571428571,0.9364983916367566,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator7 +0,12,10,10,0,2,1.0,0.8333333333333334,0.9090909090909091,0.9017713707325896,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator8 +1,12,10,9,1,3,0.9,0.75,0.8181818181818182,0.8531357950973734,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator8 +0,15,2,0,2,15,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator5 +1,15,1,0,1,15,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator5 +0,15,10,10,0,5,1.0,0.6666666666666666,0.8,0.887237174181864,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator6 +1,15,10,10,0,5,1.0,0.6666666666666666,0.8,0.9027417370019608,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator6 +0,15,9,8,1,7,0.8888888888888888,0.5333333333333333,0.6666666666666667,0.8947708589705285,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator7 +1,15,9,9,0,6,1.0,0.6,0.7499999999999999,0.9301901268344095,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator7 +0,15,10,10,0,5,1.0,0.6666666666666666,0.8,0.8659611824786241,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator8 +1,15,10,8,2,7,0.8,0.5333333333333333,0.64,0.923184217087739,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator8 +0,2,10,0,10,2,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator6 +1,1,10,0,10,1,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator6 +0,2,9,0,9,2,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator7 +1,1,9,0,9,1,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator7 +0,2,10,0,10,2,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator8 +1,1,10,0,10,1,0.0,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator8 +0,10,9,9,0,1,1.0,0.9,0.9473684210526316,0.9278523447269093,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator7 +1,10,9,9,0,1,1.0,0.9,0.9473684210526316,0.9535307206247327,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator7 +0,10,10,10,0,0,1.0,1.0,1.0,0.9577906539448385,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator8 +1,10,10,9,1,1,0.9,0.9,0.9,0.9109094930760239,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator8 +0,9,10,9,1,0,0.9,1.0,0.9473684210526316,0.9276493233974773,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator7,annotator8 +1,9,10,9,1,0,0.9,1.0,0.9473684210526316,0.9021670743456804,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator7,annotator8 +0,2,5,1,4,1,0.2,0.5,0.28571428571428575,0.5874962904342665,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator2 +1,1,5,0,5,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator2 +0,2,5,1,4,1,0.2,0.5,0.28571428571428575,0.5869737102194109,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator3 +1,1,5,0,5,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator3 +0,2,6,1,5,1,0.16666666666666666,0.5,0.25,0.5873800810997923,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator4 +1,1,6,0,6,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator4 +0,2,2,0,2,2,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator5 +1,1,1,0,1,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator5 +0,2,7,1,6,1,0.14285714285714285,0.5,0.22222222222222224,0.8503722794959908,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator6 +1,1,7,0,7,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator6 +0,2,4,0,4,2,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator7 +1,1,4,0,4,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator7 +0,2,5,1,4,1,0.2,0.5,0.28571428571428575,0.5865679012345679,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator8 +1,1,5,0,5,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator8 +0,5,5,5,0,0,1.0,1.0,1.0,0.9830528327395438,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator3 +1,5,5,5,0,0,1.0,1.0,1.0,0.8842589765319818,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator3 +0,5,6,5,1,0,0.8333333333333334,1.0,0.9090909090909091,0.9139055054060645,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator4 +1,5,6,5,1,0,0.8333333333333334,1.0,0.9090909090909091,0.9267894924005265,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator4 +0,5,2,1,1,4,0.5,0.2,0.28571428571428575,0.6932994993484671,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator5 +1,5,1,0,1,5,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator5 +0,5,7,5,2,0,0.7142857142857143,1.0,0.8333333333333333,0.8413052486579156,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator6 +1,5,7,5,2,0,0.7142857142857143,1.0,0.8333333333333333,0.9014231261801516,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator6 +0,5,4,4,0,1,1.0,0.8,0.888888888888889,0.9460253359636137,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator7 +1,5,4,4,0,1,1.0,0.8,0.888888888888889,0.915061620353991,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator7 +0,5,5,5,0,0,1.0,1.0,1.0,0.9827172162670182,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator8 +1,5,5,5,0,0,1.0,1.0,1.0,0.8646496400823812,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator8 +0,5,6,5,1,0,0.8333333333333334,1.0,0.9090909090909091,0.8994229254014009,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator4 +1,5,6,5,1,0,0.8333333333333334,1.0,0.9090909090909091,0.8490500544326984,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator4 +0,5,2,1,1,4,0.5,0.2,0.28571428571428575,0.6939167409642686,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator5 +1,5,1,0,1,5,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator5 +0,5,7,5,2,0,0.7142857142857143,1.0,0.8333333333333333,0.8309678014189392,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator6 +1,5,7,5,2,0,0.7142857142857143,1.0,0.8333333333333333,0.9012323419952064,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator6 +0,5,4,4,0,1,1.0,0.8,0.888888888888889,0.9510778852943562,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator7 +1,5,4,4,0,1,1.0,0.8,0.888888888888889,0.9760649253903212,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator7 +0,5,5,5,0,0,1.0,1.0,1.0,0.9785054584892681,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator8 +1,5,5,5,0,0,1.0,1.0,1.0,0.8873968690396374,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator8 +0,6,2,1,1,5,0.5,0.16666666666666666,0.25,0.6934366641519786,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator5 +1,6,1,0,1,6,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator5 +0,6,7,5,2,1,0.7142857142857143,0.8333333333333334,0.7692307692307692,0.7703258087673381,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator6 +1,6,7,5,2,1,0.7142857142857143,0.8333333333333334,0.7692307692307692,0.9124234165564917,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator6 +0,6,4,4,0,2,1.0,0.6666666666666666,0.8,0.8397023084298428,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator7 +1,6,4,4,0,2,1.0,0.6666666666666666,0.8,0.8466562590037736,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator7 +0,6,5,5,0,1,1.0,0.8333333333333334,0.9090909090909091,0.9095533512570096,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator8 +1,6,5,5,0,1,1.0,0.8333333333333334,0.9090909090909091,0.81741656704657,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator8 +0,2,7,0,7,2,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator6 +1,1,7,0,7,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator6 +0,2,4,1,3,1,0.25,0.5,0.3333333333333333,0.8304643028598861,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator7 +1,1,4,0,4,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator7 +0,2,5,1,4,1,0.2,0.5,0.28571428571428575,0.6943968177765585,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator8 +1,1,5,0,5,1,0.0,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator8 +0,7,4,4,0,3,1.0,0.5714285714285714,0.7272727272727273,0.8745548661664909,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator7 +1,7,4,4,0,3,1.0,0.5714285714285714,0.7272727272727273,0.9018687554595739,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator7 +0,7,5,5,0,2,1.0,0.7142857142857143,0.8333333333333333,0.8400353868919952,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator8 +1,7,5,5,0,2,1.0,0.7142857142857143,0.8333333333333333,0.8893752832864916,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator8 +0,4,5,4,1,0,0.8,1.0,0.888888888888889,0.9412317647497006,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator7,annotator8 +1,4,5,4,1,0,0.8,1.0,0.888888888888889,0.9589553057809542,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator7,annotator8 +0,43,39,36,3,7,0.9230769230769231,0.8372093023255814,0.878048780487805,0.947617116812986,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator2 +1,42,38,35,3,7,0.9210526315789473,0.8333333333333334,0.875,0.8515628120204687,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator2 +0,43,42,39,3,4,0.9285714285714286,0.9069767441860465,0.9176470588235294,0.932840998890436,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator3 +1,42,41,34,7,8,0.8292682926829268,0.8095238095238095,0.8192771084337348,0.8216569249564971,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator3 +0,43,42,38,4,5,0.9047619047619048,0.8837209302325582,0.8941176470588236,0.9500823084825004,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator4 +1,42,41,37,4,5,0.9024390243902439,0.8809523809523809,0.8915662650602411,0.793643962997207,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator4 +0,43,37,37,0,6,1.0,0.8604651162790697,0.9249999999999999,0.9367563659810282,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator5 +1,42,36,35,1,7,0.9722222222222222,0.8333333333333334,0.8974358974358975,0.8542145580681924,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator5 +0,43,38,38,0,5,1.0,0.8837209302325582,0.9382716049382717,0.936031264843296,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator6 +1,42,37,35,2,7,0.9459459459459459,0.8333333333333334,0.8860759493670887,0.8465355518881902,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator6 +0,43,16,12,4,31,0.75,0.27906976744186046,0.4067796610169491,0.9046433176373814,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator7 +1,42,15,15,0,27,1.0,0.35714285714285715,0.5263157894736842,0.863416432617893,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator7 +0,43,30,30,0,13,1.0,0.6976744186046512,0.8219178082191781,0.8708754550475533,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator8 +1,42,29,22,7,20,0.7586206896551724,0.5238095238095238,0.6197183098591549,0.767137075342983,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator8 +0,39,42,37,5,2,0.8809523809523809,0.9487179487179487,0.9135802469135802,0.9559986983157045,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator3 +1,38,41,31,10,7,0.7560975609756098,0.8157894736842105,0.7848101265822786,0.8160889437032296,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator3 +0,39,42,38,4,1,0.9047619047619048,0.9743589743589743,0.9382716049382716,0.9376728805023898,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator4 +1,38,41,38,3,0,0.926829268292683,1.0,0.9620253164556963,0.8293442888432841,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator4 +0,39,37,35,2,4,0.9459459459459459,0.8974358974358975,0.9210526315789475,0.9309781294597979,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator5 +1,38,36,32,4,6,0.8888888888888888,0.8421052631578947,0.8648648648648649,0.8184899965889777,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator5 +0,39,38,35,3,4,0.9210526315789473,0.8974358974358975,0.9090909090909091,0.9327881681048037,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator6 +1,38,37,32,5,6,0.8648648648648649,0.8421052631578947,0.8533333333333334,0.8355739781672069,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator6 +0,39,16,13,3,26,0.8125,0.3333333333333333,0.4727272727272727,0.8800994093475085,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator7 +1,38,15,15,0,23,1.0,0.39473684210526316,0.5660377358490566,0.8384481068850331,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator7 +0,39,30,30,0,9,1.0,0.7692307692307693,0.8695652173913044,0.8784180105337487,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator8 +1,38,29,21,8,17,0.7241379310344828,0.5526315789473685,0.6268656716417911,0.7257613191963544,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator8 +0,42,42,38,4,4,0.9047619047619048,0.9047619047619048,0.9047619047619048,0.9378313647010672,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator4 +1,41,41,31,10,10,0.7560975609756098,0.7560975609756098,0.7560975609756099,0.7575981791720577,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator4 +0,42,37,36,1,6,0.972972972972973,0.8571428571428571,0.9113924050632912,0.9395227369148369,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator5 +1,41,36,32,4,9,0.8888888888888888,0.7804878048780488,0.8311688311688312,0.8006380758333895,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator5 +0,42,38,37,1,5,0.9736842105263158,0.8809523809523809,0.925,0.9279221885734888,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator6 +1,41,37,32,5,9,0.8648648648648649,0.7804878048780488,0.8205128205128206,0.8072343352086867,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator6 +0,42,16,13,3,29,0.8125,0.30952380952380953,0.4482758620689655,0.8778144240925218,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator7 +1,41,15,14,1,27,0.9333333333333333,0.34146341463414637,0.5,0.883441245930468,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator7 +0,42,30,30,0,12,1.0,0.7142857142857143,0.8333333333333333,0.8826617644241683,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator8 +1,41,29,20,9,21,0.6896551724137931,0.4878048780487805,0.5714285714285714,0.7342595172265823,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator8 +0,42,37,35,2,7,0.9459459459459459,0.8333333333333334,0.8860759493670887,0.9284159185123754,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator5 +1,41,36,33,3,8,0.9166666666666666,0.8048780487804879,0.8571428571428571,0.7791880282734401,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator5 +0,42,38,37,1,5,0.9736842105263158,0.8809523809523809,0.925,0.9391208172019526,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator6 +1,41,37,33,4,8,0.8918918918918919,0.8048780487804879,0.8461538461538461,0.8251787826177679,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator6 +0,42,16,12,4,30,0.75,0.2857142857142857,0.4137931034482759,0.8873849526282614,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator7 +1,41,15,15,0,26,1.0,0.36585365853658536,0.5357142857142857,0.7977178152240757,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator7 +0,42,30,29,1,13,0.9666666666666667,0.6904761904761905,0.8055555555555556,0.8685674204202213,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator8 +1,41,29,19,10,22,0.6551724137931034,0.4634146341463415,0.5428571428571429,0.7521616258835195,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator8 +0,37,38,36,2,1,0.9473684210526315,0.972972972972973,0.9599999999999999,0.9198126325689877,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator6 +1,36,37,33,4,3,0.8918918918918919,0.9166666666666666,0.9041095890410958,0.8220312476205692,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator6 +0,37,16,14,2,23,0.875,0.3783783783783784,0.5283018867924528,0.8557122990295747,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator7 +1,36,15,15,0,21,1.0,0.4166666666666667,0.5882352941176471,0.874858520910106,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator7 +0,37,30,30,0,7,1.0,0.8108108108108109,0.8955223880597014,0.9082417059667066,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator8 +1,36,29,23,6,13,0.7931034482758621,0.6388888888888888,0.7076923076923076,0.7901277491253365,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator8 +0,38,16,13,3,25,0.8125,0.34210526315789475,0.4814814814814815,0.8757112857452926,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator7 +1,37,15,15,0,22,1.0,0.40540540540540543,0.5769230769230769,0.8450956975420935,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator7 +0,38,30,30,0,8,1.0,0.7894736842105263,0.8823529411764706,0.8879251264108207,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator8 +1,37,29,22,7,15,0.7586206896551724,0.5945945945945946,0.6666666666666667,0.7593475953177528,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator8 +0,16,30,14,16,2,0.4666666666666667,0.875,0.608695652173913,0.8342417120249133,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator7,annotator8 +1,15,29,12,17,3,0.41379310344827586,0.8,0.5454545454545454,0.8186111391925214,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator7,annotator8 +0,18,17,17,0,1,1.0,0.9444444444444444,0.9714285714285714,0.909112794572299,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator2 +1,17,16,16,0,1,1.0,0.9411764705882353,0.9696969696969697,0.8004313993504272,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator2 +0,18,20,17,3,1,0.85,0.9444444444444444,0.8947368421052632,0.9544128124588328,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator3 +1,17,19,15,4,2,0.7894736842105263,0.8823529411764706,0.8333333333333333,0.84768822572865,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator3 +0,18,16,16,0,2,1.0,0.8888888888888888,0.9411764705882353,0.915023800408177,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator4 +1,17,15,13,2,4,0.8666666666666667,0.7647058823529411,0.8125,0.8274076290048245,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator4 +0,18,3,2,1,16,0.6666666666666666,0.1111111111111111,0.1904761904761905,0.7142120975193689,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator5 +1,17,2,0,2,17,0.0,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator5 +0,18,16,16,0,2,1.0,0.8888888888888888,0.9411764705882353,0.8948114688447082,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator6 +1,17,15,14,1,3,0.9333333333333333,0.8235294117647058,0.8749999999999999,0.8113383832895196,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator6 +0,18,6,4,2,14,0.6666666666666666,0.2222222222222222,0.3333333333333333,0.8944066860313604,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator7 +1,17,5,5,0,12,1.0,0.29411764705882354,0.45454545454545453,0.9429206663360106,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator7 +0,18,16,15,1,3,0.9375,0.8333333333333334,0.8823529411764706,0.8051505802326946,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator8 +1,17,15,9,6,8,0.6,0.5294117647058824,0.5625,0.6814506028366745,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator8 +0,17,20,16,4,1,0.8,0.9411764705882353,0.8648648648648648,0.9217618717278271,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator3 +1,16,19,14,5,2,0.7368421052631579,0.875,0.7999999999999999,0.8232607258514433,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator3 +0,17,16,15,1,2,0.9375,0.8823529411764706,0.9090909090909091,0.9176583655477846,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator4 +1,16,15,13,2,3,0.8666666666666667,0.8125,0.8387096774193549,0.8687959283805916,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator4 +0,17,3,2,1,15,0.6666666666666666,0.11764705882352941,0.2,0.7216718607186512,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator5 +1,16,2,0,2,16,0.0,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator5 +0,17,16,16,0,1,1.0,0.9411764705882353,0.9696969696969697,0.9151127306878744,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator6 +1,16,15,14,1,2,0.9333333333333333,0.875,0.9032258064516129,0.8788792870205162,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator6 +0,17,6,4,2,13,0.6666666666666666,0.23529411764705882,0.3478260869565218,0.8579749118472093,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator7 +1,16,5,5,0,11,1.0,0.3125,0.47619047619047616,0.8308451865594723,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator7 +0,17,16,14,2,3,0.875,0.8235294117647058,0.8484848484848485,0.8060596047664126,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator8 +1,16,15,7,8,9,0.4666666666666667,0.4375,0.45161290322580644,0.703186786445124,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator8 +0,20,16,16,0,4,1.0,0.8,0.888888888888889,0.9107195049109813,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator4 +1,19,15,12,3,7,0.8,0.631578947368421,0.7058823529411765,0.8151396335932507,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator4 +0,20,3,2,1,18,0.6666666666666666,0.1,0.1739130434782609,0.7111335693235614,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator5 +1,19,2,0,2,19,0.0,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator5 +0,20,16,15,1,5,0.9375,0.75,0.8333333333333334,0.908064701867282,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator6 +1,19,15,13,2,6,0.8666666666666667,0.6842105263157895,0.7647058823529413,0.8317110748141734,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator6 +0,20,6,4,2,16,0.6666666666666666,0.2,0.30769230769230765,0.8750465634986316,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator7 +1,19,5,5,0,14,1.0,0.2631578947368421,0.4166666666666667,0.8915980151274269,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator7 +0,20,16,14,2,6,0.875,0.7,0.7777777777777777,0.8136596702698266,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator8 +1,19,15,6,9,13,0.4,0.3157894736842105,0.35294117647058826,0.7014299571849075,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator8 +0,16,3,2,1,14,0.6666666666666666,0.125,0.21052631578947367,0.7137784612460529,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator5 +1,15,2,0,2,15,0.0,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator5 +0,16,16,15,1,1,0.9375,0.9375,0.9375,0.9482077865600119,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator6 +1,15,15,12,3,3,0.8,0.8,0.8000000000000002,0.8168989142767292,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator6 +0,16,6,5,1,11,0.8333333333333334,0.3125,0.45454545454545453,0.8432260313742365,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator7 +1,15,5,5,0,10,1.0,0.3333333333333333,0.5,0.8313880183898668,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator7 +0,16,16,14,2,2,0.875,0.875,0.875,0.835449058648657,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator8 +1,15,15,6,9,9,0.4,0.4,0.4000000000000001,0.7299947007948426,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator8 +0,3,16,2,14,1,0.125,0.6666666666666666,0.21052631578947367,0.7133875308316666,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator6 +1,2,15,0,15,2,0.0,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator6 +0,3,6,2,4,1,0.3333333333333333,0.6666666666666666,0.4444444444444444,0.581224281918456,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator7 +1,2,5,0,5,2,0.0,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator7 +0,3,16,2,14,1,0.125,0.6666666666666666,0.21052631578947367,0.6201715145337041,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator8 +1,2,15,0,15,2,0.0,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator8 +0,16,6,5,1,11,0.8333333333333334,0.3125,0.45454545454545453,0.8162234887215527,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator7 +1,15,5,5,0,10,1.0,0.3333333333333333,0.5,0.8428775410362755,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator7 +0,16,16,14,2,2,0.875,0.875,0.875,0.8486685172562899,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator8 +1,15,15,8,7,7,0.5333333333333333,0.5333333333333333,0.5333333333333333,0.7111758096883769,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator8 +0,6,16,5,11,1,0.3125,0.8333333333333334,0.45454545454545453,0.7806293032385186,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator7,annotator8 +1,5,15,5,10,0,0.3333333333333333,1.0,0.5,0.6984625909182649,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator7,annotator8 +0,19,21,14,7,5,0.6666666666666666,0.7368421052631579,0.7,0.8739332346189098,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator2 +1,18,20,10,10,8,0.5,0.5555555555555556,0.5263157894736842,0.9016553524257931,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator2 +0,19,26,17,9,2,0.6538461538461539,0.8947368421052632,0.7555555555555555,0.8519289403447438,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator3 +1,18,25,11,14,7,0.44,0.6111111111111112,0.5116279069767442,0.7370333894294727,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator3 +0,19,23,14,9,5,0.6086956521739131,0.7368421052631579,0.6666666666666666,0.9122469095181277,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator4 +1,18,22,12,10,6,0.5454545454545454,0.6666666666666666,0.6,0.7517758349300059,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator4 +0,19,29,16,13,3,0.5517241379310345,0.8421052631578947,0.6666666666666666,0.8751171393112047,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator5 +1,18,28,10,18,8,0.35714285714285715,0.5555555555555556,0.43478260869565216,0.7647902951437866,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator5 +0,19,33,16,17,3,0.48484848484848486,0.8421052631578947,0.6153846153846154,0.8629482664174097,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator6 +1,18,32,7,25,11,0.21875,0.3888888888888889,0.28,0.759300967092306,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator6 +0,19,9,9,0,10,1.0,0.47368421052631576,0.6428571428571429,0.8857575789511518,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator7 +1,18,8,4,4,14,0.5,0.2222222222222222,0.30769230769230765,0.7403317088696282,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator7 +0,19,20,11,9,8,0.55,0.5789473684210527,0.5641025641025641,0.8146773484011756,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator8 +1,18,19,8,11,10,0.42105263157894735,0.4444444444444444,0.43243243243243246,0.7071459293892912,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator8 +0,21,26,17,9,4,0.6538461538461539,0.8095238095238095,0.7234042553191489,0.8493554066126019,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator3 +1,20,25,13,12,7,0.52,0.65,0.5777777777777778,0.8007833447181543,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator3 +0,21,23,14,9,7,0.6086956521739131,0.6666666666666666,0.6363636363636365,0.8622847087703382,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator4 +1,20,22,14,8,6,0.6363636363636364,0.7,0.6666666666666666,0.8119840324867299,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator4 +0,21,29,17,12,4,0.5862068965517241,0.8095238095238095,0.68,0.8573649232916803,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator5 +1,20,28,11,17,9,0.39285714285714285,0.55,0.45833333333333337,0.8346109254555949,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator5 +0,21,33,15,18,6,0.45454545454545453,0.7142857142857143,0.5555555555555556,0.8435035579959085,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator6 +1,20,32,11,21,9,0.34375,0.55,0.42307692307692313,0.7639146471737771,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator6 +0,21,9,8,1,13,0.8888888888888888,0.38095238095238093,0.5333333333333333,0.8882557035343703,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator7 +1,20,8,4,4,16,0.5,0.2,0.28571428571428575,0.6971646613190731,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator7 +0,21,20,16,4,5,0.8,0.7619047619047619,0.7804878048780488,0.7809169417179109,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator8 +1,20,19,12,7,8,0.631578947368421,0.6,0.6153846153846154,0.7526032579537228,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator8 +0,26,23,14,9,12,0.6086956521739131,0.5384615384615384,0.5714285714285715,0.8658892279757758,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator4 +1,25,22,10,12,15,0.45454545454545453,0.4,0.4255319148936171,0.7741702215342483,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator4 +0,26,29,18,11,8,0.6206896551724138,0.6923076923076923,0.6545454545454545,0.8852578063043516,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator5 +1,25,28,13,15,12,0.4642857142857143,0.52,0.49056603773584906,0.8153553683038294,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator5 +0,26,33,20,13,6,0.6060606060606061,0.7692307692307693,0.6779661016949152,0.8497840609457995,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator6 +1,25,32,14,18,11,0.4375,0.56,0.4912280701754386,0.7805546699632425,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator6 +0,26,9,9,0,17,1.0,0.34615384615384615,0.5142857142857142,0.9391189626453058,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator7 +1,25,8,3,5,22,0.375,0.12,0.18181818181818182,0.8726784539960667,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator7 +0,26,20,13,7,13,0.65,0.5,0.5652173913043479,0.7836480056777009,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator8 +1,25,19,8,11,17,0.42105263157894735,0.32,0.3636363636363636,0.7001946691889885,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator8 +0,23,29,17,12,6,0.5862068965517241,0.7391304347826086,0.6538461538461539,0.8734463551154517,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator5 +1,22,28,14,14,8,0.5,0.6363636363636364,0.56,0.7720516249115462,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator5 +0,23,33,16,17,7,0.48484848484848486,0.6956521739130435,0.5714285714285715,0.8645880493949335,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator6 +1,22,32,9,23,13,0.28125,0.4090909090909091,0.3333333333333333,0.7939796031150729,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator6 +0,23,9,8,1,15,0.8888888888888888,0.34782608695652173,0.5,0.8859741395881233,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator7 +1,22,8,2,6,20,0.25,0.09090909090909091,0.13333333333333333,0.7536511231373713,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator7 +0,23,20,12,8,11,0.6,0.5217391304347826,0.5581395348837209,0.7515468837909548,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator8 +1,22,19,12,7,10,0.631578947368421,0.5454545454545454,0.5853658536585366,0.7744571797461969,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator8 +0,29,33,20,13,9,0.6060606060606061,0.6896551724137931,0.6451612903225807,0.8795967501868717,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator6 +1,28,32,17,15,11,0.53125,0.6071428571428571,0.5666666666666667,0.7645888832743615,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator6 +0,29,9,9,0,20,1.0,0.3103448275862069,0.4736842105263158,0.9377838953158888,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator7 +1,28,8,3,5,25,0.375,0.10714285714285714,0.16666666666666666,0.9027434572777265,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator7 +0,29,20,17,3,12,0.85,0.5862068965517241,0.6938775510204082,0.7608446000034194,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator8 +1,28,19,14,5,14,0.7368421052631579,0.5,0.5957446808510638,0.7926027081004244,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator8 +0,33,9,8,1,25,0.8888888888888888,0.24242424242424243,0.38095238095238093,0.912603563592667,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator7 +1,32,8,2,6,30,0.25,0.0625,0.1,0.6483918128654971,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator7 +0,33,20,14,6,19,0.7,0.42424242424242425,0.5283018867924527,0.7816709208423921,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator8 +1,32,19,10,9,22,0.5263157894736842,0.3125,0.39215686274509803,0.6714200188974667,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator8 +0,9,20,8,12,1,0.4,0.8888888888888888,0.5517241379310346,0.8904074960173047,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator7,annotator8 +1,8,19,3,16,5,0.15789473684210525,0.375,0.22222222222222218,0.9080452411438328,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator7,annotator8 +0,24,14,13,1,11,0.9285714285714286,0.5416666666666666,0.6842105263157894,0.8776063510473898,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator2 +1,23,13,9,4,14,0.6923076923076923,0.391304347826087,0.5,0.7682826308978479,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator2 +0,24,24,18,6,6,0.75,0.75,0.75,0.8775671209889334,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator3 +1,23,23,15,8,8,0.6521739130434783,0.6521739130434783,0.6521739130434783,0.8270792322028853,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator3 +0,24,22,16,6,8,0.7272727272727273,0.6666666666666666,0.6956521739130435,0.8741741650372195,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator4 +1,23,21,13,8,10,0.6190476190476191,0.5652173913043478,0.5909090909090909,0.8132078054966456,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator4 +0,24,32,22,10,2,0.6875,0.9166666666666666,0.7857142857142857,0.8870386148210714,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator5 +1,23,31,18,13,5,0.5806451612903226,0.782608695652174,0.6666666666666667,0.828927285668673,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator5 +0,24,23,20,3,4,0.8695652173913043,0.8333333333333334,0.851063829787234,0.8615738740544572,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator6 +1,23,22,18,4,5,0.8181818181818182,0.782608695652174,0.8,0.828410153362565,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator6 +0,24,8,8,0,16,1.0,0.3333333333333333,0.5,0.9420813755229489,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator7 +1,23,7,4,3,19,0.5714285714285714,0.17391304347826086,0.26666666666666666,0.6275854065572515,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator7 +0,24,22,17,5,7,0.7727272727272727,0.7083333333333334,0.7391304347826088,0.7955727659762966,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator8 +1,23,21,15,6,8,0.7142857142857143,0.6521739130434783,0.6818181818181819,0.7289039241731883,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator8 +0,14,24,12,12,2,0.5,0.8571428571428571,0.631578947368421,0.8383097016755054,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator3 +1,13,23,6,17,7,0.2608695652173913,0.46153846153846156,0.33333333333333337,0.8224039185632431,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator3 +0,14,22,11,11,3,0.5,0.7857142857142857,0.6111111111111112,0.8749760269628268,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator4 +1,13,21,7,14,6,0.3333333333333333,0.5384615384615384,0.41176470588235287,0.698219059241181,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator4 +0,14,32,13,19,1,0.40625,0.9285714285714286,0.5652173913043478,0.8836467078558753,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator5 +1,13,31,6,25,7,0.1935483870967742,0.46153846153846156,0.2727272727272727,0.7469343721414591,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator5 +0,14,23,13,10,1,0.5652173913043478,0.9285714285714286,0.7027027027027025,0.8485563157372149,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator6 +1,13,22,9,13,4,0.4090909090909091,0.6923076923076923,0.5142857142857142,0.7484806260460249,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator6 +0,14,8,8,0,6,1.0,0.5714285714285714,0.7272727272727273,0.878432345867622,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator7 +1,13,7,5,2,8,0.7142857142857143,0.38461538461538464,0.5,0.7729335958676645,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator7 +0,14,22,11,11,3,0.5,0.7857142857142857,0.6111111111111112,0.8642583290572218,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator8 +1,13,21,6,15,7,0.2857142857142857,0.46153846153846156,0.35294117647058826,0.843418425999562,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator8 +0,24,22,14,8,10,0.6363636363636364,0.5833333333333334,0.6086956521739131,0.847770842219451,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator4 +1,23,21,8,13,15,0.38095238095238093,0.34782608695652173,0.3636363636363636,0.8865885689866082,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator4 +0,24,32,20,12,4,0.625,0.8333333333333334,0.7142857142857143,0.8484960354266317,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator5 +1,23,31,16,15,7,0.5161290322580645,0.6956521739130435,0.5925925925925926,0.8057528962220507,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator5 +0,24,23,17,6,7,0.7391304347826086,0.7083333333333334,0.723404255319149,0.8759102306661495,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator6 +1,23,22,11,11,12,0.5,0.4782608695652174,0.4888888888888889,0.8653754223655067,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator6 +0,24,8,8,0,16,1.0,0.3333333333333333,0.5,0.9573151262139143,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator7 +1,23,7,2,5,21,0.2857142857142857,0.08695652173913043,0.13333333333333333,0.9578619909502263,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator7 +0,24,22,16,6,8,0.7272727272727273,0.6666666666666666,0.6956521739130435,0.7953822545594478,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator8 +1,23,21,14,7,9,0.6666666666666666,0.6086956521739131,0.6363636363636365,0.7486719264879668,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator8 +0,22,32,17,15,5,0.53125,0.7727272727272727,0.6296296296296297,0.871807355569663,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator5 +1,21,31,13,18,8,0.41935483870967744,0.6190476190476191,0.5,0.8096912214860338,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator5 +0,22,23,17,6,5,0.7391304347826086,0.7727272727272727,0.7555555555555555,0.8638060544122483,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator6 +1,21,22,14,8,7,0.6363636363636364,0.6666666666666666,0.6511627906976744,0.8261759892507711,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator6 +0,22,8,8,0,14,1.0,0.36363636363636365,0.5333333333333333,0.8596504798794136,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator7 +1,21,7,4,3,17,0.5714285714285714,0.19047619047619047,0.2857142857142857,0.7182074944185377,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator7 +0,22,22,10,12,12,0.45454545454545453,0.45454545454545453,0.45454545454545453,0.824360483508016,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator8 +1,21,21,10,11,11,0.47619047619047616,0.47619047619047616,0.47619047619047616,0.7363894494210659,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator8 +0,32,23,21,2,11,0.9130434782608695,0.65625,0.7636363636363634,0.8511199300614904,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator6 +1,31,22,15,7,16,0.6818181818181818,0.4838709677419355,0.5660377358490567,0.7680041416637802,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator6 +0,32,8,8,0,24,1.0,0.25,0.4,0.9499639025233386,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator7 +1,31,7,3,4,28,0.42857142857142855,0.0967741935483871,0.15789473684210525,0.7223455791368076,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator7 +0,32,22,16,6,16,0.7272727272727273,0.5,0.5925925925925926,0.8438955207255775,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator8 +1,31,21,16,5,15,0.7619047619047619,0.5161290322580645,0.6153846153846153,0.7508778044801754,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator8 +0,23,8,8,0,15,1.0,0.34782608695652173,0.5161290322580645,0.9245603857854436,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator7 +1,22,7,4,3,18,0.5714285714285714,0.18181818181818182,0.27586206896551724,0.8128903964351015,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator7 +0,23,22,15,7,8,0.6818181818181818,0.6521739130434783,0.6666666666666666,0.8164902329527687,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator8 +1,22,21,14,7,8,0.6666666666666666,0.6363636363636364,0.6511627906976744,0.7553920373043795,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator8 +0,8,22,6,16,2,0.2727272727272727,0.75,0.39999999999999997,0.9442462381746818,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator7,annotator8 +1,7,21,2,19,5,0.09523809523809523,0.2857142857142857,0.14285714285714285,0.8126247639600755,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator7,annotator8 +0,64,47,38,9,26,0.8085106382978723,0.59375,0.6846846846846846,0.872697336013717,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator2 +1,64,46,30,16,34,0.6521739130434783,0.46875,0.5454545454545454,0.7600228418077476,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator2 +0,64,60,46,14,18,0.7666666666666667,0.71875,0.7419354838709677,0.8700425751391307,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator3 +1,64,59,40,19,24,0.6779661016949152,0.625,0.6504065040650406,0.7435058628077196,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator3 +0,64,52,38,14,26,0.7307692307692307,0.59375,0.6551724137931033,0.8355126634324379,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator4 +1,64,51,26,25,38,0.5098039215686274,0.40625,0.45217391304347826,0.7155990819544381,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator4 +0,64,65,46,19,18,0.7076923076923077,0.71875,0.7131782945736435,0.8417419269334354,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator5 +1,64,64,38,26,26,0.59375,0.59375,0.59375,0.7713515341550693,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator5 +0,64,43,37,6,27,0.8604651162790697,0.578125,0.6915887850467289,0.8690291500957831,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator6 +1,64,42,24,18,40,0.5714285714285714,0.375,0.4528301886792453,0.7920849486645221,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator6 +0,64,20,18,2,46,0.9,0.28125,0.4285714285714286,0.9346649898368986,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator7 +1,64,20,11,9,53,0.55,0.171875,0.2619047619047619,0.7448124598923044,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator7 +0,64,40,34,6,30,0.85,0.53125,0.6538461538461537,0.8434061604753981,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator8 +1,64,39,23,16,41,0.5897435897435898,0.359375,0.4466019417475728,0.7477808509288372,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator8 +0,47,60,43,17,4,0.7166666666666667,0.9148936170212766,0.8037383177570094,0.8772126647803735,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator3 +1,46,59,37,22,9,0.6271186440677966,0.8043478260869565,0.7047619047619047,0.8695780740382381,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator3 +0,47,52,35,17,12,0.6730769230769231,0.7446808510638298,0.7070707070707072,0.8327822767285861,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator4 +1,46,51,31,20,15,0.6078431372549019,0.6739130434782609,0.6391752577319587,0.7821313701146719,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator4 +0,47,65,43,22,4,0.6615384615384615,0.9148936170212766,0.7678571428571428,0.8836381254212582,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator5 +1,46,64,34,30,12,0.53125,0.7391304347826086,0.6181818181818182,0.7965275354507382,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator5 +0,47,43,36,7,11,0.8372093023255814,0.7659574468085106,0.8,0.8764919245969826,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator6 +1,46,42,29,13,17,0.6904761904761905,0.6304347826086957,0.6590909090909092,0.8107535368424483,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator6 +0,47,20,20,0,27,1.0,0.425531914893617,0.5970149253731344,0.9311419756511038,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator7 +1,46,20,15,5,31,0.75,0.32608695652173914,0.45454545454545453,0.8306493487392769,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator7 +0,47,40,35,5,12,0.875,0.7446808510638298,0.8045977011494252,0.8666670780671379,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator8 +1,46,39,30,9,16,0.7692307692307693,0.6521739130434783,0.7058823529411764,0.8033234486898079,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator8 +0,60,52,41,11,19,0.7884615384615384,0.6833333333333333,0.7321428571428571,0.8393265460055579,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator4 +1,59,51,36,15,23,0.7058823529411765,0.6101694915254238,0.6545454545454547,0.7809454707172853,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator4 +0,60,65,50,15,10,0.7692307692307693,0.8333333333333334,0.8,0.858156198810999,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator5 +1,59,64,41,23,18,0.640625,0.6949152542372882,0.6666666666666667,0.7998337003138344,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator5 +0,60,43,39,4,21,0.9069767441860465,0.65,0.7572815533980582,0.8730795325026028,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator6 +1,59,42,34,8,25,0.8095238095238095,0.576271186440678,0.6732673267326733,0.7800325526450628,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator6 +0,60,20,19,1,41,0.95,0.31666666666666665,0.4749999999999999,0.8971048203856566,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator7 +1,59,20,13,7,46,0.65,0.22033898305084745,0.3291139240506329,0.7958436913435519,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator7 +0,60,40,36,4,24,0.9,0.6,0.7200000000000001,0.8395853818730797,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator8 +1,59,39,30,9,29,0.7692307692307693,0.5084745762711864,0.6122448979591837,0.7730637004789414,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator8 +0,52,65,41,24,11,0.6307692307692307,0.7884615384615384,0.7008547008547009,0.7895267829142747,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator5 +1,51,64,29,35,22,0.453125,0.5686274509803921,0.5043478260869565,0.7207901992489762,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator5 +0,52,43,37,6,15,0.8604651162790697,0.7115384615384616,0.7789473684210527,0.8721574544864408,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator6 +1,51,42,27,15,24,0.6428571428571429,0.5294117647058824,0.5806451612903226,0.8187799207884737,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator6 +0,52,20,18,2,34,0.9,0.34615384615384615,0.5,0.8549558455055434,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator7 +1,51,20,14,6,37,0.7,0.27450980392156865,0.3943661971830986,0.7674941936848114,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator7 +0,52,40,31,9,21,0.775,0.5961538461538461,0.6739130434782609,0.7582500748595176,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator8 +1,51,39,25,14,26,0.6410256410256411,0.49019607843137253,0.5555555555555556,0.7573351445237131,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator8 +0,65,43,39,4,26,0.9069767441860465,0.6,0.7222222222222222,0.8775089794547767,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator6 +1,64,42,26,16,38,0.6190476190476191,0.40625,0.49056603773584906,0.7604380015343851,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator6 +0,65,20,20,0,45,1.0,0.3076923076923077,0.47058823529411764,0.8775394497413759,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator7 +1,64,20,13,7,51,0.65,0.203125,0.30952380952380953,0.7785569800321037,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator7 +0,65,40,37,3,28,0.925,0.5692307692307692,0.7047619047619047,0.8372373218307058,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator8 +1,64,39,23,16,41,0.5897435897435898,0.359375,0.4466019417475728,0.7552159047324601,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator8 +0,43,20,20,0,23,1.0,0.46511627906976744,0.6349206349206349,0.8780193815070568,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator7 +1,42,20,15,5,27,0.75,0.35714285714285715,0.48387096774193544,0.8166601943240686,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator7 +0,43,40,32,8,11,0.8,0.7441860465116279,0.7710843373493975,0.8124933126954783,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator8 +1,42,39,25,14,17,0.6410256410256411,0.5952380952380952,0.617283950617284,0.8022797658411972,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator8 +0,20,40,20,20,0,0.5,1.0,0.6666666666666666,0.8803608769802921,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator7,annotator8 +1,20,39,13,26,7,0.3333333333333333,0.65,0.4406779661016949,0.8281729888354119,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator7,annotator8 +0,43,40,37,3,6,0.925,0.8604651162790697,0.891566265060241,0.8602829972422352,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator2 +1,42,39,32,7,10,0.8205128205128205,0.7619047619047619,0.7901234567901233,0.80849273306778,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator2 +0,43,43,37,6,6,0.8604651162790697,0.8604651162790697,0.8604651162790697,0.8853813675279782,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator3 +1,42,42,33,9,9,0.7857142857142857,0.7857142857142857,0.7857142857142857,0.8262342138606994,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator3 +0,43,51,38,13,5,0.7450980392156863,0.8837209302325582,0.8085106382978724,0.910060034499149,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator4 +1,42,50,37,13,5,0.74,0.8809523809523809,0.8043478260869565,0.750302133903441,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator4 +0,43,54,38,16,5,0.7037037037037037,0.8837209302325582,0.7835051546391752,0.8734577239071922,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator5 +1,42,53,30,23,12,0.5660377358490566,0.7142857142857143,0.631578947368421,0.8013979281233298,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator5 +0,43,61,40,21,3,0.6557377049180327,0.9302325581395349,0.7692307692307692,0.9047077539207882,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator6 +1,42,60,34,26,8,0.5666666666666667,0.8095238095238095,0.6666666666666666,0.8068924115117729,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator6 +0,43,24,24,0,19,1.0,0.5581395348837209,0.7164179104477612,0.9003595029413468,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator7 +1,42,23,13,10,29,0.5652173913043478,0.30952380952380953,0.39999999999999997,0.7602999821943974,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator7 +0,43,31,30,1,13,0.967741935483871,0.6976744186046512,0.810810810810811,0.808408021038444,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator8 +1,42,30,24,6,18,0.8,0.5714285714285714,0.6666666666666666,0.7012111449987987,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator8 +0,40,43,34,9,6,0.7906976744186046,0.85,0.8192771084337349,0.8745001210205826,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator3 +1,39,42,29,13,10,0.6904761904761905,0.7435897435897436,0.7160493827160495,0.7800757361243574,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator3 +0,40,51,39,12,1,0.7647058823529411,0.975,0.857142857142857,0.8486830139602523,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator4 +1,39,50,32,18,7,0.64,0.8205128205128205,0.7191011235955057,0.7765312329163567,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator4 +0,40,54,36,18,4,0.6666666666666666,0.9,0.7659574468085106,0.8604861227602746,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator5 +1,39,53,34,19,5,0.6415094339622641,0.8717948717948718,0.7391304347826088,0.8004916109723546,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator5 +0,40,61,38,23,2,0.6229508196721312,0.95,0.7524752475247525,0.8325502822083339,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator6 +1,39,60,28,32,11,0.4666666666666667,0.717948717948718,0.5656565656565657,0.7733840434864582,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator6 +0,40,24,24,0,16,1.0,0.6,0.7499999999999999,0.8776479793257348,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator7 +1,39,23,15,8,24,0.6521739130434783,0.38461538461538464,0.4838709677419355,0.6971660933029714,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator7 +0,40,31,29,2,11,0.9354838709677419,0.725,0.8169014084507041,0.803985288981103,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator8 +1,39,30,22,8,17,0.7333333333333333,0.5641025641025641,0.6376811594202899,0.6919734311457075,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator8 +0,43,51,36,15,7,0.7058823529411765,0.8372093023255814,0.7659574468085107,0.866816842416989,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator4 +1,42,50,32,18,10,0.64,0.7619047619047619,0.6956521739130435,0.7965230918481158,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator4 +0,43,54,35,19,8,0.6481481481481481,0.813953488372093,0.7216494845360826,0.8458571771142893,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator5 +1,42,53,28,25,14,0.5283018867924528,0.6666666666666666,0.5894736842105263,0.7952436691560065,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator5 +0,43,61,38,23,5,0.6229508196721312,0.8837209302325582,0.7307692307692308,0.831354257580192,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator6 +1,42,60,31,29,11,0.5166666666666667,0.7380952380952381,0.607843137254902,0.7897470232289349,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator6 +0,43,24,24,0,19,1.0,0.5581395348837209,0.7164179104477612,0.8492329195022853,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator7 +1,42,23,8,15,34,0.34782608695652173,0.19047619047619047,0.24615384615384614,0.8073298595980829,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator7 +0,43,31,29,2,14,0.9354838709677419,0.6744186046511628,0.7837837837837838,0.8235084993518241,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator8 +1,42,30,22,8,20,0.7333333333333333,0.5238095238095238,0.611111111111111,0.6833530087708523,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator8 +0,51,54,43,11,8,0.7962962962962963,0.8431372549019608,0.8190476190476189,0.8472224273944884,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator5 +1,50,53,37,16,13,0.6981132075471698,0.74,0.7184466019417476,0.8157984378450361,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator5 +0,51,61,44,17,7,0.7213114754098361,0.8627450980392157,0.7857142857142857,0.8646837919837961,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator6 +1,50,60,40,20,10,0.6666666666666666,0.8,0.7272727272727272,0.7859575389761061,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator6 +0,51,24,24,0,27,1.0,0.47058823529411764,0.6399999999999999,0.9069254118481241,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator7 +1,50,23,13,10,37,0.5652173913043478,0.26,0.3561643835616438,0.8031310497933895,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator7 +0,51,31,27,4,24,0.8709677419354839,0.5294117647058824,0.6585365853658537,0.7939805941059509,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator8 +1,50,30,16,14,34,0.5333333333333333,0.32,0.4,0.7222734257377873,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator8 +0,54,61,46,15,8,0.7540983606557377,0.8518518518518519,0.7999999999999999,0.8802222826653334,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator6 +1,53,60,39,21,14,0.65,0.7358490566037735,0.6902654867256638,0.8200979953313519,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator6 +0,54,24,24,0,30,1.0,0.4444444444444444,0.6153846153846153,0.8660056068410021,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator7 +1,53,23,11,12,42,0.4782608695652174,0.20754716981132076,0.2894736842105263,0.805945104728918,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator7 +0,54,31,26,5,28,0.8387096774193549,0.48148148148148145,0.6117647058823529,0.7711359274221313,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator8 +1,53,30,17,13,36,0.5666666666666667,0.32075471698113206,0.40963855421686746,0.694423721388823,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator8 +0,61,24,24,0,37,1.0,0.39344262295081966,0.5647058823529412,0.9041158711119482,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator7 +1,60,23,10,13,50,0.43478260869565216,0.16666666666666666,0.24096385542168672,0.7667032154451514,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator7 +0,61,31,27,4,34,0.8709677419354839,0.4426229508196721,0.5869565217391305,0.7782615786432766,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator8 +1,60,30,17,13,43,0.5666666666666667,0.2833333333333333,0.3777777777777777,0.6533590041589011,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator8 +0,24,31,23,8,1,0.7419354838709677,0.9583333333333334,0.8363636363636364,0.7801952707570955,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator7,annotator8 +1,23,30,11,19,12,0.36666666666666664,0.4782608695652174,0.41509433962264153,0.6640770356969828,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator7,annotator8 +0,119,105,91,14,28,0.8666666666666667,0.7647058823529411,0.8125,0.8322075478770986,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator2 +1,118,104,72,32,46,0.6923076923076923,0.6101694915254238,0.6486486486486487,0.7906834755051815,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator2 +0,119,124,96,28,23,0.7741935483870968,0.8067226890756303,0.7901234567901235,0.848069385458213,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator3 +1,118,123,75,48,43,0.6097560975609756,0.635593220338983,0.6224066390041494,0.7756235233641562,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator3 +0,119,118,97,21,22,0.8220338983050848,0.8151260504201681,0.818565400843882,0.8300517980845281,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator4 +1,118,117,82,35,36,0.7008547008547008,0.6949152542372882,0.6978723404255319,0.7824330689822414,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator4 +0,119,150,110,40,9,0.7333333333333333,0.9243697478991597,0.8178438661710038,0.851842808895827,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator5 +1,118,149,85,64,33,0.5704697986577181,0.7203389830508474,0.6367041198501872,0.7636492857894949,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator5 +0,119,154,105,49,14,0.6818181818181818,0.8823529411764706,0.7692307692307693,0.8487732355939366,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator6 +1,118,153,76,77,42,0.49673202614379086,0.6440677966101694,0.5608856088560885,0.8113672338703134,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator6 +0,119,25,24,1,95,0.96,0.20168067226890757,0.33333333333333337,0.9018329165171316,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator7 +1,118,25,7,18,111,0.28,0.059322033898305086,0.0979020979020979,0.8564263620579082,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator7 +0,119,89,71,18,48,0.797752808988764,0.5966386554621849,0.6826923076923076,0.8197736457627263,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator8 +1,118,88,57,31,61,0.6477272727272727,0.4830508474576271,0.5533980582524272,0.7275293495946918,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator8 +0,105,124,90,34,15,0.7258064516129032,0.8571428571428571,0.7860262008733625,0.8763068240239797,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator3 +1,104,123,84,39,20,0.6829268292682927,0.8076923076923077,0.7400881057268723,0.8076240603362242,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator3 +0,105,118,93,25,12,0.788135593220339,0.8857142857142857,0.8340807174887892,0.8565413592231286,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator4 +1,104,117,90,27,14,0.7692307692307693,0.8653846153846154,0.8144796380090499,0.8265370447103414,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator4 +0,105,150,98,52,7,0.6533333333333333,0.9333333333333333,0.7686274509803921,0.8402050329990145,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator5 +1,104,149,71,78,33,0.47651006711409394,0.6826923076923077,0.5612648221343874,0.7723454538426465,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator5 +0,105,154,97,57,8,0.6298701298701299,0.9238095238095239,0.7490347490347491,0.8118087415612892,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator6 +1,104,153,67,86,37,0.43790849673202614,0.6442307692307693,0.5214007782101167,0.7690115089110438,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator6 +0,105,25,24,1,81,0.96,0.22857142857142856,0.3692307692307692,0.8911165362641057,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator7 +1,104,25,6,19,98,0.24,0.057692307692307696,0.09302325581395349,0.8137606954697549,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator7 +0,105,89,70,19,35,0.7865168539325843,0.6666666666666666,0.7216494845360824,0.8060105965442127,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator8 +1,104,88,61,27,43,0.6931818181818182,0.5865384615384616,0.6354166666666667,0.755170476179537,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator8 +0,124,118,96,22,28,0.8135593220338984,0.7741935483870968,0.7933884297520662,0.8686877963550771,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator4 +1,123,117,83,34,40,0.7094017094017094,0.6747967479674797,0.6916666666666667,0.8076452569856846,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator4 +0,124,150,102,48,22,0.68,0.8225806451612904,0.7445255474452556,0.8670606385902803,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator5 +1,123,149,77,72,46,0.5167785234899329,0.6260162601626016,0.5661764705882353,0.790835542292965,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator5 +0,124,154,104,50,20,0.6753246753246753,0.8387096774193549,0.7482014388489209,0.85229563374975,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator6 +1,123,153,83,70,40,0.5424836601307189,0.6747967479674797,0.6014492753623188,0.7722110819715192,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator6 +0,124,25,24,1,100,0.96,0.1935483870967742,0.3221476510067114,0.8784239003150226,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator7 +1,123,25,8,17,115,0.32,0.06504065040650407,0.10810810810810811,0.7820660035664058,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator7 +0,124,89,72,17,52,0.8089887640449438,0.5806451612903226,0.676056338028169,0.811325511679623,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator8 +1,123,88,59,29,64,0.6704545454545454,0.4796747967479675,0.5592417061611374,0.745414125673737,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator8 +0,118,150,105,45,13,0.7,0.8898305084745762,0.7835820895522386,0.867292357325988,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator5 +1,117,149,84,65,33,0.5637583892617449,0.717948717948718,0.631578947368421,0.7951064189060236,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator5 +0,118,154,111,43,7,0.7207792207792207,0.940677966101695,0.8161764705882353,0.8386290847507675,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator6 +1,117,153,77,76,40,0.5032679738562091,0.6581196581196581,0.5703703703703704,0.7962642354516781,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator6 +0,118,25,24,1,94,0.96,0.2033898305084746,0.33566433566433573,0.8890414976949561,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator7 +1,117,25,8,17,109,0.32,0.06837606837606838,0.11267605633802817,0.8004294641792492,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator7 +0,118,89,71,18,47,0.797752808988764,0.6016949152542372,0.6859903381642511,0.8141609556751176,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator8 +1,117,88,66,22,51,0.75,0.5641025641025641,0.6439024390243903,0.7202761393248569,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator8 +0,150,154,119,35,31,0.7727272727272727,0.7933333333333333,0.7828947368421053,0.8527933642178833,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator6 +1,149,153,92,61,57,0.6013071895424836,0.6174496644295302,0.609271523178808,0.7989951325235225,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator6 +0,150,25,24,1,126,0.96,0.16,0.2742857142857143,0.9082322006890297,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator7 +1,149,25,6,19,143,0.24,0.040268456375838924,0.06896551724137931,0.8294927251050684,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator7 +0,150,89,73,16,77,0.8202247191011236,0.4866666666666667,0.6108786610878661,0.7993870170545977,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator8 +1,149,88,45,43,104,0.5113636363636364,0.30201342281879195,0.37974683544303794,0.6902302484523228,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator8 +0,154,25,24,1,130,0.96,0.15584415584415584,0.2681564245810056,0.8712437598733693,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator7 +1,153,25,5,20,148,0.2,0.032679738562091505,0.05617977528089888,0.8503082288150763,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator7 +0,154,89,75,14,79,0.8426966292134831,0.487012987012987,0.6172839506172839,0.7988270192382636,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator8 +1,153,88,48,40,105,0.5454545454545454,0.3137254901960784,0.3983402489626556,0.6733785969940103,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator8 +0,25,89,23,66,2,0.25842696629213485,0.92,0.4035087719298246,0.854274099291516,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator7,annotator8 +1,25,88,6,82,19,0.06818181818181818,0.24,0.10619469026548671,0.8119845568597411,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator7,annotator8 +0,16,31,13,18,3,0.41935483870967744,0.8125,0.5531914893617021,0.8601646571113567,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator2 +1,15,30,7,23,8,0.23333333333333334,0.4666666666666667,0.31111111111111117,0.7229597447069809,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator2 +annotator_multi1,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator2 +annotator_multi2,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator2 +0,16,10,10,0,6,1.0,0.625,0.7692307692307693,0.8595903673555384,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator3 +1,15,9,6,3,9,0.6666666666666666,0.4,0.5,0.7457528423393983,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator3 +annotator_multi1,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator3 +0,16,13,12,1,4,0.9230769230769231,0.75,0.8275862068965517,0.916289346093098,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator4 +1,15,12,6,6,9,0.5,0.4,0.4444444444444445,0.8493778004194671,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator4 +annotator_multi1,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator4 +0,16,33,14,19,2,0.42424242424242425,0.875,0.5714285714285714,0.8155729759751537,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator5 +1,15,32,8,24,7,0.25,0.5333333333333333,0.3404255319148936,0.7808670733911863,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator5 +annotator_multi1,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator5 +0,16,23,14,9,2,0.6086956521739131,0.875,0.717948717948718,0.8994140733574932,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator6 +1,15,22,8,14,7,0.36363636363636365,0.5333333333333333,0.43243243243243246,0.7692726542591921,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator6 +annotator_multi1,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator6 +0,16,5,4,1,12,0.8,0.25,0.38095238095238093,0.8264416731894055,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator7 +1,15,4,2,2,13,0.5,0.13333333333333333,0.2105263157894737,0.7897995283018868,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator7 +annotator_multi1,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator7 +0,16,12,11,1,5,0.9166666666666666,0.6875,0.7857142857142857,0.8494938850208649,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator8 +1,15,11,5,6,10,0.45454545454545453,0.3333333333333333,0.3846153846153846,0.6757293301936158,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator8 +annotator_multi1,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator8 +0,31,10,9,1,22,0.9,0.2903225806451613,0.4390243902439024,0.8040363435372065,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator3 +1,30,9,8,1,22,0.8888888888888888,0.26666666666666666,0.41025641025641024,0.6991628574564894,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator3 +annotator_multi2,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator3 +annotator_multi3,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator3 +0,31,13,11,2,20,0.8461538461538461,0.3548387096774194,0.5,0.8756804532699135,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator4 +1,30,12,12,0,18,1.0,0.4,0.5714285714285715,0.744167825098049,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator4 +annotator_multi2,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator4 +0,31,33,26,7,5,0.7878787878787878,0.8387096774193549,0.8125,0.8523167196499111,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator5 +1,30,32,14,18,16,0.4375,0.4666666666666667,0.45161290322580644,0.7165563636458654,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator5 +annotator_multi2,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator5 +0,31,23,18,5,13,0.782608695652174,0.5806451612903226,0.6666666666666667,0.8260860404301554,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator6 +1,30,22,15,7,15,0.6818181818181818,0.5,0.576923076923077,0.6921220509480702,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator6 +annotator_multi2,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator6 +0,31,5,3,2,28,0.6,0.0967741935483871,0.16666666666666666,0.8064800827827963,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator7 +1,30,4,3,1,27,0.75,0.1,0.17647058823529416,0.6179441388071091,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator7 +annotator_multi2,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator7 +0,31,12,10,2,21,0.8333333333333334,0.3225806451612903,0.4651162790697674,0.8081240400677132,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator8 +1,30,11,6,5,24,0.5454545454545454,0.2,0.29268292682926833,0.6354705605880451,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator8 +annotator_multi2,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator8 +0,10,13,10,3,0,0.7692307692307693,1.0,0.8695652173913044,0.8343495790179837,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator4 +1,9,12,8,4,1,0.6666666666666666,0.8888888888888888,0.761904761904762,0.7891522970184701,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator4 +annotator_multi3,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator4 +annotator_multi4,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator4 +0,10,33,8,25,2,0.24242424242424243,0.8,0.372093023255814,0.8968463922057112,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator5 +1,9,32,6,26,3,0.1875,0.6666666666666666,0.29268292682926833,0.7457054593516464,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator5 +annotator_multi3,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator5 +0,10,23,9,14,1,0.391304347826087,0.9,0.5454545454545454,0.8570029084529952,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator6 +1,9,22,9,13,0,0.4090909090909091,1.0,0.5806451612903226,0.7875687955233205,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator6 +annotator_multi3,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator6 +0,10,5,4,1,6,0.8,0.4,0.5333333333333333,0.8225171601843602,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator7 +1,9,4,2,2,7,0.5,0.2222222222222222,0.30769230769230765,0.8565217391304347,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator7 +annotator_multi3,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator7 +0,10,12,10,2,0,0.8333333333333334,1.0,0.9090909090909091,0.8512046213449554,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator8 +1,9,11,4,7,5,0.36363636363636365,0.4444444444444444,0.39999999999999997,0.6263605259883815,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator8 +annotator_multi3,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator8 +0,13,33,10,23,3,0.30303030303030304,0.7692307692307693,0.43478260869565216,0.8182003376961703,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator5 +1,12,32,7,25,5,0.21875,0.5833333333333334,0.31818181818181823,0.8307056508654254,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator5 +annotator_multi4,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator5 +annotator_multi5,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator5 +0,13,23,13,10,0,0.5652173913043478,1.0,0.7222222222222222,0.8438600513099674,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator6 +1,12,22,11,11,1,0.5,0.9166666666666666,0.6470588235294118,0.8418400482794421,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator6 +annotator_multi4,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator6 +0,13,5,4,1,9,0.8,0.3076923076923077,0.4444444444444444,0.8678828902810093,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator7 +1,12,4,3,1,9,0.75,0.25,0.375,0.8216432672359725,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator7 +annotator_multi4,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator7 +0,13,12,9,3,4,0.75,0.6923076923076923,0.7199999999999999,0.8668472048782928,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator8 +1,12,11,5,6,7,0.45454545454545453,0.4166666666666667,0.43478260869565216,0.6320655864018455,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator8 +annotator_multi4,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator8 +0,33,23,16,7,17,0.6956521739130435,0.48484848484848486,0.5714285714285715,0.8332525179813037,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator6 +1,32,22,10,12,22,0.45454545454545453,0.3125,0.3703703703703703,0.7923692270082779,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator6 +annotator_multi5,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator6 +annotator_multi6,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator6 +0,33,5,3,2,30,0.6,0.09090909090909091,0.15789473684210525,0.7596024420127384,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator7 +1,32,4,2,2,30,0.5,0.0625,0.1111111111111111,0.9464653397391901,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator7 +annotator_multi5,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator7 +0,33,12,9,3,24,0.75,0.2727272727272727,0.39999999999999997,0.7970871467131684,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator8 +1,32,11,1,10,31,0.09090909090909091,0.03125,0.046511627906976744,0.5283018867924528,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator8 +annotator_multi5,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator8 +0,23,5,2,3,21,0.4,0.08695652173913043,0.14285714285714285,0.8169908544908545,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator7 +1,22,4,2,2,20,0.5,0.09090909090909091,0.15384615384615385,0.96875,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator7 +annotator_multi6,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator7 +annotator_multi7,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator7 +0,23,12,10,2,13,0.8333333333333334,0.43478260869565216,0.5714285714285714,0.7756669179459128,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator8 +1,22,11,3,8,19,0.2727272727272727,0.13636363636363635,0.1818181818181818,0.6379931878854829,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator8 +annotator_multi6,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator8 +0,5,12,4,8,1,0.3333333333333333,0.8,0.47058823529411764,0.8526361810251954,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator7,annotator8 +1,4,11,0,11,4,0.0,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator7,annotator8 +annotator_multi7,1,0,0,0,1,,0.0,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator7,annotator8 +annotator_multi8,0,1,0,1,0,0.0,,,,Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator7,annotator8 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_summary.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_summary.csv new file mode 100644 index 0000000..d797940 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_summary.csv @@ -0,0 +1,29 @@ +annotator_a,annotator_b,bout_precision_mean,bout_recall_mean,bout_f1_mean,mean_matched_iou,total_true_bouts,total_pred_bouts +annotator1,annotator2,0.49474607369022827,0.4862089185502458,0.7537181511875509,0.8537543078590213,822,774 +annotator1,annotator3,0.47228041017198563,0.5046823643610721,0.7545740033597036,0.8507306564922478,822,860 +annotator1,annotator4,0.4419856601602343,0.4867497475773719,0.712522995762225,0.8477763754246438,822,868 +annotator1,annotator5,0.32261481104818046,0.35732933302830433,0.6482645420371722,0.8234389519498435,822,878 +annotator1,annotator6,0.46460364419832945,0.5030183213514022,0.7419670556258369,0.8682038666738101,822,928 +annotator1,annotator7,0.49681820232118606,0.25604868789602914,0.5260833694495791,0.867124446977489,822,340 +annotator1,annotator8,0.48558239290110045,0.4133096710208694,0.6909253274463392,0.7869655518593348,822,646 +annotator2,annotator3,0.4952973841824252,0.5355554509151033,0.7521489785751027,0.8602009369088023,774,860 +annotator2,annotator4,0.48787793733418244,0.5437432075921533,0.751719159597716,0.8554207202263321,774,868 +annotator2,annotator5,0.3277642678184365,0.3593428402436261,0.6151156425930313,0.8083124372484958,774,878 +annotator2,annotator6,0.48702883179939527,0.5419643374107268,0.7570764564549836,0.8486256317171471,774,928 +annotator2,annotator7,0.564860865843922,0.31114577186042536,0.5780676464906199,0.8517638500895313,774,340 +annotator2,annotator8,0.526534949407497,0.45894115835281823,0.7244400513831845,0.8126856451401713,774,646 +annotator3,annotator4,0.4756263767509258,0.5006538245095707,0.726062048331008,0.8532588179111643,860,868 +annotator3,annotator5,0.3302700353447233,0.34130184054915225,0.562272893201128,0.8106941416497,860,878 +annotator3,annotator6,0.48650290115540795,0.5108704495373366,0.7294428457267054,0.8587889534159081,860,928 +annotator3,annotator7,0.5388806879434545,0.27239752503083425,0.5228504886024592,0.8913052014909536,860,340 +annotator3,annotator8,0.5238271522123874,0.4257721825703795,0.69971320398332,0.8035301110532811,860,646 +annotator4,annotator5,0.3222781448201444,0.3413605026981668,0.5917971814592281,0.8059829908686457,868,878 +annotator4,annotator6,0.4986034615101873,0.5029511738807261,0.7362114589805063,0.8596429116868157,868,928 +annotator4,annotator7,0.5518141765562908,0.25546818019859074,0.5102285292220187,0.8508357570505104,868,340 +annotator4,annotator8,0.49964208074367344,0.38556089065631105,0.647277035322768,0.7972217354717787,868,646 +annotator5,annotator6,0.33821631649171774,0.333495805700366,0.6471656158685001,0.8156751708180393,878,928 +annotator5,annotator7,0.3559265010351967,0.1827084109518028,0.3941996429523916,0.8351607303598277,878,340 +annotator5,annotator8,0.3425060745497093,0.26881234333680565,0.5362606192247302,0.7562580540644194,878,646 +annotator6,annotator7,0.538928038606747,0.2742940294472686,0.5191503274581463,0.867013164280109,928,340 +annotator6,annotator8,0.5159375094019665,0.4069224393806366,0.6736516056559885,0.7965915525564102,928,646 +annotator7,annotator8,0.32502504701873475,0.5140544075923446,0.602652353498808,0.8463976142829054,340,646 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_agreement.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_agreement.csv new file mode 100644 index 0000000..ef6e713 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_agreement.csv @@ -0,0 +1,337 @@ +record_key,mouse_record,sex_group,version,annotator_a,annotator_b,n_frames_compared,accuracy,balanced_accuracy,macro_f1,cohen_kappa,mcc,ari,nmi +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator2,24292,0.9731187222130743,0.9675061681262417,0.9652249248762114,0.9459645589826463,0.9459842766898172,0.9190618796449983,0.8466353938724892 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator3,24292,0.9585871891980899,0.9332316164938649,0.9470889262907188,0.9148114633546556,0.9155766985295241,0.8721769358945488,0.7907534142638043 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator4,24292,0.9389922608266096,0.9158686454878951,0.9180087554777554,0.8756521224899514,0.876224211641261,0.8241602022604979,0.7157698030875547 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator5,24292,0.96558537790219,0.9539451896487569,0.7167341618522026,0.9310132681190149,0.9311352214106675,0.9063713909350986,0.8225889743947744 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator6,24292,0.9524534826280256,0.9569178820598246,0.7077754469459772,0.9064236027737058,0.9077684302633688,0.8565803610670513,0.7830841072372936 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator7,24292,0.9192738350074099,0.8750218400095323,0.8916981863229226,0.832079030046568,0.8340643312959093,0.7616300930667008,0.6435619329042239 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator1,annotator8,24292,0.8925983863000164,0.8400120952391217,0.8482569106411338,0.7792046326448318,0.7800017594096168,0.7132227394615916,0.562809818215548 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator3,24292,0.9587930182776223,0.9298520421305896,0.9456233087450352,0.9155254605865452,0.9165522102872631,0.8757233671138501,0.7922022369782317 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator4,24292,0.9398567429606455,0.9146324549170388,0.9188357763846756,0.8778241204268084,0.8785738978454256,0.8276192782801708,0.7187019236662895 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator5,24292,0.9636505845545859,0.9480716806930817,0.7139640542107982,0.9273600340676933,0.9274315217570336,0.9035220835475998,0.8146527504202203 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator6,24292,0.9552527581096657,0.9541410406971034,0.7074279115232762,0.9121819232529045,0.913245448743664,0.8717588860546106,0.7890532096640636 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator7,24292,0.9216202865140787,0.874086359416649,0.8925296445803049,0.8375470215504991,0.8399298268491721,0.773203019685529,0.6556687469099707 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator2,annotator8,24292,0.8925983863000164,0.8392116055644365,0.8492338464245276,0.7799648131611576,0.7809965485419207,0.7107624014197271,0.5644102360120955 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator4,24292,0.9328173884406389,0.9199607775189746,0.9080465680088072,0.8601568275001699,0.8604671825564333,0.8070662749398404,0.6940903148338873 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator5,24292,0.9500246994895439,0.9481834543237234,0.7013215192383063,0.8978554105990759,0.8991452795342674,0.8579799100453965,0.7610445799222478 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator6,24292,0.9364399802404083,0.9496898687122687,0.6908574743926453,0.8726652972028179,0.8765574691081247,0.8115250475724693,0.7325096807335193 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator7,24292,0.9263955211592294,0.8950808486848532,0.8997935100024314,0.8433838871172228,0.8437672249903394,0.7779532912522497,0.6555678814702881 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator3,annotator8,24292,0.8896344475547505,0.8473947595008169,0.8441690730360515,0.768144386818991,0.768220622140163,0.6913921276842114,0.5409751436482737 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator5,24292,0.9302239420385312,0.908679041218421,0.6794802635391162,0.8587017357927665,0.8598503421351174,0.8114673456251295,0.6975374812319957 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator6,24292,0.9341758603655524,0.9337147284935318,0.6867393185746213,0.8692925386628391,0.8727175607171669,0.809795976763289,0.7178008584007789 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator7,24292,0.8977441132883254,0.8417935091332424,0.855444684490418,0.7845344903340886,0.7851988545084112,0.7149336769335998,0.570558569739184 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator4,annotator8,24292,0.8677342334925078,0.8028069960142464,0.8088679962897842,0.7247475579967959,0.7248659725605373,0.6563695709680952,0.49244390445160485 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator6,24292,0.9472254240079039,0.7117795661172595,0.5606317880226275,0.8966721496977132,0.8974419195982081,0.8562829250494305,0.7642618402940302 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator7,24292,0.9108760085624897,0.6531293684646242,0.6633601459373989,0.8158782816769213,0.8186804139000385,0.7426923428135683,0.6235230326077306 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator5,annotator8,24292,0.8898402766342829,0.6323715669054465,0.6362948514950626,0.7750201200987652,0.7764065205675744,0.7136147211252807,0.5673401516575869 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator7,24292,0.9035896591470443,0.6383558762957171,0.6559396503153605,0.8049995851278451,0.81113881775384,0.7248358587068137,0.6219748464433238 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator6,annotator8,24292,0.8909105878478512,0.6256820626451787,0.6375169874039793,0.7816579164042782,0.7855447845162051,0.7138476897285578,0.5828946002184389 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v1,annotator7,annotator8,24292,0.9043718096492672,0.8707132269173212,0.8627665923053396,0.7965781984209626,0.796819643203208,0.7210047722135342,0.5694833682017866 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator2,24292,0.9571463856413635,0.956009421788285,0.9439923009363698,0.9150483056468597,0.9154105755299947,0.8790855022901243,0.7989456261038295 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator3,24292,0.9682199901202042,0.9466284048393961,0.9580994728005271,0.9350865519379056,0.9355660664441701,0.904381266955703,0.8297317026323268 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator4,24292,0.9594105055162193,0.9309717840007913,0.9465459956907813,0.9161467562069868,0.9179012511169006,0.8756929629462973,0.7984759528181796 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator5,24292,0.9718014161040672,0.9595363729683918,0.9618042439590377,0.943084220517241,0.943117803606034,0.9192672538063589,0.8430549387926728 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator6,24292,0.9608513090729458,0.9470144109788002,0.9448399283816032,0.9223348571194123,0.923470949653062,0.8956539599576089,0.8166677661538695 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator7,24292,0.947760579614688,0.9229744444958795,0.9324009495328243,0.8932058481552593,0.8938199918542165,0.8461258587807617,0.7511124582392423 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator1,annotator8,24292,0.9378807837971349,0.8965853663186999,0.9085736785866122,0.8724433384928822,0.8733593967015787,0.838589677788836,0.7196957126424836 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator3,24292,0.9445084801580768,0.9036320883201627,0.9242233437811033,0.8883555954918343,0.8899365194053228,0.845913782488791,0.7537800441387074 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator4,24292,0.9461139469784291,0.9032836864226615,0.9282385803670025,0.8904159749162939,0.8937960587799628,0.8389279914161827,0.7560926430684776 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator5,24292,0.946360941873868,0.9179298804027991,0.9307486148684226,0.8932898902642329,0.8938570653648582,0.846214289379041,0.7533181679053836 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator6,24292,0.9395685822493002,0.9075547148162975,0.9157582234747427,0.8817295074445975,0.882560522867621,0.8437828305157856,0.7450006566006825 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator7,24292,0.9357401613699984,0.8913349635642785,0.9101574451907578,0.870608928694933,0.872339010116514,0.8289135052509808,0.7232503704494287 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator2,annotator8,24292,0.9141692738350075,0.8597009417265046,0.8801307483964479,0.8264490170819881,0.8285981274990979,0.7737152073023776,0.6583709400975377 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator4,24292,0.9540589494483781,0.9345355994249531,0.9390040427877012,0.9034094357035014,0.903886741082397,0.8591131122614841,0.764440620678702 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator5,24292,0.9637740820023053,0.9622799781359861,0.9524296247980025,0.9257081936943408,0.9259754652367927,0.8908462674128622,0.8075053296549722 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator6,24292,0.9504363576486086,0.9507649822002353,0.936957618847497,0.9002314604269583,0.9028037179404457,0.8534182648213158,0.776332860792253 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator7,24292,0.944796640869422,0.9304847533151882,0.928781396742683,0.8852333016804763,0.8852807578180169,0.8320906016369887,0.729660653267643 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator3,annotator8,24292,0.9366869751358472,0.9089708783403064,0.9106991919510604,0.8677493532970258,0.8678577308007428,0.8207538188244434,0.6976392849188482 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator5,24292,0.9523711509962127,0.9534179175261319,0.9379194193792345,0.901183155629265,0.9024702598099605,0.8533944787574316,0.7631415617989257 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator6,24292,0.9374279598221636,0.9390806290314814,0.9185007437525612,0.8728091194947517,0.8779039602875279,0.818944326652561,0.7429599069754775 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator7,24292,0.9485015643010044,0.9366189294652587,0.9299972615704664,0.891590419018179,0.8918652976889041,0.8454336816801932,0.7384075179329738 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator4,annotator8,24292,0.9303474394862506,0.9039287014818228,0.9010278592249904,0.8526435579773094,0.8527628880281376,0.8003664527222666,0.6729948583642917 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator6,24292,0.9590811789889676,0.9487734351209213,0.9441874354119711,0.9185479468375,0.9200469989731265,0.8869046907871507,0.8097483663213612 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator7,24292,0.940680059278775,0.9160027006403615,0.9233096945845142,0.8782374339656833,0.8785977319173511,0.8258899115458188,0.7246518330384515 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator5,annotator8,24292,0.9353696690268402,0.8974787404649494,0.9076451401251346,0.8667371788837828,0.8673439141677008,0.8259276437256994,0.7056674501599255 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator7,24292,0.9305944343816894,0.9057763631855523,0.9111664331517391,0.8602382679142039,0.8632541244949178,0.8020170419438812,0.7122247051077845 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator6,annotator8,24292,0.9285361435863659,0.8917489504019341,0.9004318485437534,0.8554962000177451,0.8590619056423953,0.8085990258879896,0.7047496724711327 +Female_likely/Mouse060_20160526_18-16-27,Mouse060_20160526_18-16-27,Female_likely,v2,annotator7,annotator8,24292,0.9403095669356166,0.9078719366354587,0.9111117148138006,0.8751783113192623,0.8752145490224439,0.8394045966548226,0.7116643618872295 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator2,18646,0.9666416389574172,0.9686585975662424,0.9654196897693327,0.9493972984421105,0.9498769344759901,0.904787827044569,0.8636015322189403 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator3,18646,0.9576852944331223,0.9560139875733936,0.956440221811686,0.9355612086771633,0.935668191716238,0.8789760335330905,0.8319384891176251 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator4,18646,0.9553791697951303,0.9551677857026015,0.9531682268809986,0.9322339098414214,0.9326988956858813,0.8750998341628827,0.824047596460837 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator5,18646,0.9641209910972863,0.9611540224807701,0.7218710524600513,0.9453175670218029,0.9454181568653846,0.898211729092458,0.8480749227769204 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator6,18646,0.9522685830741178,0.957191046330575,0.9508686599665831,0.9278350765621075,0.929402474965191,0.8654911839834784,0.8265913721632028 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator7,18646,0.9370910651077979,0.931926189100718,0.9336013955380613,0.9040474161923303,0.9041831877779894,0.8287604593593733,0.7741651288902508 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator1,annotator8,18646,0.9066823983696235,0.8980094876848281,0.9008745302073713,0.8573872429169099,0.8578636772821544,0.7553091451565973,0.6992927895342226 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator3,18646,0.9511959669634238,0.9475494541055834,0.9500221635416158,0.9259568104798613,0.92682385287502,0.8603894220245825,0.8121265823596755 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator4,18646,0.970771210983589,0.9681792310369408,0.9691736539036864,0.9557222656107915,0.9558255618749029,0.917836667225775,0.8716367400632539 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator5,18646,0.9561300010726161,0.9503385968862522,0.7151842967250821,0.9333790168416544,0.9341107654757059,0.8798820301117555,0.840806699504222 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator6,18646,0.9679824090957846,0.969288414849793,0.9668573773075239,0.9516709180357471,0.9520328321649083,0.9092673129880277,0.8677122531670731 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator7,18646,0.950981443741285,0.9447152431542465,0.9483634320012767,0.925521659178974,0.9265243239384857,0.8655865553647389,0.8283045790386278 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator2,annotator8,18646,0.9094712002574279,0.8997134222636894,0.9037890399847583,0.8622473047156627,0.8639422814132048,0.765689390390938,0.7220118596412638 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator4,18646,0.9500697200471951,0.9512427222407016,0.9484470888688842,0.9241694162277783,0.925145566683261,0.857835017278898,0.8080474383839059 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator5,18646,0.9427222996889413,0.9403704628755367,0.7056221375333813,0.9126732326453615,0.9129660607267203,0.8387203853725773,0.7850043074045595 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator6,18646,0.9464228252708355,0.9528654207993347,0.9456315087638195,0.9189958356771019,0.9210610155967095,0.846643406908862,0.8098800137428159 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator7,18646,0.9378955271908184,0.9342594273693443,0.9352775381901367,0.9052303267125917,0.9054608560537539,0.8275697583406887,0.7730025563212134 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator3,annotator8,18646,0.9054488898423254,0.8986570130175732,0.9008535308492798,0.8554238831144757,0.8559236900685091,0.7474929226823792,0.6893627909787893 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator5,18646,0.9487825807143623,0.9425054509145925,0.7087125902309754,0.9221140452560684,0.9226350278577699,0.8626754401181654,0.8197525299887246 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator6,18646,0.9637992062640781,0.9659715748359887,0.9620973451092687,0.9453127285235386,0.9460092118343323,0.8985586128032337,0.857026799068071 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator7,18646,0.9413815295505739,0.9348106890657708,0.9377039484570688,0.9108136675159554,0.9116086382949964,0.8410680570156079,0.7937117124915926 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator4,annotator8,18646,0.9044835353427009,0.893928952100408,0.897745601332108,0.8544376083920316,0.8558125267287745,0.754359246666827,0.7069012458887944 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator6,18646,0.950445135685938,0.7171871216620771,0.7110391910851012,0.9250177572988716,0.927103477540907,0.8655548408332694,0.8412178010062433 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator7,18646,0.9345704172476671,0.6970541987801361,0.6975395734972529,0.9000544218991399,0.9000866180557991,0.8260032455810855,0.78212289253296 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator5,annotator8,18646,0.9092030462297543,0.6745462049811883,0.6761876365795354,0.8610097290043364,0.861244405975923,0.7682252778031223,0.7191776307025843 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator7,18646,0.9414887911616433,0.9351807353376613,0.9390838796654187,0.9114350530495159,0.9139220682040438,0.8404879378087864,0.8115464143318863 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator6,annotator8,18646,0.9110264936179342,0.9026038582904801,0.9065153389830551,0.8651866804324481,0.8687013004276536,0.7692443836600591,0.7372022170426338 +Female_likely/Mouse061_20160526_18-38-49,Mouse061_20160526_18-38-49,Female_likely,v1,annotator7,annotator8,18646,0.9270084736672745,0.9201939546929084,0.9220565140399183,0.8881587439258399,0.888277051291953,0.804464059142174,0.7517588082981577 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator2,19638,0.977441694673592,0.7347937785061097,0.7347012411382055,0.9657649359072175,0.9658354847042137,0.9298529178021623,0.9028057751876959 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator3,19638,0.9577350035645178,0.7216291213814849,0.7212167489666914,0.9359188246139706,0.9369960638599335,0.8705314808578848,0.8512071737268859 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator4,19638,0.9665953763112334,0.7281079113760287,0.7271761704375376,0.9493677409386572,0.949659122951641,0.897208747296037,0.8695939805811965 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator5,19638,0.9661880028516142,0.7275813634896895,0.5816633372511119,0.9487414132663214,0.9492301390079451,0.8961644560714328,0.8719133636959584 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator6,19638,0.9599755575924228,0.7225248139090241,0.7225297785272737,0.9392327529711626,0.9399918346082913,0.8778161089637961,0.8534459445954072 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator7,19638,0.9356349933801813,0.7058155971356969,0.7060015821911969,0.9023334156243858,0.9027352308986034,0.8080648825010796,0.7858755143423181 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator1,annotator8,19638,0.9112435074854873,0.6902639079060169,0.6894112392938625,0.8654929625173675,0.866396456231973,0.7444238363794066,0.735990304763706 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator3,19638,0.9558509013137794,0.9604802104027454,0.9595140301400443,0.9330561703259121,0.9347328540624633,0.8642075173686042,0.8498541444287233 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator4,19638,0.9658824727568999,0.9700936218763759,0.9686141495815151,0.9482681656696832,0.948907870997891,0.8939600652749613,0.8708404271945859 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator5,19638,0.9646094306955901,0.9685407521387658,0.7256578569099583,0.9463334266558935,0.9472542059245351,0.890603306387145,0.8699336042247617 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator6,19638,0.9667990630410429,0.968742135382486,0.9691906006437571,0.9495347862407543,0.9499121441294084,0.8969776260828906,0.8685563642094608 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator7,19638,0.935584071697729,0.9409824335207074,0.9409677324910156,0.9022249919159322,0.9030026167612417,0.8071536139879915,0.7887148009845871 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator2,annotator8,19638,0.9110907424381302,0.9203100490949173,0.9187592781083436,0.8652461177594457,0.8666875804680206,0.7432435996729622,0.7375560620345973 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator4,19638,0.9638456054588044,0.9678905789712197,0.9663575027932726,0.945152622281751,0.9454383066009814,0.888355942501598,0.8573185482070752 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator5,19638,0.9605356960993991,0.9645936504450633,0.7229625179418004,0.9400982013307583,0.9402196035165683,0.8781570205823014,0.8503809668795809 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator6,19638,0.9425603421937061,0.9498273740852151,0.9471059106861438,0.9129849095231436,0.916532821958084,0.8268808401318909,0.8242930612683445 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator7,19638,0.9302882167226805,0.936283007822113,0.9364701291721329,0.894102150367749,0.8942681321171386,0.7921610866194199,0.7739288873304699 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator3,annotator8,19638,0.9055912007332723,0.914662552904702,0.9138469174892494,0.8566570697497746,0.8566693582934365,0.7288521083942989,0.7228711090990995 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator5,19638,0.9646603523780426,0.9668253552843326,0.7256418175041816,0.9464152036509137,0.9464520452825761,0.8909546630719098,0.8623139413627525 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator6,19638,0.9534575822385172,0.9570431271726534,0.9571281214310737,0.9294472523153633,0.9313902551648877,0.8578523326677204,0.8447388049722053 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator7,19638,0.9324269273856808,0.9360666437859368,0.937594351639735,0.8974565022812833,0.8975223906040628,0.7988144083358163,0.7719656343907031 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator4,annotator8,19638,0.9032997250229148,0.910978503801409,0.9114313186747368,0.8533766419567386,0.8535719041544071,0.7240742568722581,0.7165741963753723 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator6,19638,0.9471432936144211,0.7145865799064735,0.7137694969083186,0.9198867218724023,0.9222896792608581,0.8401832244351896,0.8341926387033196 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator7,19638,0.931663102148895,0.7025719375463855,0.7031955965774206,0.8962532196239705,0.8962789030462349,0.7964731225220787,0.7737249383999555 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator5,annotator8,19638,0.9037070984825338,0.684500772121122,0.6842829726748616,0.8539189661226703,0.8539990380782376,0.7250844056897671,0.7232505098903694 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator7,19638,0.9413891434973012,0.9477498418611393,0.9457973478401324,0.9110729582138382,0.9132818993113029,0.8238404482578314,0.8093599758367318 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator6,annotator8,19638,0.9192891333129647,0.929710145814885,0.9260639240821931,0.8777776748118558,0.8810122381564133,0.7645942419695076,0.7625613434179176 +Female_likely/Mouse062_20160526_18-56-26,Mouse062_20160526_18-56-26,Female_likely,v1,annotator7,annotator8,19638,0.9373154089011101,0.9435592526883599,0.9423742579984972,0.9048312723501684,0.9049914714864274,0.8118703568293791,0.7855550090197392 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator2,19307,0.9483089035065002,0.9302245090040859,0.9289064610030263,0.8896290149677425,0.8897046420392445,0.8471498613444741,0.7395867893047301 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator3,19307,0.9295592272232869,0.8794502992359265,0.90528171137435,0.8428152806693479,0.8452097074623406,0.7777024530037863,0.6723884962360226 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator4,19307,0.9343761330087533,0.9076167801803093,0.9087131488565482,0.8580060391219109,0.8582543432765082,0.8066536671509809,0.6846545204928073 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator5,19307,0.953436577407158,0.9215307766066622,0.9352198405798738,0.8978722816611235,0.8987149115039875,0.8595503919314835,0.76892455798273 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator6,19307,0.94375097115036,0.9297273836331391,0.9260525215687352,0.8810312798724661,0.8816658278884777,0.8301657940596611,0.7265526963500998 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator7,19307,0.9020562490288496,0.8119548478581052,0.8491864902160366,0.7776743553585879,0.7820743661601883,0.7253254974368797,0.5920836699002195 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator1,annotator8,19307,0.8975501113585747,0.8206392421225811,0.8422307767258435,0.7692028695944239,0.7732379762680897,0.719247271038572,0.5773316133890379 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator3,19307,0.9465996788729476,0.8985823370329366,0.926133116160977,0.88153010398399,0.8848504799463386,0.8314099868174425,0.7425913028484626 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator4,19307,0.9633811570932822,0.9432272246474876,0.9457718628961599,0.9211832332429428,0.9218162110797344,0.8947398717081599,0.8024962561476033 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator5,19307,0.9504842803128399,0.9129201696220282,0.9272634482289316,0.8919907497110374,0.8933614453254501,0.8590469865251793,0.7602572053549028 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator6,19307,0.9614129590304035,0.9430163194740193,0.9405775340026383,0.918751423918829,0.9190365386732198,0.8964334610880063,0.7969791187540346 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator7,19307,0.9098772465944994,0.8190823572017806,0.8573930442955655,0.7966553094921506,0.8020727559520264,0.7530791909015341,0.6266579552311108 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator2,annotator8,19307,0.8934065364893562,0.812724775667195,0.8340603168699049,0.7613588134784437,0.7663803155677478,0.7125882275728478,0.565855058454775 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator4,19307,0.9509504324856269,0.9535123714575198,0.9243745945310952,0.8894729393899643,0.8909509858846696,0.8507654015856158,0.7425239465829484 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator5,19307,0.9411612368570985,0.9324941169649591,0.9171606491382619,0.8653648998071,0.8658275881447417,0.8130467905369703,0.6999133723931279 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator6,19307,0.9396591909670068,0.9523332738750133,0.9167947058206654,0.8676075996698951,0.8728159048195158,0.8132430337364324,0.726956373639536 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator7,19307,0.9101362200238255,0.8415140334747674,0.8589701058663667,0.7861959198730816,0.7866533035078891,0.7357765448169012,0.5827883514846174 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator3,annotator8,19307,0.8944942248925261,0.8331859286875977,0.8324037277842672,0.7509692216268593,0.7513760157138397,0.6942369979885767,0.5291556216630716 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator5,19307,0.9389858600507588,0.8951436784817487,0.9065600575014813,0.8649377374894613,0.8652142418968504,0.8304652443731342,0.7079515654262193 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator6,19307,0.9551458020407106,0.9385655121523788,0.9310490102097778,0.9044267188333415,0.906163672949593,0.8789004605812099,0.7822232965191277 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator7,19307,0.9062516185839333,0.807002290130269,0.840800691853838,0.7850343388526131,0.7882839174346852,0.7537638813290228,0.5983245630601268 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator4,annotator8,19307,0.8909721862536903,0.8052248001712138,0.8260152124488327,0.7517758723951764,0.7542392185503629,0.7054777145218469,0.5437332186800746 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator6,19307,0.9436991764644947,0.9412765969732985,0.9221719114994404,0.8784687646525215,0.8812359790128985,0.8336070557570933,0.7423653374431551 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator7,19307,0.9191484953643756,0.842662739274291,0.8712471831625083,0.8115814174863644,0.8133326183526336,0.7671696962342873,0.6264604852517894 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator5,annotator8,19307,0.9104987828248822,0.848642665078287,0.8607592101652811,0.7930039322587799,0.7944050576380538,0.7462683550132653,0.5974050140471501 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator7,19307,0.9068213601284508,0.8195369219066765,0.8591363131104289,0.7921601417940212,0.7996437762725873,0.7360294225711485,0.6194634156284468 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator6,annotator8,19307,0.8936655099186823,0.8232740819546894,0.8444510743911375,0.7647498706762252,0.7721253867862369,0.7021248649393098,0.5788573059954897 +Female_likely/Mouse063_20160526_19-14-46,Mouse063_20160526_19-14-46,Female_likely,v1,annotator7,annotator8,19307,0.9188377272491842,0.8788576981347922,0.8581052058740645,0.804620003726122,0.8052332988990305,0.767570766234083,0.5998963503674563 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator2,6641,0.9203433217888872,0.9307054851011288,0.9143163083685807,0.868443223585881,0.8691597456947314,0.7629961109381328,0.6885693195164855 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator3,6641,0.9001656377051649,0.8977854564596033,0.9025139065050659,0.8326722156518778,0.8398378345518285,0.6942090710032437,0.6611433163717303 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator4,6641,0.9010691160969734,0.9142668156796585,0.896882946517481,0.8366385989415951,0.8381162811401146,0.7073165454383634,0.6391226880928046 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator5,6641,0.9022737539527179,0.8842462667079314,0.6708389866050427,0.8364395895059707,0.8367285184783961,0.7168725154814766,0.6405734739909832 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator6,6641,0.9126637554585153,0.9211609556325872,0.9052278210672507,0.8559259701463954,0.8569422813253957,0.7427178495479748,0.6708978312951428 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator7,6641,0.8540882397229332,0.7582716956909605,0.7921073144429144,0.7477487171309644,0.7536092685651464,0.6212781010260607,0.5598280597878736 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator1,annotator8,6641,0.8409878030417106,0.8050499955311428,0.8082372295828103,0.7333185077570745,0.7406758382521571,0.5758569057397924,0.49759851254749704 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator3,6641,0.9069417256437283,0.8830610566358418,0.9012311564924912,0.8450139327940474,0.8503533417136248,0.7229235389270593,0.6737662837476338 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator4,6641,0.9338954976660142,0.9276356310082269,0.927151380181872,0.891617373881451,0.8918620300288093,0.8064244233683213,0.7268153390536084 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator5,6641,0.9210962204487276,0.8855142490553654,0.6793852379188094,0.8691784539054466,0.8701783816610318,0.7854512674159804,0.7121913122943546 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator6,6641,0.9343472368619183,0.9283782182497785,0.9276290805591958,0.8927126357402054,0.8945127082066088,0.8076914797347317,0.7537991050076015 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator7,6641,0.8324047583195302,0.7342440593504573,0.7631545593425404,0.7136267739832438,0.7214361344987029,0.5795494421799426,0.507384061623627 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator2,annotator8,6641,0.8433970787531998,0.792929817993277,0.8056483989445594,0.7389595527603963,0.744516854973107,0.5947638807703,0.49939141367454326 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator4,6641,0.9012196958289415,0.9223114593238235,0.8948692804614907,0.8350133595181303,0.8385383995396176,0.7076406190137009,0.6443144444673323 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator5,6641,0.8991115795813883,0.8868195605409444,0.6647889342642281,0.8297785196832753,0.8348916651619513,0.7111547917128123,0.6426458621251186 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator6,6641,0.8765246197861768,0.9085498015018992,0.8771125067883281,0.7962770721902502,0.8082305153613112,0.637506972297918,0.6336313112586361 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator7,6641,0.8462580936605932,0.7525219313908544,0.7814791467504151,0.7307632608642047,0.7372118332969115,0.5969801965210565,0.5075496924647672 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator3,annotator8,6641,0.8486673693720825,0.800268335643692,0.8013176178443171,0.7400853229202422,0.740125811725307,0.6055144410598452,0.47347260743921665 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator5,6641,0.9107062189429302,0.8739732524459676,0.6702687203513089,0.8519120193112015,0.8533195412467669,0.758313984936578,0.6805363535735618 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator6,6641,0.9192892636651107,0.9157864546017404,0.914413409306649,0.8682350244593809,0.8715127803472504,0.7609866046696129,0.7045287712060904 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator7,6641,0.8150880891432013,0.7202071553441817,0.7491975979888124,0.6837907937370078,0.6912670310981825,0.5336108494719444,0.46426725976645383 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator4,annotator8,6641,0.8287908447522964,0.7818613934732438,0.7960605232116924,0.7137283166976299,0.7174275790774084,0.5535299078892024,0.4546859129036492 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator6,6641,0.904381870200271,0.6881143214257701,0.6670183092565528,0.8418504076284099,0.8441998220782068,0.737867124673116,0.6899483790167293 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator7,6641,0.8364704110826683,0.5630850537549343,0.5889548523587982,0.715384647258938,0.7191887450326109,0.5593147074156384,0.48087674252585894 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator5,annotator8,6641,0.8465592531245294,0.6199660450280067,0.6161578936861607,0.7408881241029968,0.7463380653371969,0.5857106763233176,0.5012913621401462 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator7,6641,0.820207800030116,0.7284492781359114,0.7560710701090531,0.6941049491447588,0.7041713859642186,0.5448946933899105,0.50343006463375 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator6,annotator8,6641,0.8268333082367113,0.7899137512915998,0.7986086995579771,0.7142312653966716,0.7262859974714964,0.542358809070687,0.49232195337510704 +Female_likely/Mouse163_20161018_18-22-45,Mouse163_20161018_18-22-45,Female_likely,v1,annotator7,annotator8,6641,0.8557446167745821,0.8617491571644917,0.7800502440092364,0.7470456953446483,0.7541945166814761,0.6181769402864671,0.5110116257400872 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator2,18497,0.9451262366870303,0.910412616913,0.859164688063508,0.8661819220591525,0.8665406497855693,0.8456818273648313,0.7041831340030131 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator3,18497,0.9257176839487484,0.841047125249517,0.8425323620305047,0.807092795148871,0.8147614614537596,0.7641476513428342,0.6417722047716216 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator4,18497,0.9304752121965724,0.8962441036946089,0.7532069667717995,0.8294011025441066,0.8301956955064529,0.8031037260634838,0.6568179604126722 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator5,18497,0.9206357787749365,0.7659525824406332,0.8081106079305356,0.7969786817191686,0.8015827650531531,0.7801143152038944,0.6314835138006077 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator6,18497,0.9256636211277505,0.7585289595379263,0.5500867381599688,0.8152362380142061,0.8175698556616098,0.8200768177803563,0.6544470330750992 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator7,18497,0.9069038222414446,0.6025055381396993,0.6256049767148782,0.7541155597429585,0.7675683963774352,0.7002195340686451,0.581110726375198 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator1,annotator8,18497,0.8765205168405688,0.6954246862357504,0.7007830637785692,0.6603029816299393,0.6809642763822336,0.646208440453163,0.5089388586581476 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator3,18497,0.9359355571173704,0.8297028230448884,0.8762885081211607,0.8303820622873701,0.8357649289469827,0.7899577540101713,0.66605799340976 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator4,18497,0.9348002378764124,0.8595188631272858,0.7716812733065411,0.8372552181874273,0.837413787861814,0.8308156260210452,0.6702512245575817 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator5,18497,0.9287992647456345,0.748305037589752,0.8097717793159762,0.8143978987146141,0.817048191565678,0.8104127371661756,0.6505380723205678 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator6,18497,0.9257176839487484,0.758361046262982,0.5664772072529465,0.8120098548583367,0.8129415670594833,0.8121071142517461,0.6423289074912945 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator7,18497,0.9036600529815646,0.6045795699481109,0.616423566023382,0.7405443104072862,0.7513867323095661,0.6986918611418275,0.5595416687843225 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator2,annotator8,18497,0.8907390387630426,0.7106945187642275,0.7554510858003847,0.6927400531097648,0.7103923166778997,0.6701882130803949,0.5455542490404095 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator4,18497,0.9190138941449965,0.9013980136348235,0.7457860139072496,0.7837757436133443,0.7875558864392903,0.7636296798724507,0.597645752095012 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator5,18497,0.9282045737146565,0.8005378852045535,0.8304500747547607,0.798395435867364,0.7990317862500071,0.7842811468579048,0.6011271119650027 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator6,18497,0.9237714223928205,0.8226198184021385,0.5694070868568584,0.7933858357591692,0.7969480246321785,0.7716840234741802,0.6150182691618017 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator7,18497,0.9158241877061145,0.6259657293327567,0.6234399352590336,0.7534753336397333,0.7554494287579413,0.7102622778529188,0.5455694086275115 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator3,annotator8,18497,0.9003081580796886,0.7616259473189516,0.7724369656792722,0.6929841504117231,0.6973184722261732,0.6720724354896807,0.4860022076509945 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator5,18497,0.9289073903876304,0.6778478598817618,0.72225938508284,0.8132395194618055,0.8150606706251349,0.8182751807845963,0.6607325691962436 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator6,18497,0.9189598313239985,0.8228965911223546,0.6534214347849568,0.7934575674719608,0.7941178944811151,0.7918375584428922,0.6110414818961509 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator7,18497,0.9013353516786506,0.6029173039054816,0.6115165907326927,0.7318184874634273,0.7406754176733048,0.7012261210827853,0.5416393205440174 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator4,annotator8,18497,0.8828999297183328,0.6201620575629015,0.6919181690824531,0.6673436312758928,0.6820281100040392,0.662926133455885,0.5030654338434181 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator6,18497,0.9240957993188085,0.8164128491282028,0.5420978485397524,0.7975069620902804,0.7987788536577626,0.825940428316389,0.6428480564035766 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator7,18497,0.9062550683894686,0.6079592426729887,0.6028875552268813,0.7314902602907483,0.7359600709514038,0.710106180750548,0.5263707842743826 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator5,annotator8,18497,0.9148510569281505,0.8200921171657256,0.7788290931059327,0.7438000032134076,0.7518200304031738,0.7245399708024327,0.5580728077332034 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator7,18497,0.8854949451262367,0.48192037504764007,0.4679149432342419,0.684521652690496,0.6936469522536166,0.6641422441311625,0.5004864575678116 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator6,annotator8,18497,0.8817105476563767,0.48968530830985946,0.5287818030502456,0.6582839295173595,0.6714129035485678,0.6591334935861273,0.483860462477663 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v1,annotator7,annotator8,18497,0.9413959020381684,0.8289588592357626,0.6393661797464304,0.8145381325435882,0.8179037167099604,0.8293517316352996,0.6582140277357593 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator2,18497,0.9398821430502243,0.9217816294701174,0.8896769755382782,0.850216719244717,0.8526882095087077,0.8338378152473807,0.705665043178021 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator3,18497,0.9457209277180083,0.8268355443885947,0.8761630700593988,0.8504036968121201,0.8536241799153953,0.813254503579401,0.6878726191211912 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator4,18497,0.9423149699951343,0.8603394287177556,0.8553445399309625,0.8414297832295382,0.8447669612001169,0.8209772911532445,0.6816297648376087 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator5,18497,0.9427474725631183,0.8504739031413244,0.8885644584056638,0.8443309477425908,0.845900104738327,0.8359523189963054,0.6860146732332417 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator6,18497,0.9360436827593663,0.9010454929869601,0.873381952000466,0.8435435157851304,0.8485171799304876,0.8085706584290521,0.7044441004029509 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator7,18497,0.9147429312861545,0.6188982155735954,0.6229150405240663,0.7637539651299388,0.7708538619726186,0.7350225445580263,0.5883971868251655 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator1,annotator8,18497,0.9129588581932205,0.7779626856672421,0.8077610247129487,0.7591811812278387,0.7621518762700554,0.7566376372466317,0.5740693907901211 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator3,18497,0.9304211493755744,0.7732483432213881,0.8409664912193489,0.8182961291954721,0.8280774732805408,0.7703689775641799,0.6651442530273897 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator4,18497,0.9321511596475104,0.8068647276493659,0.8292933890028875,0.8231870451826313,0.8327637260159387,0.7768103185982849,0.6726993253101411 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator5,18497,0.9114450992052765,0.7854126127518128,0.8390934632988478,0.7717095911017536,0.7789511259657479,0.7642760618260224,0.6153340734171374 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator6,18497,0.9408552738281883,0.8505022527990387,0.8478670540445572,0.8613311585677275,0.8635509212909428,0.8434928264019548,0.7143129483900974 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator7,18497,0.9036059901605665,0.5951947556160653,0.6216337867736852,0.746623158266871,0.75884779870345,0.6849576365764195,0.5729109662916494 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator2,annotator8,18497,0.8857652592312267,0.707848451359499,0.752344318296335,0.7008344299911284,0.7100375859117081,0.690723021975048,0.5392049314072255 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator4,18497,0.9424230956371303,0.9153052356720035,0.8461021624617533,0.8319577839175537,0.8320236569760092,0.7965284551568961,0.6449169751762426 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator5,18497,0.9312861545115424,0.8650799421178927,0.8440585299225909,0.8022465960579761,0.8030990184591813,0.7838655170140483,0.6121396032934081 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator6,18497,0.9143104287181705,0.9493098503049976,0.8506011255343653,0.7811859297343532,0.796154478228368,0.72006874813807,0.6283133767635412 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator7,18497,0.9125263556252365,0.6262599998843071,0.6176596847047876,0.7419555812257494,0.7439440931190664,0.6932187440333278,0.5431870695547001 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator3,annotator8,18497,0.9092825863653565,0.8456419380250744,0.8190695068254445,0.7335976005458585,0.7338364821224354,0.7103354720481876,0.5342023394861772 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator5,18497,0.9422609071741364,0.8005761040996336,0.8234086390941522,0.8343149960173124,0.8356131233383589,0.8476243500291111,0.6795756656805078 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator6,18497,0.9152835594961345,0.9012721300310107,0.8672340997481056,0.7842767687295319,0.7996093455317118,0.7292949442864118,0.6423469306412043 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator7,18497,0.9249608044547765,0.6415394424825286,0.6389753799353602,0.7790073607270296,0.7805111081977687,0.7343413871230232,0.5877142928832196 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator4,annotator8,18497,0.9080932043034006,0.7614060012125954,0.7900490325627516,0.7308626886085527,0.7313847453342788,0.7275099218159699,0.5321674351145445 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator6,18497,0.9090663350813646,0.8876626053055505,0.8195920314512181,0.7700330404290673,0.7802661933817312,0.7412932304409388,0.6179516980594643 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator7,18497,0.9067956965994486,0.6141168942161137,0.5965125031728115,0.7303975475268298,0.735419346017062,0.7243067492726258,0.5380185635983205 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator5,annotator8,18497,0.9195545223549765,0.8132284555031197,0.8089325459305607,0.7674347474010559,0.7678406654758821,0.7476171284661636,0.5595290256250727 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator7,18497,0.8786830296804887,0.5900087457022992,0.5930702493190987,0.6897157503796785,0.7094876908513972,0.6364238735662452,0.5319451351171405 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator6,annotator8,18497,0.8804671027734228,0.7536702499321207,0.7955398261879636,0.6934604345232147,0.7065529559485101,0.6593426799750073,0.5282322379743403 +Male_likely/Mouse069_20160709_16-02-03,Mouse069_20160709_16-02-03,Male_likely,v2,annotator7,annotator8,18497,0.9169595069470725,0.8250634998758023,0.6053313396282689,0.7545615112838121,0.7581470689095351,0.762402575618638,0.566239549450603 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator2,20199,0.9130650032179811,0.8466594873867529,0.8412156163382912,0.8156523551609771,0.8159082180291027,0.8094608876631887,0.6354793352849328 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator3,20199,0.8825684439823753,0.748298648584432,0.7692836243037616,0.7321089570250505,0.7383892479841662,0.7026191243063722,0.5335208495209679 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator4,20199,0.8903906133967028,0.7643744208207521,0.7601063883799432,0.7606465387494394,0.7671045809611835,0.7786983125217937,0.6028582389180911 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator5,20199,0.9014802713005594,0.792023759611793,0.6108795599734785,0.7790974540608716,0.7819349034721822,0.7673658484598535,0.5955587115931362 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator6,20199,0.8917768206346849,0.7997186684338033,0.591762956937206,0.7717436146931289,0.7741217940589799,0.7824628016544459,0.6061240823240825 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator7,20199,0.8341006980543592,0.6393256814343292,0.5857809432754676,0.6144946913550314,0.6344538211473766,0.5955043062471084,0.4578532273747675 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator1,annotator8,20199,0.8445962671419377,0.6392089106485277,0.6500669801213192,0.6192148476266599,0.6443150459596444,0.592011976783854,0.4484727737366257 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator3,20199,0.9072231298579138,0.7858393193881997,0.8204394013351339,0.7907945324494878,0.7988730233264819,0.7542941296787045,0.6165026575106475 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator4,20199,0.9163324917075103,0.8044849609347334,0.8174766155536468,0.8189106117614102,0.8249514119781238,0.82376710471756,0.6747106982653821 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator5,20199,0.9071241150552007,0.7971428324985125,0.6183742398526471,0.7942120651924607,0.7986201327887534,0.7818483191484662,0.6317322585052617 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator6,20199,0.9092529333135304,0.8287394398411436,0.6199509660572735,0.8102323436011752,0.8115649152679685,0.8025236552486713,0.6399791687348797 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator7,20199,0.8422694192781821,0.6523638645074766,0.6168066696743921,0.637101460965456,0.6577659069844388,0.588773385912427,0.4667198442993929 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator2,annotator8,20199,0.8566265656715679,0.6606210351167932,0.6848870949433877,0.6534907283809647,0.682547876675558,0.5978320211220193,0.4951421006755518 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator4,20199,0.919847517203822,0.8454279385548192,0.8312590409260743,0.8115759301642564,0.8164726706827002,0.7754082188732376,0.6280205270553587 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator5,20199,0.9127184514084856,0.8345029189431514,0.6088088166910975,0.7894959680115163,0.7911093474475902,0.7743915260662273,0.5991863048056866 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator6,20199,0.8881627803356602,0.8216113056495132,0.5799513348566927,0.7486878847604882,0.7569738647260712,0.710544988260386,0.559612885658958 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator7,20199,0.8639536610723303,0.6329306968129496,0.6030472454504007,0.6533819952413911,0.6607525340354365,0.6068724910864458,0.4289360483056797 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator3,annotator8,20199,0.8848457844447745,0.6698178394817426,0.6969902222172238,0.6884203100127748,0.6966394959054486,0.6306979941710947,0.46966201594630086 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator5,20199,0.8938066240903015,0.7739559924598843,0.5644026128816609,0.7565726872584789,0.7634559478633594,0.7835152477823172,0.6132130526127396 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator6,20199,0.9172236249319273,0.8566726502059364,0.6085255970576788,0.8206008021444531,0.823390049102989,0.806667350116147,0.646762670905911 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator7,20199,0.8797960295064112,0.6282248603374397,0.631189527262446,0.7065768074378687,0.7138390678672978,0.6288283072561712,0.47454546451857843 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator4,annotator8,20199,0.86608247933066,0.6044450649261892,0.6349277984677054,0.6587049427275407,0.6773648685002475,0.6012637069182147,0.47019024119732516 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator6,20199,0.8741521857517699,0.5884198149881648,0.5622344306331533,0.7228756731102217,0.7294828495300806,0.7228200982510433,0.5573329825809915 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator7,20199,0.8487548888558839,0.48195204192805946,0.44214960844964957,0.627261267101046,0.641408235739663,0.624063082805093,0.4445111265220188 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator5,annotator8,20199,0.8770731224318036,0.5130002860463072,0.5264169903599952,0.677560556437607,0.6917733518409743,0.6437020530484495,0.47479995733550545 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator7,20199,0.8512797663250656,0.4721898499200079,0.4584187746337381,0.6564018852760442,0.6731020530395965,0.5858170497881112,0.46622300496883523 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator6,annotator8,20199,0.8371701569384623,0.44591847764020853,0.459616825729189,0.6076543476626903,0.634923443234803,0.556586888115142,0.45561785977015423 +Male_likely/Mouse070_20160709_16-58-36,Mouse070_20160709_16-58-36,Male_likely,v1,annotator7,annotator8,20199,0.8986583494232387,0.8721327999746019,0.6909377494141218,0.7142456547769216,0.7232211537746822,0.6568325270250746,0.4889650849662982 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator2,19389,0.9402754138944762,0.8381695054925515,0.8483892188368447,0.8268337875960661,0.8271678118843289,0.8447435217808005,0.661460800270355 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator3,19389,0.9217597606890505,0.7610686443476773,0.7989265078606927,0.7566436934827359,0.763917375727055,0.7439603612994109,0.5838265614084498 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator4,19389,0.9404817164371551,0.8449172546054147,0.8533434769699227,0.8279578767143195,0.8288542989794385,0.8245005413573526,0.6590218465636651 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator5,19389,0.9322811903656713,0.791545543870917,0.6118818144041871,0.7971193154224909,0.7999519633493437,0.8225619100690558,0.628803386204921 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator6,19389,0.9303213162102223,0.8001013560714251,0.8264095743275512,0.7906822164988243,0.7935306656311926,0.7845939884203068,0.6033577590484291 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator7,19389,0.8944246737841044,0.7020178674230921,0.6998399621289408,0.6811266877391984,0.687980248085418,0.688339262380698,0.4906034995340089 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator1,annotator8,19389,0.8810665841456496,0.653511232044045,0.6800551151558972,0.6228899701544262,0.6321291131910552,0.6362733650602841,0.4562459848795748 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator3,19389,0.9310949507452679,0.7815749308440255,0.8163344485550784,0.7807232767576444,0.7854639636696732,0.7731647588091345,0.5970495022999436 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator4,19389,0.9325906441796895,0.8163476789298022,0.8152124320324968,0.8012461792233387,0.8017301675043326,0.8331227174022955,0.6426901910435655 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator5,19389,0.9450719480117592,0.8253934747120764,0.6338929729017571,0.8318984713017756,0.8332360236571319,0.8577446341306255,0.6709795692562258 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator6,19389,0.9306307700242406,0.7960999167982585,0.8146474522914424,0.7871156303246184,0.7883748873801136,0.8027014829897643,0.6053607624863057 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator7,19389,0.9052039816390737,0.7183177766907235,0.7205875010700349,0.7072934505667037,0.7124662920296345,0.7230510889900228,0.50745775834666 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator2,annotator8,19389,0.9065965238021558,0.7170309067154169,0.7528796043247811,0.6967070311252961,0.703902895585241,0.6839835020069287,0.5022853441440138 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator4,19389,0.9271236267987003,0.8446186767534914,0.8001408144451361,0.7687471827922994,0.7737894539969495,0.7640723723075377,0.5878216619680281 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator5,19389,0.9254216308215999,0.7942953037218867,0.5802583549375651,0.7529459047156566,0.7542959305506363,0.7727973441047442,0.5562717128396666 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator6,19389,0.933931610707102,0.8243150671942541,0.8025849629442895,0.7804722057732856,0.7816544084326196,0.7957786343754663,0.596569790028185 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator7,19389,0.9093816081283201,0.7324328259786315,0.7175802918960558,0.6963066954903561,0.6996337008038418,0.6948789735003423,0.480011895761926 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator3,annotator8,19389,0.9017999896848728,0.6859262918353103,0.6935831536665638,0.6514193282130579,0.6517975431970082,0.6659013036617644,0.4376964456354561 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator5,19389,0.9277425344267368,0.7658948565580759,0.587064683522712,0.7793624571488116,0.7805251455219289,0.8148540776022669,0.6165748850170155 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator6,19389,0.9362525143122389,0.8074653867420589,0.8260353136237398,0.8048772568167357,0.8063524289576703,0.808766623454617,0.6207388938442217 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator7,19389,0.906390221259477,0.7126602818220881,0.7282412990120576,0.7108703226793591,0.7141589575887463,0.6939262875818105,0.5015268986625632 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator4,annotator8,19389,0.8720408479034504,0.6067457287551113,0.6316929381477955,0.5855660755344952,0.5917288612156832,0.6117327342061267,0.43318756037728295 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator6,19389,0.9348599721491567,0.6062973664266768,0.606500753075549,0.7924694367237051,0.7925124370917068,0.8056601154327706,0.6114250886013669 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator7,19389,0.9012842333281758,0.51708948379207,0.5132023740957271,0.6826880453740356,0.6847246416764545,0.6954743582780228,0.46724909491953914 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator5,annotator8,19389,0.8928258290783434,0.49496423117112776,0.5110844342348949,0.6370461057526244,0.6396728826760427,0.666380832352792,0.43144256296187755 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator7,19389,0.8992212079013874,0.6985508291509155,0.6910175341828295,0.6754012023830314,0.677912454603706,0.6781964168530096,0.4697471571273756 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator6,annotator8,19389,0.8812728866883285,0.6334144619308218,0.6500298484972211,0.5966976311523307,0.5989875875143102,0.6248557309981643,0.41600915770990676 +Male_likely/Mouse074_20160709_19-15-23,Mouse074_20160709_19-15-23,Male_likely,v1,annotator7,annotator8,19389,0.8966940017535716,0.6505095890385194,0.6517404696819944,0.6455833857550266,0.6492980014714091,0.6770147369895256,0.43534333421095356 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator2,19955,0.8795790528689551,0.8039660039371263,0.6046825425229332,0.7767505490011927,0.7796227046768719,0.7239507770774711,0.5640662416093198 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator3,19955,0.824204460035079,0.668307892666259,0.6753759656070771,0.651066802225396,0.6602843262855617,0.5980512394778469,0.4383212841220321 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator4,19955,0.8722625908293661,0.7593666723963746,0.7682444287511642,0.7559352341805,0.7626353076321225,0.7079964290402119,0.5529165926040979 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator5,19955,0.857880230518667,0.731437424827425,0.7523364908382101,0.7218538487539441,0.7272276684729668,0.6967833084241082,0.5321959282858107 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator6,19955,0.8704084189426209,0.7969895960381409,0.6008639393067268,0.7563677577165828,0.7572291394396341,0.7088464719815245,0.530106263487259 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator7,19955,0.7227261338010523,0.597605897940227,0.5450269447813475,0.49646668893484747,0.5225270417827446,0.3460927149568656,0.2863523228115882 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator1,annotator8,19955,0.7873214733149586,0.6267663049861916,0.6199216224323866,0.5834498587102035,0.5905187686454448,0.5266741477026139,0.3902607476212761 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator3,19955,0.8550238035580056,0.5155830002231452,0.5336778463993469,0.7116663836013613,0.7212557840788887,0.6417755980232192,0.49939097211990924 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator4,19955,0.899173139564019,0.5850931753046391,0.6028625852796905,0.8059551657512669,0.8083061912412727,0.7507106752674644,0.6021644974149418 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator5,19955,0.8532197444249562,0.5400049682191662,0.5515603425647204,0.713792294872398,0.7227188990509851,0.6750842751223535,0.5255019471290986 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator6,19955,0.8504134302179904,0.5788310160050628,0.5685743692975009,0.7204305044528324,0.7264717957388238,0.6749401116710552,0.5100529794148032 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator7,19955,0.7671761463292408,0.45234098987848603,0.43385681036736434,0.5666652711536938,0.582693346526619,0.41236377248242057,0.32343954690814464 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator2,annotator8,19955,0.8166875469807066,0.4715266796924402,0.4780120100330089,0.6391743259265463,0.6445538826295371,0.5726818264320462,0.42432533114863685 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator4,19955,0.8812828864946128,0.7718011474707214,0.750935428543483,0.7535421997730312,0.757910296693244,0.6844289678529911,0.526485246895725 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator5,19955,0.8592833876221498,0.7366985436700592,0.7107131457518734,0.7003587760673069,0.7024642491175939,0.6693421080957604,0.46093452482433556 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator6,19955,0.8417439238286144,0.7848601532713589,0.5368487057893453,0.6816629630921512,0.6909002830016343,0.6287497275085983,0.4665810991356859 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator7,19955,0.753796041092458,0.5844195096397171,0.5496904339159455,0.5173296048552729,0.5463485458310566,0.3435140178026581,0.279685035517777 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator3,annotator8,19955,0.8394387371586068,0.6701715778703474,0.6660733610707562,0.6555989190765823,0.6563091695659928,0.5789844696336877,0.3869960832235458 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator5,19955,0.8749185667752443,0.7599777527985859,0.743579864178955,0.7465644449674295,0.7535843659997146,0.724386210941266,0.5663670314624534 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator6,19955,0.8690553745928339,0.8257211728929269,0.5684089864668795,0.7476112357402415,0.7572605933283982,0.7271035421125541,0.5702227519174066 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator7,19955,0.7742420446003508,0.5999315141684711,0.5853934749836329,0.5644254809437179,0.5799956378091267,0.38590872637603857,0.3156084452669765 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator4,annotator8,19955,0.8211475820596342,0.6177720797093467,0.6248389377149639,0.632978792920176,0.6342583873452554,0.5496999213965645,0.40629639180912247 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator6,19955,0.8581307942871461,0.7819263459904798,0.5653841334944878,0.7181075295884999,0.7217161003271225,0.6972300998077484,0.5136836432374227 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator7,19955,0.7265848158356302,0.5923080208458389,0.5409439988781629,0.4815835725960572,0.5150785107405713,0.323021074946246,0.2705770130087446 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator5,annotator8,19955,0.811676271611125,0.6520868784985202,0.6496067319796064,0.6059652692798974,0.6091328920504386,0.5474718210654447,0.35854202894002696 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator7,19955,0.7034327236281633,0.44618310086791574,0.39811932336599404,0.46242650036461586,0.4938540165333958,0.3124867665208459,0.26403697303591717 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator6,annotator8,19955,0.7864695565021298,0.488445931522051,0.48382434077155645,0.5770773918048828,0.5849777444967044,0.4954851289471205,0.3472245900259149 +Male_likely/Mouse077_20160709_18-29-34,Mouse077_20160709_18-29-34,Male_likely,v1,annotator7,annotator8,19955,0.7912803808569281,0.7331300515390899,0.626437558556547,0.5913935155574939,0.6159232544553515,0.41921025147643476,0.3496614203007648 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator2,17935,0.8709785335935322,0.8896248116541224,0.8264592763467764,0.7620958845374377,0.7732773395427279,0.6026836651012274,0.5443197534546121 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator3,17935,0.8428212991357681,0.771822920513013,0.7988220561618258,0.7025312379983264,0.7153412193136016,0.511343660876115,0.45699495537397306 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator4,17935,0.8836911067744634,0.8770137366823393,0.8650527284126553,0.7831741985084876,0.7832293143492379,0.62128883580868,0.535995109880994 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator5,17935,0.8561471982157792,0.8052550335923323,0.47791307217789586,0.7309629637209643,0.7444947381927122,0.5612706709667684,0.5005360534320038 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator6,17935,0.8531363256202955,0.8231826252149069,0.8229516890588876,0.7243680873573781,0.7352756256611408,0.537332441567065,0.47912828554684933 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator7,17935,0.8200167270699749,0.7702485216569807,0.7776437316318674,0.6612819177569147,0.6729981219159874,0.4567422603768572,0.40384872803145533 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator1,annotator8,17935,0.7784778366322832,0.7464560572829749,0.7507610148508009,0.5821521974336483,0.6094442730263986,0.360793047977237,0.35529009918826376 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator3,17935,0.8909952606635071,0.7622087613972508,0.8087802272171759,0.7901691684282768,0.7917304994367174,0.6789062521489698,0.5651395429648565 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator4,17935,0.8887092277669362,0.8282058559978603,0.8511410549399036,0.7949615796886446,0.8051644338949657,0.6528340457286118,0.5962451444643314 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator5,17935,0.914970727627544,0.8173323700112505,0.509649960479025,0.8380273297958726,0.8387467349321855,0.7539058119222607,0.6292060517551833 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator6,17935,0.8959576247560636,0.8096484088274299,0.8480380017596661,0.8017801466256812,0.80243582606969,0.6796183345857273,0.5695260298561223 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator7,17935,0.8745469751881796,0.7503642707543149,0.7840877661841062,0.7598113939338127,0.7607709370549396,0.637667846451072,0.5093179703599221 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator2,annotator8,17935,0.820183997769724,0.7100255907442135,0.7505320036153198,0.6483464459112158,0.6552466264121956,0.482348197030087,0.39097618845942206 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator4,17935,0.8757736269863395,0.9020884614302055,0.8360881097791516,0.7651293303735651,0.7777930132092751,0.6055438570926737,0.5643518429156003 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator5,17935,0.8896570950655144,0.8457353392145815,0.4817087539483559,0.7833928936231305,0.7836586164638789,0.666544691659118,0.5390765248302917 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator6,17935,0.9266239197100641,0.9212549926703572,0.8786567094002381,0.8559401129781774,0.8562022321305276,0.7617373687027191,0.6514535458488397 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator7,17935,0.9005854474491218,0.8356567881444814,0.8125675513206273,0.8037263815538512,0.8037949373329297,0.6879907196594287,0.5548333950352261 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator3,annotator8,17935,0.8336771675494843,0.7832530141051214,0.7637006740706399,0.6636815476603015,0.6681514039504516,0.4881565317681748,0.38244381765011426 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator5,17935,0.8698076386952885,0.7979194016866525,0.4795421801593019,0.7567073720192405,0.7692958863095312,0.6078387894122282,0.5489925499112375 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator6,17935,0.8880959018678561,0.8522114032525155,0.8634523655479663,0.7901964686045236,0.8007982905545642,0.6377494281526648,0.5862333012851919 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator7,17935,0.8531920825202118,0.7957460737598915,0.8132522565839153,0.7239727941541323,0.735533000469889,0.5448404833906223,0.49086383722847043 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator4,annotator8,17935,0.7838304990242543,0.7419620419372595,0.7561455109118401,0.5921835718564605,0.6182061797670052,0.37372101772686556,0.36568204838333346 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator6,17935,0.8900473933649289,0.49593032143704774,0.49990294883008524,0.7865955742962377,0.7867556438582766,0.6611586681656482,0.5427787248746417 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator7,17935,0.8544744912182882,0.4368484600896675,0.4426606211699443,0.7160042712492634,0.7160719530195487,0.57938316234073,0.4475437084223905 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator5,annotator8,17935,0.7972121550041817,0.39845688727945083,0.40617634906456185,0.5945677698258369,0.5984166959290729,0.4303539130055205,0.312482018083229 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator7,17935,0.8901031502648453,0.7895558034873518,0.7971901024002968,0.7855173411068007,0.7856127671402708,0.6657428868264286,0.5348896298854833 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator6,annotator8,17935,0.8340674658488988,0.7377300378933832,0.7492478102993788,0.6689470567023286,0.6747618077071534,0.5037359498737969,0.40055133015234234 +Undetermined/Mouse162_20161017_19-58-28,Mouse162_20161017_19-58-28,Undetermined,v1,annotator7,annotator8,17935,0.8478951770281572,0.7806395412282375,0.7854101824703338,0.6943501002081816,0.6991161828257386,0.5264251658004571,0.4185421240258996 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_summary.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_summary.csv new file mode 100644 index 0000000..64a2d8b --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_summary.csv @@ -0,0 +1,29 @@ +annotator_a,annotator_b,n_shared_record_versions,total_frames_compared,accuracy_mean,balanced_accuracy_mean,macro_f1_mean,cohen_kappa_mean,mcc_mean,ari_mean,nmi_mean +annotator1,annotator2,12,227288,0.9359922541744194,0.8915426679121471,0.8601791036448633,0.869331546261,0.8709313827134416,0.8252743827749467,0.7212765620003686 +annotator1,annotator3,12,227288,0.9178954098127846,0.8293433051444302,0.845978714481025,0.8247260606331185,0.8295976977702978,0.7609461485108531,0.6649559222222192 +annotator1,annotator4,12,227288,0.9262865642653595,0.8628546116120441,0.8504828144231292,0.8472158251222428,0.849136688703405,0.7952247579333188,0.6850753378958757 +annotator1,annotator5,12,227288,0.9278815217544804,0.8453901866155523,0.7281514688697742,0.8477342959592921,0.8504530745125917,0.8050405091198578,0.6974768748486467 +annotator1,annotator6,12,227288,0.9241094870226106,0.8595085992355928,0.7852292505206858,0.844560414125875,0.8471238382572359,0.7925257031176934,0.6879039368066405 +annotator1,annotator7,12,227288,0.8824016623415791,0.7447138613715679,0.7468172931083771,0.7523606910972487,0.7605723223194695,0.6760921262299213,0.5762327045846927 +annotator1,annotator8,12,227288,0.8723237105517638,0.7574908309722602,0.7672444321741296,0.7253533825588505,0.7342548714683668,0.6554373492323816,0.5438330504972086 +annotator2,annotator3,12,227288,0.9262153018789347,0.830942863895563,0.8586028368090427,0.8426224688651502,0.8470345775857764,0.7840862570812258,0.6861337929861616 +annotator2,annotator4,12,227288,0.9353048274317448,0.8606305926543683,0.8621210627858398,0.8638789111285575,0.8667592790919311,0.8206307320840693,0.7171120341874802 +annotator2,annotator5,12,227288,0.9302468631690499,0.8414088853817385,0.7212121889395813,0.8521307685354027,0.8546262077890914,0.8173326502235504,0.70620430934061 +annotator2,annotator6,12,227288,0.9315158937064064,0.8577752436669495,0.8077496481845433,0.8582900694219507,0.8597200089660376,0.8201079268010809,0.7085551253147017 +annotator2,annotator7,12,227288,0.890222544484494,0.7481338869669414,0.7588301631662021,0.7670186441757963,0.7741029734770658,0.6960825765770755,0.5891140224356203 +annotator2,annotator8,12,227288,0.8783943458576301,0.7585281495585509,0.7804773121451466,0.7387932194501675,0.7464835743629364,0.6665432907313064,0.553756976232506 +annotator3,annotator4,12,227288,0.9265355365812655,0.9025077485377714,0.8659508872928416,0.8393418976046121,0.8424976800287357,0.7800323265614103,0.6716647766049849 +annotator3,annotator5,12,227288,0.9253250739052188,0.8759659296798615,0.7266902549100186,0.8331928360085717,0.8343903842174941,0.7867606249550603,0.6596025167448332 +annotator3,annotator6,12,227288,0.9183822834832646,0.8990001590555488,0.7943758042234074,0.8258414255656493,0.8314028946612991,0.7648150879940258,0.668429355669343 +annotator3,annotator7,12,227288,0.8959864600583458,0.7773174626541636,0.7720630807725847,0.7675903698851401,0.7718804456386935,0.6916057252602021,0.5712267395951516 +annotator3,annotator8,12,227288,0.887489627567799,0.8016313397276029,0.8012619952070105,0.7437275911206389,0.745352149642459,0.6695325519505287,0.5292066222783239 +annotator4,annotator5,12,227288,0.9235978138891813,0.8430597219463157,0.7124033527854934,0.8360021145002295,0.8389563831572557,0.8003137511195112,0.6847240989385314 +annotator4,annotator6,12,227288,0.9256805400767601,0.8930334467112148,0.8130838121344399,0.8442119568163885,0.8489355036047672,0.7938714800585762,0.6938853723026531 +annotator4,annotator7,12,227288,0.8901091980227164,0.76312655778338,0.771608922042537,0.7641576220953885,0.7692049432696146,0.6813913222980529,0.5707611263737383 +annotator4,annotator8,12,227288,0.8716435422241745,0.7542685267621456,0.7725600748391379,0.7181957715282333,0.7244709367721781,0.6434192154300102,0.5248268706157345 +annotator5,annotator6,12,227288,0.9201940465762642,0.7415305770401601,0.6762108565173547,0.8306693303521192,0.8334858295259071,0.7895418751624823,0.6806462130451919 +annotator5,annotator7,12,227288,0.8847964739926472,0.6353983451671226,0.6321636422592558,0.7489011988374165,0.754568982370781,0.6811581362088118,0.5588025179251784 +annotator5,annotator8,12,227288,0.8840309236442837,0.6707779041869376,0.6718644874501324,0.7430793751984801,0.7465327933450226,0.6770747332017714,0.5424167099397231 +annotator6,annotator7,12,227288,0.8793588342727027,0.704454810074899,0.7060771351106943,0.7439996072870708,0.7532523338685744,0.6679095533636813,0.5704439816403739 +annotator6,annotator8,12,227288,0.872618241077344,0.7093164431356009,0.7233856939581376,0.7217683665918359,0.7303623379542041,0.6499924072292891,0.5408402275525868 +annotator7,annotator8,12,227288,0.8980392434353476,0.8394482972604456,0.7729740007115461,0.769257043767669,0.7740283257276013,0.7028107066588242,0.5705279220170613 diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/make_composite_calms21_task2_annotator_figure.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/make_composite_calms21_task2_annotator_figure.py new file mode 100644 index 0000000..ad9d5d7 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/make_composite_calms21_task2_annotator_figure.py @@ -0,0 +1,118 @@ +from pathlib import Path +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.gridspec import GridSpec + +OUTDIR = Path("results/calms21_task2_annotator_bias") +FIGDIR = Path("figures/calms21_task2_annotator_bias") +FIGDIR.mkdir(parents=True, exist_ok=True) + +summary = pd.read_csv(OUTDIR / "calms21_task2_pairwise_interannotator_summary.csv") +annot = pd.read_csv(OUTDIR / "calms21_task2_annotator_file_summary.csv") +label = pd.read_csv(OUTDIR / "calms21_task2_label_distribution_by_annotator.csv") +bout = pd.read_csv(OUTDIR / "calms21_task2_bout_stats_by_annotator.csv") + +summary["pair"] = ( + summary["annotator_a"].str.replace("annotator", "A", regex=False) + + "-" + + summary["annotator_b"].str.replace("annotator", "A", regex=False) +) + +# Heatmap tables +label_pivot = label.pivot_table( + index="behavior", + columns="annotator", + values="fraction_within_annotator", + fill_value=0, +) +label_pivot = label_pivot.loc[label_pivot.mean(axis=1).sort_values(ascending=False).index] + +bout_pivot = bout.pivot_table( + index="behavior", + columns="annotator", + values="mean_bout_duration_s", + fill_value=np.nan, +) +bout_pivot = bout_pivot.loc[bout_pivot.mean(axis=1).sort_values(ascending=False).index] + +# ---- Figure ---- +fig = plt.figure(figsize=(12, 9)) +gs = GridSpec(2, 2, figure=fig, height_ratios=[1, 1.2], width_ratios=[1, 1]) + +ax_a = fig.add_subplot(gs[0, 0]) +ax_b = fig.add_subplot(gs[0, 1]) +ax_c = fig.add_subplot(gs[1, 0]) +ax_d = fig.add_subplot(gs[1, 1]) + +# ------------------------ +# Panel a: annotation coverage +# ------------------------ +ax_a.bar(annot["annotator"], annot["total_frames"]) +ax_a.set_ylabel("Annotated frames") +ax_a.set_xlabel("Annotator") +ax_a.set_title("Annotation coverage") +ax_a.tick_params(axis="x", rotation=45) + +# ------------------------ +# Panel b: combined agreement +# ------------------------ +x = np.arange(len(summary)) +width = 0.35 +ax_b.bar(x - width/2, summary["macro_f1_mean"], width, label="Macro-F1") +ax_b.bar(x + width/2, summary["cohen_kappa_mean"], width, label="Cohen's kappa") +ax_b.axhline(0, linewidth=1) +ax_b.set_xticks(x) +ax_b.set_xticklabels(summary["pair"], rotation=45, ha="right") +ax_b.set_ylabel("Agreement score") +ax_b.set_xlabel("Annotator pair") +ax_b.set_title("Pairwise inter-annotator agreement") +ax_b.set_ylim(-0.15, 1.0) +ax_b.legend(frameon=False) + +# ------------------------ +# Panel c: label distribution heatmap +# ------------------------ +im1 = ax_c.imshow(label_pivot.values, aspect="auto") +ax_c.set_xticks(range(len(label_pivot.columns))) +ax_c.set_xticklabels(label_pivot.columns, rotation=45, ha="right") +ax_c.set_yticks(range(len(label_pivot.index))) +ax_c.set_yticklabels(label_pivot.index) +ax_c.set_title("Behavior label distribution") +ax_c.set_xlabel("Annotator") +ax_c.set_ylabel("Behavior") +cbar1 = fig.colorbar(im1, ax=ax_c, fraction=0.046, pad=0.04) +cbar1.set_label("Fraction within annotator") + +# ------------------------ +# Panel d: bout duration heatmap +# ------------------------ +im2 = ax_d.imshow(bout_pivot.values, aspect="auto") +ax_d.set_xticks(range(len(bout_pivot.columns))) +ax_d.set_xticklabels(bout_pivot.columns, rotation=45, ha="right") +ax_d.set_yticks(range(len(bout_pivot.index))) +ax_d.set_yticklabels(bout_pivot.index) +ax_d.set_title("Mean bout duration") +ax_d.set_xlabel("Annotator") +ax_d.set_ylabel("Behavior") +cbar2 = fig.colorbar(im2, ax=ax_d, fraction=0.046, pad=0.04) +cbar2.set_label("Mean bout duration (s)") + +# Panel letters +for ax, letter in zip([ax_a, ax_b, ax_c, ax_d], ["a", "b", "c", "d"]): + ax.text(-0.12, 1.05, letter, transform=ax.transAxes, + fontsize=16, fontweight="bold", va="top", ha="left") + +# Remove top/right spines for cleaner journal style +for ax in [ax_a, ax_b, ax_c, ax_d]: + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + +plt.tight_layout() +plt.savefig(FIGDIR / "calms21_task2_annotator_bias_composite.png", dpi=600) +plt.savefig(FIGDIR / "calms21_task2_annotator_bias_composite.pdf") +plt.close() + +print("Saved:") +print(FIGDIR / "calms21_task2_annotator_bias_composite.png") +print(FIGDIR / "calms21_task2_annotator_bias_composite.pdf") diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/plot_calms21_task2_annotator_bias.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/plot_calms21_task2_annotator_bias.py new file mode 100644 index 0000000..2e9e25c --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/plot_calms21_task2_annotator_bias.py @@ -0,0 +1,133 @@ +from pathlib import Path +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np + +OUTDIR = Path("results/calms21_task2_annotator_bias") +FIGDIR = Path("figures/calms21_task2_annotator_bias") +FIGDIR.mkdir(parents=True, exist_ok=True) + +summary = pd.read_csv(OUTDIR / "calms21_task2_pairwise_interannotator_summary.csv") +annot = pd.read_csv(OUTDIR / "calms21_task2_annotator_file_summary.csv") +label = pd.read_csv(OUTDIR / "calms21_task2_label_distribution_by_annotator.csv") +bout = pd.read_csv(OUTDIR / "calms21_task2_bout_stats_by_annotator.csv") + +summary["pair"] = ( + summary["annotator_a"].str.replace("annotator", "A", regex=False) + + "-" + + summary["annotator_b"].str.replace("annotator", "A", regex=False) +) + +# ------------------------------------------------------------ +# Plot 1: annotated frames per annotator +# ------------------------------------------------------------ +plt.figure(figsize=(5.5, 4)) +plt.bar(annot["annotator"], annot["total_frames"]) +plt.xticks(rotation=45, ha="right") +plt.ylabel("Annotated frames") +plt.xlabel("Annotator") +plt.title("CalMS21 Task 2 annotation coverage") +plt.tight_layout() +plt.savefig(FIGDIR / "annotated_frames_per_annotator.png", dpi=600) +plt.savefig(FIGDIR / "annotated_frames_per_annotator.pdf") +plt.close() + +# ------------------------------------------------------------ +# Plot 2: pairwise Macro-F1 +# ------------------------------------------------------------ +plt.figure(figsize=(7, 4)) +plt.bar(summary["pair"], summary["macro_f1_mean"]) +plt.xticks(rotation=45, ha="right") +plt.ylabel("Macro-F1") +plt.xlabel("Annotator pair") +plt.title("Pairwise inter-annotator agreement") +plt.ylim(0, 1) +plt.tight_layout() +plt.savefig(FIGDIR / "pairwise_macro_f1.png", dpi=600) +plt.savefig(FIGDIR / "pairwise_macro_f1.pdf") +plt.close() + +# ------------------------------------------------------------ +# Plot 3: pairwise Cohen's kappa +# ------------------------------------------------------------ +plt.figure(figsize=(7, 4)) +plt.bar(summary["pair"], summary["cohen_kappa_mean"]) +plt.axhline(0, linewidth=1) +plt.xticks(rotation=45, ha="right") +plt.ylabel("Cohen's kappa") +plt.xlabel("Annotator pair") +plt.title("Pairwise inter-annotator reliability") +plt.ylim(-0.15, 0.85) +plt.tight_layout() +plt.savefig(FIGDIR / "pairwise_cohen_kappa.png", dpi=600) +plt.savefig(FIGDIR / "pairwise_cohen_kappa.pdf") +plt.close() + +# ------------------------------------------------------------ +# Plot 4: behavior label distribution by annotator +# ------------------------------------------------------------ +pivot = label.pivot_table( + index="behavior", + columns="annotator", + values="fraction_within_annotator", + fill_value=0, +) +pivot = pivot.loc[pivot.mean(axis=1).sort_values(ascending=False).index] + +plt.figure(figsize=(6.5, max(4, 0.35 * len(pivot)))) +plt.imshow(pivot.values, aspect="auto") +plt.xticks(range(len(pivot.columns)), pivot.columns, rotation=45, ha="right") +plt.yticks(range(len(pivot.index)), pivot.index) +plt.colorbar(label="Fraction within annotator") +plt.title("Behavior label distribution by annotator") +plt.tight_layout() +plt.savefig(FIGDIR / "label_distribution_by_annotator.png", dpi=600) +plt.savefig(FIGDIR / "label_distribution_by_annotator.pdf") +plt.close() + +# ------------------------------------------------------------ +# Plot 5: mean bout duration by annotator +# ------------------------------------------------------------ +bout_pivot = bout.pivot_table( + index="behavior", + columns="annotator", + values="mean_bout_duration_s", + fill_value=np.nan, +) +bout_pivot = bout_pivot.loc[bout_pivot.mean(axis=1).sort_values(ascending=False).index] + +plt.figure(figsize=(6.5, max(4, 0.35 * len(bout_pivot)))) +plt.imshow(bout_pivot.values, aspect="auto") +plt.xticks(range(len(bout_pivot.columns)), bout_pivot.columns, rotation=45, ha="right") +plt.yticks(range(len(bout_pivot.index)), bout_pivot.index) +plt.colorbar(label="Mean bout duration (s)") +plt.title("Mean bout duration by annotator") +plt.tight_layout() +plt.savefig(FIGDIR / "bout_duration_by_annotator.png", dpi=600) +plt.savefig(FIGDIR / "bout_duration_by_annotator.pdf") +plt.close() + +# ------------------------------------------------------------ +# Plot 6: combined agreement figure, better for reviewer/rebuttal +# ------------------------------------------------------------ +x = np.arange(len(summary)) +width = 0.35 + +plt.figure(figsize=(8, 4.5)) +plt.bar(x - width / 2, summary["macro_f1_mean"], width, label="Macro-F1") +plt.bar(x + width / 2, summary["cohen_kappa_mean"], width, label="Cohen's kappa") +plt.axhline(0, linewidth=1) +plt.xticks(x, summary["pair"], rotation=45, ha="right") +plt.ylabel("Agreement score") +plt.xlabel("Annotator pair") +plt.title("CalMS21 Task 2 annotator-style variability") +plt.legend(frameon=False) +plt.ylim(-0.15, 1.0) +plt.tight_layout() +plt.savefig(FIGDIR / "combined_pairwise_agreement.png", dpi=600) +plt.savefig(FIGDIR / "combined_pairwise_agreement.pdf") +plt.close() + +print("Saved plots:") +for p in sorted(FIGDIR.glob("*")): + print(p) diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/plot_combined_calms21_mars1_annotator_bias.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/plot_combined_calms21_mars1_annotator_bias.py new file mode 100644 index 0000000..5848a2f --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/annotator_bout_metrics/scripts/plot_combined_calms21_mars1_annotator_bias.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python3 +"""Reproduce the final six-panel CalMS21 Task 2 and MARS1 annotator/bout figure. + +This is a portable extraction of the final executed code cell from +``annotator-bias.ipynb``. Only file-location handling and notebook display +calls were changed; calculations, panel definitions, and plotting settings +were preserved. +""" + +# ============================================================ +# FIXED combined annotator-bias plot +# CalMS21 Task 2 + MARS1 +# +# Panels: +# a: CalMS21 pairwise agreement +# b: MARS1 pairwise agreement, same dotplot style as a +# c: CalMS21 label distribution +# d: MARS1 positive-frame fraction +# e: CalMS21 bout duration +# f: MARS1 bout duration +# +# Fixes: +# - remove global title +# - use one shared centered legend for panels a and b +# - move MARS coverage inset to x ~ 0.26 in main data coordinates +# ============================================================ + +from pathlib import Path +import numpy as np +import pandas as pd +import matplotlib as mpl +import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec +from mpl_toolkits.axes_grid1.inset_locator import inset_axes +from matplotlib.colors import LinearSegmentedColormap, Normalize + +# ============================================================ +# Paths +# ============================================================ + +ROOT = Path(__file__).resolve().parents[1] + +CALMS_DIR = ROOT / "results/calms21_task2_annotator_bias" +MARS_DIR = ROOT / "results/mars1_annotator_bias" + +FIGDIR = ROOT / "figures" +FIGDIR.mkdir(parents=True, exist_ok=True) + +STYLE = ROOT / "lisbet.mplstyle" + +required = [ + CALMS_DIR / "calms21_task2_pairwise_interannotator_summary.csv", + CALMS_DIR / "calms21_task2_annotator_file_summary.csv", + CALMS_DIR / "calms21_task2_label_distribution_by_annotator.csv", + CALMS_DIR / "calms21_task2_bout_stats_by_annotator.csv", + MARS_DIR / "mars1_pairwise_interannotator_summary.csv", + MARS_DIR / "mars1_annotator_file_summary.csv", + MARS_DIR / "mars1_behavior_positive_fraction_by_annotator.csv", + MARS_DIR / "mars1_bout_stats_by_annotator.csv", +] + +missing = [p for p in required if not p.exists()] +if missing: + print("Missing required files:") + for p in missing: + print(" -", p) + raise FileNotFoundError( + "Please copy/extract annotator_bias_results_csvs_FIXED_20260701.tar.gz first." + ) + +# ============================================================ +# Colors +# ============================================================ + +COL_BLUE = "#68a8e0" +COL_GREEN = "#9ad175" +COL_ORANGE = "#f9a24b" +COL_GRAY = "#8a8a8a" +COL_LINE = "#d6d6d6" +COL_BOX = "#d0d0d0" + +label_cmap = LinearSegmentedColormap.from_list( + "lisbet_label", + ["#f4f4f4", COL_BLUE, COL_GREEN], +) + +bout_cmap = LinearSegmentedColormap.from_list( + "lisbet_bout", + ["#f4f4f4", COL_BLUE, COL_ORANGE, COL_GREEN], +) + +# ============================================================ +# Helper functions +# ============================================================ + +def short_annotator(x): + return str(x).replace("annotator", "A") + + +def add_panel_letter(ax, letter, x=-0.16, y=1.05): + ax.text( + x, + y, + letter, + transform=ax.transAxes, + fontsize=9, + fontweight="normal", + va="top", + ha="left", + color="black", + ) + + +def prepare_calms_pairwise(df): + df = df.copy() + df["pair"] = ( + df["annotator_a"].map(short_annotator) + + "–" + + df["annotator_b"].map(short_annotator) + ) + return df.sort_values("macro_f1_mean", ascending=True).reset_index(drop=True) + + +def prepare_mars_pairwise(df): + df = df.copy() + df["pair"] = ( + df["annotator_a"].map(short_annotator) + + "–" + + df["annotator_b"].map(short_annotator) + ) + return df.sort_values("macro_f1_mean", ascending=True).reset_index(drop=True) + + +def prepare_annot(df): + df = df.copy() + df["annotator_short"] = df["annotator"].map(short_annotator) + return df + + +def prepare_label_pivot(df): + piv = df.pivot_table( + index="behavior", + columns="annotator", + values="fraction_within_annotator", + fill_value=0, + ) + piv.columns = [short_annotator(c) for c in piv.columns] + piv = piv.loc[piv.mean(axis=1).sort_values(ascending=False).index] + return piv + + +def prepare_mars_positive_pivot(df): + piv = df.pivot_table( + index="behavior", + columns="annotator", + values="fraction_within_annotator", + fill_value=0, + ) + piv.columns = [short_annotator(c) for c in piv.columns] + + preferred = ["sniff", "mount", "aggressivemount", "attack"] + order = [b for b in preferred if b in piv.index] + rest = [b for b in piv.index if b not in order] + + return piv.loc[order + rest] + + +def prepare_bout_pivot(df, dataset): + piv = df.pivot_table( + index="behavior", + columns="annotator", + values="mean_bout_duration_s", + fill_value=np.nan, + ) + piv.columns = [short_annotator(c) for c in piv.columns] + + if dataset == "calms": + preferred = ["other", "mount", "attack", "investigation"] + else: + preferred = ["sniff", "mount", "aggressivemount", "attack"] + + order = [b for b in preferred if b in piv.index] + rest = [b for b in piv.index if b not in order] + + if order: + piv = piv.loc[order + rest] + else: + piv = piv.loc[piv.mean(axis=1).sort_values(ascending=False).index] + + return piv + + +# ============================================================ +# Plot functions +# ============================================================ + +def plot_calms_pairwise(ax, df, annot_df, title): + y = np.arange(len(df)) + + for i, row in df.iterrows(): + ax.plot( + [row["cohen_kappa_mean"], row["macro_f1_mean"]], + [i, i], + color=COL_LINE, + linewidth=0.55, + alpha=0.95, + zorder=1, + solid_capstyle="round", + ) + + ax.scatter( + df["cohen_kappa_mean"], + y, + s=30, + color=COL_BLUE, + edgecolor="white", + linewidth=0.4, + label="Cohen's κ", + zorder=3, + ) + + ax.scatter( + df["macro_f1_mean"], + y, + s=30, + color=COL_GREEN, + edgecolor="white", + linewidth=0.4, + label="Macro-F1", + zorder=4, + ) + + ax.axvline(0, color="black", linewidth=0.5) + ax.set_yticks(y) + ax.set_yticklabels(df["pair"], fontsize=6) + ax.set_xlabel("Agreement score") + ax.set_ylabel("Annotator pair") + ax.set_xlim(-0.12, 1.0) + ax.set_title(title, loc="left", pad=4) + ax.grid(axis="x", color="#ededed", linewidth=0.4) + ax.set_axisbelow(True) + + # No local legend. Shared legend is added later. + + ax_inset = inset_axes( + ax, + width="100%", + height="100%", + bbox_to_anchor=(0.56, 0.34, 0.36, 0.24), + bbox_transform=ax.transAxes, + loc="lower left", + borderpad=0, + ) + + ax_inset.bar( + annot_df["annotator_short"], + annot_df["total_frames"] / 1000, + color=COL_GRAY, + width=0.72, + zorder=3, + ) + + ax_inset.set_facecolor("white") + for spine in ax_inset.spines.values(): + spine.set_visible(True) + spine.set_color(COL_BOX) + spine.set_linewidth(0.8) + + ax_inset.set_title("coverage", fontsize=6, pad=2) + ax_inset.set_ylabel("frames\n×10³", fontsize=5) + ax_inset.tick_params(axis="both", labelsize=5, length=2) + ax_inset.grid(axis="y", color="#ececec", linewidth=0.35) + ax_inset.set_axisbelow(True) + + +def plot_mars_pairwise(ax, df, annot_df, title): + y = np.arange(len(df)) + + for i, row in df.iterrows(): + ax.plot( + [row["cohen_kappa_mean"], row["macro_f1_mean"]], + [i, i], + color=COL_LINE, + linewidth=0.45, + alpha=0.95, + zorder=1, + solid_capstyle="round", + ) + + ax.scatter( + df["cohen_kappa_mean"], + y, + s=20, + color=COL_BLUE, + edgecolor="white", + linewidth=0.35, + label="Cohen's κ", + zorder=3, + ) + + ax.scatter( + df["macro_f1_mean"], + y, + s=20, + color=COL_GREEN, + edgecolor="white", + linewidth=0.35, + label="Macro-F1", + zorder=4, + ) + + ax.axvline(0, color="black", linewidth=0.5) + ax.set_yticks(y) + ax.set_yticklabels(df["pair"], fontsize=4.3) + ax.set_xlabel("Agreement score") + ax.set_ylabel("Annotator pair") + + # Keep the better MARS panel range. + xlim = (-0.03, 0.92) + ax.set_xlim(*xlim) + + ax.set_title(title, loc="left", pad=4) + ax.grid(axis="x", color="#ededed", linewidth=0.4) + ax.set_axisbelow(True) + + # No local legend. Shared legend is added later. + + # Coverage inset positioned around x ≈ 0.26 in main data coordinates. + # This avoids overlap with y-axis labels but keeps it in the empty zone. + inset_w = 0.32 + inset_h = 0.22 + inset_y0 = 0.36 + + desired_x_center_data = 0.26 + x_center_frac = (desired_x_center_data - xlim[0]) / (xlim[1] - xlim[0]) + inset_x0 = x_center_frac - inset_w / 2 + inset_x0 = max(0.02, min(1 - inset_w - 0.02, inset_x0)) + + ax_inset = inset_axes( + ax, + width="100%", + height="100%", + bbox_to_anchor=(inset_x0, inset_y0, inset_w, inset_h), + bbox_transform=ax.transAxes, + loc="lower left", + borderpad=0, + ) + + ax_inset.bar( + annot_df["annotator_short"], + annot_df["total_frames"] / 1000, + color=COL_GRAY, + width=0.72, + zorder=3, + ) + + ax_inset.set_facecolor("white") + for spine in ax_inset.spines.values(): + spine.set_visible(True) + spine.set_color(COL_BOX) + spine.set_linewidth(0.8) + + ax_inset.set_title("coverage", fontsize=6, pad=2) + ax_inset.set_ylabel("frames\n×10³", fontsize=5) + ax_inset.tick_params(axis="both", labelsize=5, length=2) + ax_inset.grid(axis="y", color="#ececec", linewidth=0.35) + ax_inset.set_axisbelow(True) + + +def plot_heatmap(ax, pivot, title, cmap, cbar_label, vmax=None): + vals = pivot.values.astype(float) + + if vmax is None: + if np.isfinite(vals).any(): + vmax = np.nanpercentile(vals, 95) + vmax = max(0.01, vmax) + else: + vmax = 1.0 + + im = ax.imshow( + vals, + aspect="auto", + cmap=cmap, + norm=Normalize(vmin=0, vmax=vmax), + ) + + ax.set_xticks(np.arange(pivot.shape[1])) + ax.set_xticklabels(pivot.columns, fontsize=6) + + ax.set_yticks(np.arange(pivot.shape[0])) + ax.set_yticklabels(pivot.index, fontsize=6) + + ax.set_title(title, loc="left", pad=4) + ax.set_xlabel("Annotator") + ax.set_ylabel("Behavior") + + ax.set_xticks(np.arange(-0.5, pivot.shape[1], 1), minor=True) + ax.set_yticks(np.arange(-0.5, pivot.shape[0], 1), minor=True) + ax.grid(which="minor", color="white", linestyle="-", linewidth=0.35) + ax.tick_params(which="minor", bottom=False, left=False) + + cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.03) + cbar.set_label(cbar_label) + cbar.ax.tick_params(labelsize=5, length=2) + + +# ============================================================ +# Load fixed data +# ============================================================ + +calms_pair_raw = pd.read_csv(CALMS_DIR / "calms21_task2_pairwise_interannotator_summary.csv") +calms_annot_raw = pd.read_csv(CALMS_DIR / "calms21_task2_annotator_file_summary.csv") +calms_label_raw = pd.read_csv(CALMS_DIR / "calms21_task2_label_distribution_by_annotator.csv") +calms_bout_raw = pd.read_csv(CALMS_DIR / "calms21_task2_bout_stats_by_annotator.csv") + +mars_pair_raw = pd.read_csv(MARS_DIR / "mars1_pairwise_interannotator_summary.csv") +mars_annot_raw = pd.read_csv(MARS_DIR / "mars1_annotator_file_summary.csv") +mars_positive_raw = pd.read_csv(MARS_DIR / "mars1_behavior_positive_fraction_by_annotator.csv") +mars_bout_raw = pd.read_csv(MARS_DIR / "mars1_bout_stats_by_annotator.csv") + +calms_pair = prepare_calms_pairwise(calms_pair_raw) +calms_annot = prepare_annot(calms_annot_raw) +calms_label = prepare_label_pivot(calms_label_raw) +calms_bout = prepare_bout_pivot(calms_bout_raw, dataset="calms") + +mars_pair = prepare_mars_pairwise(mars_pair_raw) +mars_annot = prepare_annot(mars_annot_raw) +mars_positive = prepare_mars_positive_pivot(mars_positive_raw) +mars_bout = prepare_bout_pivot(mars_bout_raw, dataset="mars") + +# ============================================================ +# Save numerical summary +# ============================================================ + +summary = pd.DataFrame([ + { + "dataset": "CalMS21 Task 2", + "n_annotators": calms_annot["annotator"].nunique(), + "n_pairwise_comparisons": len(calms_pair_raw), + "total_frames_annotated": int(calms_annot["total_frames"].sum()), + "accuracy_mean": calms_pair_raw["accuracy_mean"].mean(), + "balanced_accuracy_mean": calms_pair_raw["balanced_accuracy_mean"].mean(), + "macro_f1_mean": calms_pair_raw["macro_f1_mean"].mean(), + "cohen_kappa_mean": calms_pair_raw["cohen_kappa_mean"].mean(), + "mcc_mean": calms_pair_raw["mcc_mean"].mean(), + }, + { + "dataset": "MARS1", + "n_annotators": mars_annot["annotator"].nunique(), + "n_pairwise_comparisons": len(mars_pair_raw), + "total_frames_annotated": int(mars_annot["total_frames"].sum()), + "accuracy_mean": mars_pair_raw["accuracy_mean"].mean(), + "balanced_accuracy_mean": mars_pair_raw["balanced_accuracy_mean"].mean(), + "macro_f1_mean": mars_pair_raw["macro_f1_mean"].mean(), + "cohen_kappa_mean": mars_pair_raw["cohen_kappa_mean"].mean(), + "mcc_mean": mars_pair_raw["mcc_mean"].mean(), + }, +]) + +summary.to_csv(FIGDIR / "combined_annotator_bias_summary_shared_legend.csv", index=False) + +print("Combined numerical summary:") +print(summary.to_string(index=False)) + +print("MARS positive-frame fraction:") +print(mars_positive.to_string(index=False)) + +print("MARS bout duration:") +print(mars_bout.to_string(index=False)) + +# ============================================================ +# Plot +# ============================================================ + +plt.close("all") + +context = plt.style.context(STYLE) if STYLE.exists() else plt.rc_context() + +with context: + mpl.rcParams["figure.constrained_layout.use"] = False + mpl.rcParams["figure.autolayout"] = False + mpl.rcParams["savefig.bbox"] = "standard" + mpl.rcParams["pdf.fonttype"] = 42 + mpl.rcParams["ps.fonttype"] = 42 + mpl.rcParams["svg.fonttype"] = "none" + + fig = plt.figure(figsize=(8.2, 9.0), constrained_layout=False) + + try: + fig.set_layout_engine(None) + except Exception: + pass + + gs = GridSpec( + 3, + 2, + figure=fig, + height_ratios=[2.75, 1.05, 1.05], + width_ratios=[1.0, 1.0], + wspace=0.45, + hspace=0.72, + ) + + ax_a = fig.add_subplot(gs[0, 0]) + ax_b = fig.add_subplot(gs[0, 1]) + ax_c = fig.add_subplot(gs[1, 0]) + ax_d = fig.add_subplot(gs[1, 1]) + ax_e = fig.add_subplot(gs[2, 0]) + ax_f = fig.add_subplot(gs[2, 1]) + + plot_calms_pairwise( + ax_a, + calms_pair, + calms_annot, + "CalMS21 Task 2 agreement", + ) + + plot_mars_pairwise( + ax_b, + mars_pair, + mars_annot, + "MARS1 agreement", + ) + + # Shared legend for panels a and b + handles, labels = ax_a.get_legend_handles_labels() + + fig.legend( + handles, + labels, + loc="upper center", + bbox_to_anchor=(0.52, 0.965), + ncol=2, + frameon=False, + fontsize=6, + handletextpad=0.4, + columnspacing=1.2, + ) + + plot_heatmap( + ax_c, + calms_label, + "CalMS21 label distribution", + label_cmap, + "Fraction", + ) + + plot_heatmap( + ax_d, + mars_positive, + "MARS1 positive-frame fraction", + label_cmap, + "Positive fraction", + ) + + plot_heatmap( + ax_e, + calms_bout, + "CalMS21 bout duration", + bout_cmap, + "Mean duration (s)", + ) + + plot_heatmap( + ax_f, + mars_bout, + "MARS1 bout duration", + bout_cmap, + "Mean duration (s)", + ) + + add_panel_letter(ax_a, "a", x=-0.18, y=1.04) + add_panel_letter(ax_b, "b", x=-0.18, y=1.04) + add_panel_letter(ax_c, "c", x=-0.18, y=1.10) + add_panel_letter(ax_d, "d", x=-0.18, y=1.10) + add_panel_letter(ax_e, "e", x=-0.18, y=1.10) + add_panel_letter(ax_f, "f", x=-0.18, y=1.10) + + # No global title. + + fig.subplots_adjust( + left=0.09, + right=0.965, + bottom=0.075, + top=0.925, + wspace=0.45, + hspace=0.72, + ) + + out_png = FIGDIR / "combined_calms21_mars1_annotator_bias_shared_legend.png" + out_pdf = FIGDIR / "combined_calms21_mars1_annotator_bias_shared_legend.pdf" + out_svg = FIGDIR / "combined_calms21_mars1_annotator_bias_shared_legend.svg" + + fig.savefig(out_png, dpi=600) + fig.savefig(out_pdf) + fig.savefig(out_svg) + + plt.show() + +print("Saved:") +print(out_png) +print(out_pdf) +print(out_svg) +print(FIGDIR / "combined_annotator_bias_summary_shared_legend.csv") \ No newline at end of file diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/plot_calms21_posthoc_allseeds.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/plot_calms21_posthoc_allseeds.py new file mode 100644 index 0000000..5bb5c2a --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/plot_calms21_posthoc_allseeds.py @@ -0,0 +1,256 @@ +from pathlib import Path +import argparse +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib as mpl + +parser = argparse.ArgumentParser( + description="Aggregate five matched CalMS21 decoding runs and reproduce the all-seed figure." +) +parser.add_argument( + "--root", + type=Path, + required=True, + help="Directory containing reviewer_2_2_calms21_posthoc_seed0 through seed4", +) +parser.add_argument( + "--output", + type=Path, + default=None, + help="Output directory; defaults to /reviewer_2_2_calms21_posthoc_allseeds", +) +args = parser.parse_args() +ROOT = args.root.expanduser().resolve() +OUT = (args.output.expanduser().resolve() if args.output else ROOT / "reviewer_2_2_calms21_posthoc_allseeds") +FIG = OUT / "figures" +RES = OUT / "results" +FIG.mkdir(parents=True, exist_ok=True) +RES.mkdir(parents=True, exist_ok=True) + +mpl.rcParams.update({ + "font.family": "sans-serif", + "font.sans-serif": ["DejaVu Sans"], + "pdf.fonttype": 42, + "ps.fonttype": 42, + "svg.fonttype": "none", + "font.size": 8, + "axes.titlesize": 9, + "axes.labelsize": 8, + "xtick.labelsize": 7, + "ytick.labelsize": 7, + "legend.fontsize": 7, + "axes.linewidth": 0.7, +}) + +MODEL_ORDER = ["all_tasks", "without_cons", "without_order", "without_shift", "without_warp"] +MODEL_LABELS = ["All tasks", "Without cons", "Without order", "Without shift", "Without warp"] +TASK_ORDER = ["cons", "order", "shift", "warp"] +CLASS_ORDER = ["attack", "investigation", "mount", "other"] + +label_map = dict(zip(MODEL_ORDER, MODEL_LABELS)) + +summary_rows = [] +class_rows = [] + +for seed in range(5): + base = ROOT / f"reviewer_2_2_calms21_posthoc_seed{seed}" / "results" + + s = pd.read_csv(base / "calms21_model_summary.csv") + s["seed"] = seed + summary_rows.append(s) + + c = pd.read_csv(base / "calms21_per_class_metrics.csv") + c["seed"] = seed + class_rows.append(c) + +summary = pd.concat(summary_rows, ignore_index=True) +perclass = pd.concat(class_rows, ignore_index=True) + +summary.to_csv(RES / "allseed_calms21_model_summary_long.csv", index=False) +perclass.to_csv(RES / "allseed_calms21_per_class_metrics_long.csv", index=False) + +# Aggregate overall metrics +overall = ( + summary + .groupby(["model", "label", "removed_task"], as_index=False) + .agg( + accuracy_mean=("accuracy", "mean"), + accuracy_sd=("accuracy", "std"), + balanced_accuracy_mean=("balanced_accuracy", "mean"), + balanced_accuracy_sd=("balanced_accuracy", "std"), + macro_f1_mean=("macro_f1", "mean"), + macro_f1_sd=("macro_f1", "std"), + ) +) + +overall["model"] = pd.Categorical(overall["model"], MODEL_ORDER, ordered=True) +overall = overall.sort_values("model") +overall.to_csv(RES / "allseed_calms21_model_summary_mean_sd.csv", index=False) + +# Per-class F1 mean +pc = ( + perclass + .groupby(["model", "label", "class_name"], as_index=False) + .agg(f1_mean=("f1", "mean"), f1_sd=("f1", "std")) +) + +pc["model"] = pd.Categorical(pc["model"], MODEL_ORDER, ordered=True) +pc["class_name"] = pd.Categorical(pc["class_name"], CLASS_ORDER, ordered=True) +pc = pc.sort_values(["class_name", "model"]) +pc.to_csv(RES / "allseed_calms21_per_class_f1_mean_sd.csv", index=False) + +# Task-removal effect per seed/class: all_tasks - without_task +effect_rows = [] + +for seed in range(5): + sub = perclass[perclass["seed"] == seed].copy() + all_f1 = sub[sub["model"] == "all_tasks"].set_index("class_name")["f1"] + + for task in TASK_ORDER: + model = f"without_{task}" + w = sub[sub["model"] == model].set_index("class_name")["f1"] + + for cls in CLASS_ORDER: + if cls in all_f1.index and cls in w.index: + effect_rows.append({ + "seed": seed, + "removed_task": task, + "class_name": cls, + "delta_f1": all_f1.loc[cls] - w.loc[cls], + }) + +effects = pd.DataFrame(effect_rows) +effects.to_csv(RES / "allseed_task_removal_effects_long.csv", index=False) + +eff = ( + effects + .groupby(["class_name", "removed_task"], as_index=False) + .agg(delta_f1_mean=("delta_f1", "mean"), delta_f1_sd=("delta_f1", "std")) +) + +eff["class_name"] = pd.Categorical(eff["class_name"], CLASS_ORDER, ordered=True) +eff["removed_task"] = pd.Categorical(eff["removed_task"], TASK_ORDER, ordered=True) +eff = eff.sort_values(["class_name", "removed_task"]) +eff.to_csv(RES / "allseed_task_removal_effects_mean_sd.csv", index=False) + +def clean_axis(ax): + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.tick_params(width=0.7, length=3) + +def panel_label(ax, label): + ax.text(-0.15, 1.08, label, transform=ax.transAxes, + fontsize=12, fontweight="normal", ha="left", va="top") + +fig, axes = plt.subplots( + 1, 3, + figsize=(11.2, 3.2), + gridspec_kw={"width_ratios": [1.5, 1.25, 1.1], "wspace": 0.50} +) + +ax_c, ax_d, ax_e = axes + +# Panel c: per-behavior decoding +mat = ( + pc.pivot(index="class_name", columns="label", values="f1_mean") + .reindex(index=CLASS_ORDER, columns=MODEL_LABELS) +) + +im = ax_c.imshow(mat.values, aspect="auto", vmin=0, vmax=1, cmap="viridis") +ax_c.set_title("Per-behavior decoding") +ax_c.set_ylabel("CalMS21 behavior") +ax_c.set_xlabel("Frozen LISBET encoder") +ax_c.set_xticks(np.arange(len(MODEL_LABELS))) +ax_c.set_xticklabels(MODEL_LABELS, rotation=40, ha="right") +ax_c.set_yticks(np.arange(len(CLASS_ORDER))) +ax_c.set_yticklabels(CLASS_ORDER) + +for i in range(mat.shape[0]): + for j in range(mat.shape[1]): + val = mat.values[i, j] + color = "white" if val < 0.45 else "black" + ax_c.text(j, i, f"{val:.2f}", ha="center", va="center", fontsize=7, color=color) + +cbar = fig.colorbar(im, ax=ax_c, fraction=0.046, pad=0.025) +cbar.set_label("F1") +panel_label(ax_c, "c") +clean_axis(ax_c) + +# Panel d: task-removal effect +emat = ( + eff.pivot(index="class_name", columns="removed_task", values="delta_f1_mean") + .reindex(index=CLASS_ORDER, columns=TASK_ORDER) +) + +v = np.nanmax(np.abs(emat.values)) +v = max(v, 0.01) +im2 = ax_d.imshow(emat.values, aspect="auto", cmap="coolwarm", vmin=-v, vmax=v) +ax_d.set_title("Task-removal effect") +ax_d.set_ylabel("CalMS21 behavior") +ax_d.set_xlabel("Removed task") +ax_d.set_xticks(np.arange(len(TASK_ORDER))) +ax_d.set_xticklabels(TASK_ORDER) +ax_d.set_yticks(np.arange(len(CLASS_ORDER))) +ax_d.set_yticklabels(CLASS_ORDER) + +for i in range(emat.shape[0]): + for j in range(emat.shape[1]): + val = emat.values[i, j] + ax_d.text(j, i, f"{val:+.3f}", ha="center", va="center", fontsize=7, color="black") + +cbar2 = fig.colorbar(im2, ax=ax_d, fraction=0.046, pad=0.025) +cbar2.set_label(r"$F1_{all} - F1_{without}$") +panel_label(ax_d, "d") +clean_axis(ax_d) + +# Panel e: overall decoding across seeds +x = np.arange(len(overall)) +offset = 0.09 + +ax_e.errorbar( + x - offset, + overall["macro_f1_mean"], + yerr=overall["macro_f1_sd"], + fmt="o", + color="#222222", + ecolor="#222222", + capsize=3, + markersize=4, + label="Macro-F1", +) + +ax_e.errorbar( + x + offset, + overall["balanced_accuracy_mean"], + yerr=overall["balanced_accuracy_sd"], + fmt="o", + color="#1b8a3a", + ecolor="#1b8a3a", + capsize=3, + markersize=4, + label="Balanced accuracy", +) + +ax_e.set_title("Overall behavioral decoding") +ax_e.set_ylabel("Test performance") +ax_e.set_xlabel("Frozen LISBET encoder") +ax_e.set_xticks(x) +ax_e.set_xticklabels(MODEL_LABELS, rotation=40, ha="right") + +vals = np.r_[overall["macro_f1_mean"], overall["balanced_accuracy_mean"]] +errs = np.r_[overall["macro_f1_sd"].fillna(0), overall["balanced_accuracy_sd"].fillna(0)] +ax_e.set_ylim(np.nanmin(vals - errs) - 0.02, np.nanmax(vals + errs) + 0.02) + +ax_e.legend(frameon=False, loc="best") +panel_label(ax_e, "e") +clean_axis(ax_e) + +fig.savefig(FIG / "Fig_calms21_knn_task_ablation_posthoc_allseeds.png", dpi=600, bbox_inches="tight") +fig.savefig(FIG / "Fig_calms21_knn_task_ablation_posthoc_allseeds.pdf", dpi=600, bbox_inches="tight") +fig.savefig(FIG / "Fig_calms21_knn_task_ablation_posthoc_allseeds.svg", dpi=600, bbox_inches="tight") + +print("Saved:") +print(FIG / "Fig_calms21_knn_task_ablation_posthoc_allseeds.png") +print(FIG / "Fig_calms21_knn_task_ablation_posthoc_allseeds.pdf") +print(FIG / "Fig_calms21_knn_task_ablation_posthoc_allseeds.svg") diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed0.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed0.py new file mode 100644 index 0000000..6b72d4f --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed0.py @@ -0,0 +1,994 @@ +#!/usr/bin/env python3 +"""Reviewer 2.2: LISBET leave-one-task-out analysis on CalMS21. + +Stages +------ +validate Validate the five matched checkpoints and training histories. +prepare Convert CalMS21 task 1 JSON files to multi-animal DLC CSV files. +embed Compute frozen embeddings for train/test videos and all five models. +evaluate Tune a fixed kNN probe on training videos and evaluate official test videos. +plot Generate training, downstream, and combined figures. +all Run all stages in sequence. + +The downstream comparison uses the official CalMS21 task 1 train/test separation. +The k value is selected using only the all-task model and training videos, then held +fixed for every leave-one-task-out model. Confidence intervals resample test videos, +not frames. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import yaml +from matplotlib.colors import TwoSlopeNorm +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import GroupKFold +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import StandardScaler + + +TASKS = ("cons", "order", "shift", "warp") +RUNS = { + "all_tasks": { + "folder": "all_tasks_seed0_75ep", + "model_id": "all_tasks_seed0_75ep", + "label": "All tasks", + "removed": None, + }, + "without_cons": { + "folder": "triple_order_shift_warp_seed0_75ep", + "model_id": "triple_order_shift_warp_seed0_75ep", + "label": "Without cons", + "removed": "cons", + }, + "without_order": { + "folder": "triple_cons_shift_warp_seed0_75ep", + "model_id": "triple_cons_shift_warp_seed0_75ep", + "label": "Without order", + "removed": "order", + }, + "without_shift": { + "folder": "triple_cons_order_warp_seed0_75ep", + "model_id": "triple_cons_order_warp_seed0_75ep", + "label": "Without shift", + "removed": "shift", + }, + "without_warp": { + "folder": "triple_cons_order_shift_seed0_75ep", + "model_id": "triple_cons_order_shift_seed0_75ep", + "label": "Without warp", + "removed": "warp", + }, +} + +RUN_COLORS = { + "all_tasks": "#202020", + "without_cons": "#4477AA", + "without_order": "#EE6677", + "without_shift": "#228833", + "without_warp": "#CCBB44", +} + + +@dataclass(frozen=True) +class Paths: + lisbet_root: Path + ablation_root: Path + calms_root: Path + work: Path + prepared: Path + embedders: Path + embeddings: Path + results: Path + figures: Path + logs: Path + + @classmethod + def from_root(cls, lisbet_root: Path) -> "Paths": + lisbet_root = lisbet_root.expanduser().resolve() + ablation_root = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "results" / "auxiliary_task_assessment" / "task_full_combinations_w200_75ep_train1600" + work = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "reviewer_2_2_calms21_posthoc_seed0" + return cls( + lisbet_root=lisbet_root, + ablation_root=ablation_root, + calms_root=lisbet_root / "lisbet_datasets" / "datasets" / "CalMS21", + work=work, + prepared=work / "prepared_calms21_task1_dlc", + embedders=work / "exported_embedders", + embeddings=work / "embeddings", + results=work / "results", + figures=work / "figures", + logs=work / "logs", + ) + + def create_outputs(self) -> None: + for path in ( + self.work, + self.prepared, + self.embedders, + self.embeddings, + self.results, + self.figures, + self.logs, + ): + path.mkdir(parents=True, exist_ok=True) + + +def model_paths(paths: Paths, run_key: str) -> tuple[Path, Path, Path]: + run = RUNS[run_key] + base = paths.ablation_root / run["folder"] / "models" / run["model_id"] + return ( + base / "model_config.yml", + base / "weights" / "weights_last.pt", + base / "training_history" / "version_0" / "metrics.csv", + ) + + +def validate_runs(paths: Paths) -> pd.DataFrame: + """Check that encoder settings match and only the expected head is removed.""" + rows = [] + backbone_reference = None + input_reference = None + for run_key, run in RUNS.items(): + config_path, weights_path, metrics_path = model_paths(paths, run_key) + for required in (config_path, weights_path, metrics_path): + if not required.exists(): + raise FileNotFoundError(f"Missing required file: {required}") + + config = yaml.safe_load(config_path.read_text()) + metrics = pd.read_csv(metrics_path) + backbone = config.get("backbone") + input_features = config.get("input_features") + if backbone_reference is None: + backbone_reference = backbone + input_reference = input_features + if backbone != backbone_reference: + raise ValueError(f"Backbone mismatch in {run_key}") + if input_features != input_reference: + raise ValueError(f"Input-feature mismatch in {run_key}") + + observed_heads = set(config.get("out_heads", {})) + expected_heads = set(TASKS) + if run["removed"] is not None: + expected_heads.remove(run["removed"]) + if observed_heads != expected_heads: + raise ValueError( + f"Unexpected heads for {run_key}: observed={sorted(observed_heads)}, " + f"expected={sorted(expected_heads)}" + ) + if config.get("window_size") != 200 or config.get("backbone", {}).get("max_length") != 200: + raise ValueError(f"{run_key} is not a matched window-200 model") + if "epoch" not in metrics or int(metrics["epoch"].max()) != 599: + print(f"[warn] {run_key} does not contain all 75 epochs by old validation; continuing for 75ep analysis") + + row = { + "run": run_key, + "label": run["label"], + "removed_task": run["removed"] or "none", + "epochs": int(metrics["epoch"].max()) + 1, + "checkpoint_bytes": weights_path.stat().st_size, + "heads": ",".join(sorted(observed_heads)), + } + for task in TASKS: + for kind in ("score", "loss"): + col = f"{task}_train_{kind}" + row[f"final_{task}_{kind}"] = float(metrics[col].dropna().iloc[-1]) if col in metrics else np.nan + rows.append(row) + + out = pd.DataFrame(rows) + paths.create_outputs() + out.to_csv(paths.results / "validated_run_inventory.csv", index=False) + print(out.to_string(index=False)) + return out + + +def safe_name(value: str) -> str: + value = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value)).strip("_") + return value or "record" + + +def task1_json(paths: Paths, split: str) -> Path: + expected = ( + paths.calms_root + / "task1_classic_classification" + / f"calms21_task1_{split}.json" + ) + if expected.exists(): + return expected + cache_root = paths.lisbet_root / "lisbet_datasets" / "datasets" / ".cache" / "lisbet" + cached = sorted( + cache_root.glob( + "*-task1_classic_classification.zip.unzip/" + f"task1_classic_classification/calms21_task1_{split}.json" + ) + ) + if len(cached) == 1: + print(f"Using cached CalMS21 task 1 {split} data: {cached[0]}") + return cached[0] + if len(cached) > 1: + raise RuntimeError( + f"Found multiple cached CalMS21 task 1 {split} files: {cached}. " + "Remove stale cache copies or materialize the intended dataset path." + ) + raise FileNotFoundError( + f"CalMS21 task 1 {split} JSON was not found at:\n{expected}\n" + "The CalMS21 directory may be an unmaterialized link. Confirm it with " + "`find -L lisbet_datasets/datasets/CalMS21 -maxdepth 3 -type f`." + ) + + +def extract_pose_arrays(record: dict) -> tuple[np.ndarray, np.ndarray]: + """Normalize CalMS21 task 1 arrays to (frames, individuals, keypoints, axes).""" + # Task 1 JSON stores keypoints as + # (frames, individuals, coordinates, keypoints). + positions = np.asarray(record["keypoints"], dtype=np.float32).transpose((0, 1, 3, 2)) + scores = np.asarray(record["scores"], dtype=np.float32) + if positions.ndim != 4 or positions.shape[1:] != (2, 7, 2): + raise ValueError(f"Unexpected CalMS21 position shape after conversion: {positions.shape}") + # CalMS21 task 1 stores confidence as (frames, individuals, keypoints). + # Some converted variants use (frames, keypoints, individuals), so accept + # and normalize either representation for the DLC writer. + if scores.shape == (positions.shape[0], 7, 2): + scores = scores.transpose((0, 2, 1)) + if scores.shape != positions.shape[:3]: + raise ValueError(f"Position/score shape mismatch: {positions.shape}, {scores.shape}") + return positions, scores + + +def write_dlc_csv(path: Path, positions: np.ndarray, scores: np.ndarray) -> None: + individuals = ("resident", "intruder") + keypoints = ("nose", "left_ear", "right_ear", "neck", "left_hip", "right_hip", "tail") + columns = [] + values = [] + for ind_i, individual in enumerate(individuals): + for kp_i, keypoint in enumerate(keypoints): + for coord_i, coord in enumerate(("x", "y")): + columns.append(("calms21", individual, keypoint, coord)) + values.append(positions[:, ind_i, kp_i, coord_i]) + columns.append(("calms21", individual, keypoint, "likelihood")) + values.append(scores[:, ind_i, kp_i]) + frame = pd.DataFrame( + np.column_stack(values), + columns=pd.MultiIndex.from_tuples( + columns, names=("scorer", "individuals", "bodyparts", "coords") + ), + ) + frame.to_csv(path, index=True) + + +def prepare_calms21(paths: Paths, force: bool = False) -> pd.DataFrame: + """Create DLC inputs and exact frame-label tables for official task 1 splits.""" + paths.create_outputs() + manifest_path = paths.prepared / "record_manifest.csv" + if manifest_path.exists() and not force: + manifest = pd.read_csv(manifest_path) + print(f"Using existing prepared dataset: {manifest_path}") + return manifest + + rows = [] + global_vocab = None + seen_ids = set() + for split in ("train", "test"): + pose_dir = paths.prepared / split / "poses" + label_dir = paths.prepared / split / "labels" + pose_dir.mkdir(parents=True, exist_ok=True) + label_dir.mkdir(parents=True, exist_ok=True) + source_path = task1_json(paths, split) + print(f"Loading CalMS21 {split} JSON (this may take several minutes): {source_path}") + # json.load avoids holding an additional full-size text copy of these + # large (approximately 0.6-1.2 GB) source files in memory. + with source_path.open("r", encoding="utf-8") as source: + raw = json.load(source) + + for condition, condition_records in raw.items(): + for original_id, record in condition_records.items(): + stem = safe_name(f"{condition}__{original_id}") + print(f"Preparing {split} record: {stem}") + if stem in seen_ids: + raise ValueError(f"Duplicate generated record ID: {stem}") + seen_ids.add(stem) + + positions, scores = extract_pose_arrays(record) + annotations = np.asarray(record["annotations"], dtype=int) + if len(annotations) != len(positions): + raise ValueError(f"Annotation/pose length mismatch for {stem}") + + vocab_map = record.get("metadata", {}).get("vocab") + if not vocab_map: + raise ValueError(f"Missing behavior vocabulary for {stem}") + vocab = [name for name, idx in sorted(vocab_map.items(), key=lambda item: item[1])] + if global_vocab is None: + global_vocab = vocab + if vocab != global_vocab: + raise ValueError(f"Behavior vocabulary differs in {stem}: {vocab} != {global_vocab}") + if annotations.min() < 0 or annotations.max() >= len(vocab): + raise ValueError(f"Annotation IDs outside vocabulary for {stem}") + + # LISBET's DLC loader scans sequence subdirectories and accepts + # filenames matching `tracking*.csv`. + record_pose_dir = pose_dir / stem + record_pose_dir.mkdir(parents=True, exist_ok=True) + pose_path = record_pose_dir / "tracking.csv" + labels_path = label_dir / f"{stem}.csv" + if force or not pose_path.exists(): + write_dlc_csv(pose_path, positions, scores) + label_df = pd.DataFrame( + { + "record_id": stem, + "frame_idx": np.arange(len(annotations), dtype=int), + "label_id": annotations, + "label_name": [vocab[i] for i in annotations], + } + ) + label_df.to_csv(labels_path, index=False) + rows.append( + { + "split": split, + "condition": condition, + "original_id": original_id, + "record_id": stem, + "n_frames": len(annotations), + "pose_csv": str(pose_path), + "labels_csv": str(labels_path), + } + ) + + manifest = pd.DataFrame(rows).sort_values(["split", "record_id"]) + manifest.to_csv(manifest_path, index=False) + (paths.prepared / "behavior_vocabulary.json").write_text(json.dumps(global_vocab, indent=2)) + print(f"Prepared {len(manifest)} records in {paths.prepared}") + print(manifest.groupby("split")["n_frames"].agg(["count", "sum"])) + return manifest + + +def run_command(command: list[str], stdout_path: Path, stderr_path: Path) -> None: + print("Running:", " ".join(command)) + completed = subprocess.run(command, text=True, capture_output=True) + stdout_path.write_text(completed.stdout) + stderr_path.write_text(completed.stderr) + if completed.returncode != 0: + raise RuntimeError( + f"Command failed with exit code {completed.returncode}. See {stderr_path}\n" + f"Last stderr lines:\n{completed.stderr[-3000:]}" + ) + + +def find_exported_embedder(output_path: Path) -> tuple[Path, Path] | None: + configs = sorted(output_path.rglob("model_config.yml")) + if not configs: + configs = sorted(output_path.rglob("*.yml")) + sorted(output_path.rglob("*.yaml")) + weights = sorted(output_path.rglob("*.pt")) + sorted(output_path.rglob("*.pth")) + if len(configs) == 1 and len(weights) == 1: + return configs[0], weights[0] + if len(configs) == 0 and len(weights) == 0: + return None + raise ValueError( + f"Expected one exported config and one weight file under {output_path}; " + f"found configs={configs}, weights={weights}" + ) + + +def export_embedder(paths: Paths, run_key: str, force: bool = False) -> tuple[Path, Path]: + """Export the shared trained backbone with the embedding inference head.""" + output_path = paths.embedders / run_key + output_path.mkdir(parents=True, exist_ok=True) + existing = find_exported_embedder(output_path) + if existing is not None and not force: + print(f"[skip] {run_key}: using exported embedder {existing[0]}") + return existing + + source_config, source_weights, _ = model_paths(paths, run_key) + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "export_embedder", + str(source_config), + str(source_weights), + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"export_embedder_{run_key}_stdout.txt", + paths.logs / f"export_embedder_{run_key}_stderr.txt", + ) + exported = find_exported_embedder(output_path) + if exported is None: + raise FileNotFoundError(f"export_embedder produced no model files in {output_path}") + print(f"Exported {run_key} embedder: {exported[0]}, {exported[1]}") + return exported + + +def compute_embeddings(paths: Paths, force: bool = False) -> None: + """Run betman for each model and official data split.""" + validate_runs(paths) + manifest = prepare_calms21(paths, force=False) + paths.create_outputs() + for run_key in RUNS: + config_path, weights_path = export_embedder(paths, run_key, force=False) + for split in ("train", "test"): + data_path = paths.prepared / split / "poses" + output_path = paths.embeddings / run_key / split + output_path.mkdir(parents=True, exist_ok=True) + expected = list(output_path.rglob("features_lisbet_embedding.csv")) + expected_records = int((manifest["split"] == split).sum()) + if len(expected) == expected_records and not force: + print(f"[skip] {run_key}/{split}: found {len(expected)} embedding files") + continue + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "compute_embeddings", + str(data_path), + str(config_path), + str(weights_path), + "--data_format", + "maDLC", + "--window_size", + "200", + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"embedding_{run_key}_{split}_stdout.txt", + paths.logs / f"embedding_{run_key}_{split}_stderr.txt", + ) + + +def embedding_columns(frame: pd.DataFrame) -> list[str]: + cols = [c for c in frame.columns if str(c).isdigit()] + if cols: + return sorted(cols, key=lambda c: int(str(c))) + excluded = {"frame_idx", "time", "index"} + cols = [ + c + for c in frame.columns + if c not in excluded + and not str(c).startswith("Unnamed") + and pd.api.types.is_numeric_dtype(frame[c]) + ] + if not cols: + raise ValueError(f"No embedding dimensions found. Columns: {frame.columns.tolist()}") + return cols + + +def index_embedding_files(paths: Paths, run_key: str, split: str, record_ids: list[str]) -> dict[str, Path]: + files = sorted((paths.embeddings / run_key / split).rglob("features_lisbet_embedding.csv")) + mapping = {} + for record_id in record_ids: + matches = [f for f in files if record_id in f.parts or record_id in str(f)] + if len(matches) != 1: + raise ValueError( + f"Expected exactly one embedding file for {run_key}/{split}/{record_id}; " + f"found {len(matches)}. Available examples: {files[:5]}" + ) + mapping[record_id] = matches[0] + return mapping + + +def load_split_embeddings( + paths: Paths, run_key: str, split: str, manifest: pd.DataFrame +) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[str]]: + subset = manifest[manifest["split"] == split].sort_values("record_id") + record_ids = subset["record_id"].tolist() + file_map = index_embedding_files(paths, run_key, split, record_ids) + x_parts, y_parts, group_parts = [], [], [] + class_names = json.loads((paths.prepared / "behavior_vocabulary.json").read_text()) + + for row in subset.itertuples(index=False): + labels = pd.read_csv(row.labels_csv) + emb = pd.read_csv(file_map[row.record_id]) + cols = embedding_columns(emb) + if "frame_idx" in emb.columns: + merged = labels.merge(emb[["frame_idx"] + cols], on="frame_idx", how="inner", validate="one_to_one") + if len(merged) != len(labels): + raise ValueError( + f"Frame-index alignment lost rows for {run_key}/{row.record_id}: " + f"labels={len(labels)}, merged={len(merged)}" + ) + x = merged[cols].to_numpy(dtype=np.float32) + y = merged["label_id"].to_numpy(dtype=int) + else: + if len(emb) != len(labels): + raise ValueError( + f"Embedding/label length mismatch for {run_key}/{row.record_id}: " + f"embeddings={len(emb)}, labels={len(labels)}. No silent truncation is performed." + ) + x = emb[cols].to_numpy(dtype=np.float32) + y = labels["label_id"].to_numpy(dtype=int) + if not np.isfinite(x).all(): + raise ValueError(f"Non-finite embeddings in {run_key}/{row.record_id}") + x_parts.append(x) + y_parts.append(y) + group_parts.append(np.repeat(row.record_id, len(y))) + + return ( + np.concatenate(x_parts), + np.concatenate(y_parts), + np.concatenate(group_parts), + class_names, + ) + + +def balanced_sample(y: np.ndarray, max_per_class: int, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + selected = [] + for cls in np.unique(y): + idx = np.flatnonzero(y == cls) + if len(idx) > max_per_class: + idx = rng.choice(idx, size=max_per_class, replace=False) + selected.append(np.sort(idx)) + return np.sort(np.concatenate(selected)) + + +def metrics_from_cm(cm: np.ndarray) -> dict[str, np.ndarray | float]: + cm = np.asarray(cm, dtype=float) + tp = np.diag(cm) + support = cm.sum(axis=1) + predicted = cm.sum(axis=0) + recall = np.divide(tp, support, out=np.zeros_like(tp), where=support > 0) + precision = np.divide(tp, predicted, out=np.zeros_like(tp), where=predicted > 0) + f1 = np.divide(2 * precision * recall, precision + recall, out=np.zeros_like(tp), where=(precision + recall) > 0) + return { + "accuracy": float(tp.sum() / cm.sum()), + "balanced_accuracy": float(np.mean(recall)), + "macro_f1": float(np.mean(f1)), + "precision": precision, + "recall": recall, + "f1": f1, + "support": support, + } + + +def predict_in_chunks(clf: KNeighborsClassifier, x: np.ndarray, chunk_size: int) -> np.ndarray: + predictions = [] + for start in range(0, len(x), chunk_size): + predictions.append(clf.predict(x[start : start + chunk_size])) + return np.concatenate(predictions) + + +def tune_k_on_all_task_training( + x: np.ndarray, + y: np.ndarray, + groups: np.ndarray, + k_grid: list[int], + max_train_per_class: int, + max_val_per_class: int, + seed: int, +) -> tuple[int, pd.DataFrame]: + n_groups = len(np.unique(groups)) + if n_groups < 3: + raise ValueError("At least three training videos are required for grouped k selection") + splitter = GroupKFold(n_splits=min(5, n_groups)) + rows = [] + labels = np.arange(len(np.unique(y))) + for fold, (train_idx, val_idx) in enumerate(splitter.split(x, y, groups)): + train_keep = train_idx[balanced_sample(y[train_idx], max_train_per_class, seed + fold)] + val_keep = val_idx[balanced_sample(y[val_idx], max_val_per_class, seed + 100 + fold)] + scaler = StandardScaler() + x_train = scaler.fit_transform(x[train_keep]).astype(np.float32) + x_val = scaler.transform(x[val_keep]).astype(np.float32) + for k in k_grid: + clf = KNeighborsClassifier(n_neighbors=k, weights="distance", metric="euclidean", n_jobs=-1) + clf.fit(x_train, y[train_keep]) + pred = predict_in_chunks(clf, x_val, chunk_size=5000) + cm = confusion_matrix(y[val_keep], pred, labels=labels) + metric = metrics_from_cm(cm) + rows.append({"fold": fold, "k": k, "macro_f1": metric["macro_f1"]}) + results = pd.DataFrame(rows) + summary = results.groupby("k", as_index=False)["macro_f1"].agg(["mean", "std"]).reset_index() + best_k = int(summary.sort_values(["mean", "k"], ascending=[False, True]).iloc[0]["k"]) + print("Grouped training-only k selection:") + print(summary.to_string(index=False)) + print("Selected k:", best_k) + return best_k, results + + +def evaluate(paths: Paths, args: argparse.Namespace) -> None: + """Evaluate frozen representations on the untouched official task 1 test videos.""" + paths.create_outputs() + manifest = pd.read_csv(paths.prepared / "record_manifest.csv") + labels = None + + x_all, y_train, groups_train, class_names = load_split_embeddings( + paths, "all_tasks", "train", manifest + ) + labels = np.arange(len(class_names)) + best_k, tuning = tune_k_on_all_task_training( + x_all, + y_train, + groups_train, + args.k_grid, + args.max_train_per_class, + args.max_validation_per_class, + args.seed, + ) + tuning.to_csv(paths.results / "knn_k_selection_grouped_training.csv", index=False) + (paths.results / "selected_knn_k.json").write_text(json.dumps({"k": best_k}, indent=2)) + del x_all, y_train, groups_train + + model_cms: dict[str, dict[str, np.ndarray]] = {} + summary_rows = [] + per_class_rows = [] + cm_rows = [] + + for model_i, run_key in enumerate(RUNS): + print(f"Evaluating {RUNS[run_key]['label']}...") + x_train, y_train, _, names_train = load_split_embeddings(paths, run_key, "train", manifest) + if names_train != class_names: + raise ValueError("Class-name mismatch across models") + train_keep = balanced_sample(y_train, args.max_train_per_class, args.seed) + scaler = StandardScaler() + x_train_scaled = scaler.fit_transform(x_train[train_keep]).astype(np.float32) + clf = KNeighborsClassifier( + n_neighbors=best_k, weights="distance", metric="euclidean", n_jobs=-1 + ) + clf.fit(x_train_scaled, y_train[train_keep]) + del x_train, x_train_scaled, y_train + + x_test, y_test, groups_test, names_test = load_split_embeddings(paths, run_key, "test", manifest) + if names_test != class_names: + raise ValueError("Class-name mismatch across splits") + model_cms[run_key] = {} + for record_id in sorted(np.unique(groups_test)): + idx = np.flatnonzero(groups_test == record_id) + x_record = scaler.transform(x_test[idx]).astype(np.float32) + pred = predict_in_chunks(clf, x_record, args.prediction_chunk_size) + cm = confusion_matrix(y_test[idx], pred, labels=labels) + model_cms[run_key][record_id] = cm + for true_i in labels: + for pred_i in labels: + cm_rows.append( + { + "model": run_key, + "record_id": record_id, + "true_class": class_names[true_i], + "predicted_class": class_names[pred_i], + "count": int(cm[true_i, pred_i]), + } + ) + total_cm = sum(model_cms[run_key].values()) + metric = metrics_from_cm(total_cm) + summary_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "removed_task": RUNS[run_key]["removed"] or "none", + "k": best_k, + "n_train_probe": len(train_keep), + "n_test_frames": int(total_cm.sum()), + "accuracy": metric["accuracy"], + "balanced_accuracy": metric["balanced_accuracy"], + "macro_f1": metric["macro_f1"], + } + ) + for class_i, class_name in enumerate(class_names): + per_class_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "class_id": class_i, + "class_name": class_name, + "precision": metric["precision"][class_i], + "recall": metric["recall"][class_i], + "f1": metric["f1"][class_i], + "support": int(metric["support"][class_i]), + } + ) + del x_test, y_test, groups_test + + pd.DataFrame(cm_rows).to_csv(paths.results / "test_video_confusion_matrices.csv", index=False) + per_class = pd.DataFrame(per_class_rows) + per_class.to_csv(paths.results / "calms21_per_class_metrics.csv", index=False) + + rng = np.random.default_rng(args.seed) + test_videos = sorted(next(iter(model_cms.values())).keys()) + bootstrap_rows = [] + delta_rows = [] + for bootstrap_i in range(args.bootstrap_replicates): + draw = rng.choice(test_videos, size=len(test_videos), replace=True) + boot_metrics = {} + for run_key in RUNS: + cm = sum(model_cms[run_key][video] for video in draw) + boot_metrics[run_key] = metrics_from_cm(cm) + bootstrap_rows.append( + { + "bootstrap": bootstrap_i, + "model": run_key, + "macro_f1": boot_metrics[run_key]["macro_f1"], + "balanced_accuracy": boot_metrics[run_key]["balanced_accuracy"], + } + ) + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + delta_rows.append( + { + "bootstrap": bootstrap_i, + "removed_task": run["removed"], + "delta_macro_f1_all_minus_without": ( + boot_metrics["all_tasks"]["macro_f1"] - boot_metrics[run_key]["macro_f1"] + ), + "delta_per_class_f1_all_minus_without": ( + boot_metrics["all_tasks"]["f1"] - boot_metrics[run_key]["f1"] + ).tolist(), + } + ) + + bootstrap = pd.DataFrame(bootstrap_rows) + bootstrap.to_csv(paths.results / "test_video_bootstrap_metrics.csv", index=False) + delta_long = [] + for row in delta_rows: + for class_i, class_name in enumerate(class_names): + delta_long.append( + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "class_name": class_name, + "delta_f1_all_minus_without": row["delta_per_class_f1_all_minus_without"][class_i], + } + ) + pd.DataFrame(delta_long).to_csv(paths.results / "paired_bootstrap_per_class_task_effects.csv", index=False) + delta_overall = pd.DataFrame( + [ + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "delta_macro_f1_all_minus_without": row["delta_macro_f1_all_minus_without"], + } + for row in delta_rows + ] + ) + delta_overall.to_csv(paths.results / "paired_bootstrap_macro_f1_task_effects.csv", index=False) + + summary = pd.DataFrame(summary_rows) + ci = ( + bootstrap.groupby("model")["macro_f1"] + .quantile([0.025, 0.975]) + .unstack() + .rename(columns={0.025: "macro_f1_ci_low", 0.975: "macro_f1_ci_high"}) + .reset_index() + ) + summary = summary.merge(ci, on="model", how="left") + summary.to_csv(paths.results / "calms21_model_summary.csv", index=False) + + contribution_rows = [] + all_pc = per_class[per_class["model"] == "all_tasks"].set_index("class_name") + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + ablated_pc = per_class[per_class["model"] == run_key].set_index("class_name") + for class_name in class_names: + contribution_rows.append( + { + "removed_task": run["removed"], + "class_name": class_name, + "delta_f1_all_minus_without": ( + all_pc.loc[class_name, "f1"] - ablated_pc.loc[class_name, "f1"] + ), + } + ) + pd.DataFrame(contribution_rows).to_csv(paths.results / "per_class_task_contribution.csv", index=False) + print(summary.to_string(index=False)) + + +def set_plot_style() -> None: + mpl.rcParams.update( + { + "font.family": "sans-serif", + "font.sans-serif": ["DejaVu Sans"], + "font.size": 8, + "axes.titlesize": 9, + "axes.labelsize": 8, + "xtick.labelsize": 7, + "ytick.labelsize": 7, + "legend.fontsize": 7, + "axes.linewidth": 0.7, + "pdf.fonttype": 42, + "ps.fonttype": 42, + "svg.fonttype": "none", + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.facecolor": "white", + "axes.spines.top": False, + "axes.spines.right": False, + } + ) + + +def panel_label(ax: plt.Axes, label: str, x: float = -0.14, y: float = 1.08) -> None: + ax.text(x, y, label, transform=ax.transAxes, ha="left", va="top", fontsize=12, fontweight="normal") + + +def save_figure(fig: plt.Figure, paths: Paths, stem: str) -> None: + for suffix in ("pdf", "png", "svg"): + kwargs = {"dpi": 600} if suffix == "png" else {} + fig.savefig(paths.figures / f"{stem}.{suffix}", bbox_inches="tight", **kwargs) + print("Saved figure:", paths.figures / stem) + + +def plot_training(paths: Paths) -> plt.Figure: + """Plot raw and lightly smoothed trajectories; no false replicate bands.""" + set_plot_style() + histories = {} + for run_key in RUNS: + _, _, metrics_path = model_paths(paths, run_key) + histories[run_key] = pd.read_csv(metrics_path) + + fig, axes = plt.subplots(4, 2, figsize=(8.2, 8.8), sharex=True) + for task_i, task in enumerate(TASKS): + for col_i, kind in enumerate(("score", "loss")): + ax = axes[task_i, col_i] + metric_col = f"{task}_train_{kind}" + for run_key, history in histories.items(): + if metric_col not in history: + continue + x = history["epoch"].to_numpy() + raw = history[metric_col].to_numpy() + smooth = pd.Series(raw).rolling(15, center=True, min_periods=1).mean().to_numpy() + ax.plot(x, raw, color=RUN_COLORS[run_key], alpha=0.16, linewidth=0.45) + ax.plot(x, smooth, color=RUN_COLORS[run_key], linewidth=1.25, label=RUNS[run_key]["label"]) + ax.set_title(task) + if col_i == 0: + ax.set_ylabel("Training score") + ax.set_ylim(0.4, 1.02) + else: + ax.set_ylabel("Training loss") + ax.set_ylim(bottom=0) + if task_i == len(TASKS) - 1: + ax.set_xlabel("Epoch") + panel_label(axes[0, 0], "a") + panel_label(axes[0, 1], "b") + handles = [ + mpl.lines.Line2D([0], [0], color=RUN_COLORS[k], lw=1.5, label=RUNS[k]["label"]) + for k in RUNS + ] + fig.legend(handles=handles, loc="lower center", ncol=5, frameon=False, bbox_to_anchor=(0.5, 0.005)) + fig.subplots_adjust(left=0.10, right=0.98, top=0.97, bottom=0.08, hspace=0.42, wspace=0.28) + save_figure(fig, paths, "Fig_training_curves_75ep_leave_one_task_out") + return fig + + +def annotate_heatmap(ax: plt.Axes, values: np.ndarray, fmt: str, threshold: float | None = None) -> None: + if threshold is None: + threshold = float(np.nanmedian(values)) + for i in range(values.shape[0]): + for j in range(values.shape[1]): + value = values[i, j] + ax.text(j, i, format(value, fmt), ha="center", va="center", fontsize=7, color="white" if value < threshold else "black") + + +def plot_downstream(paths: Paths) -> plt.Figure: + set_plot_style() + summary = pd.read_csv(paths.results / "calms21_model_summary.csv") + per_class = pd.read_csv(paths.results / "calms21_per_class_metrics.csv") + contribution = pd.read_csv(paths.results / "per_class_task_contribution.csv") + model_order = list(RUNS) + model_labels = [RUNS[k]["label"] for k in model_order] + class_order = per_class.sort_values("class_id")["class_name"].drop_duplicates().tolist() + + absolute = ( + per_class.pivot(index="class_name", columns="model", values="f1") + .loc[class_order, model_order] + ) + effect = ( + contribution.pivot(index="class_name", columns="removed_task", values="delta_f1_all_minus_without") + .loc[class_order, list(TASKS)] + ) + + fig = plt.figure(figsize=(11.2, 3.65)) + gs = fig.add_gridspec(1, 3, width_ratios=(1.35, 1.15, 1.0), wspace=0.48) + ax0 = fig.add_subplot(gs[0, 0]) + im0 = ax0.imshow(absolute.values, cmap="viridis", vmin=0, vmax=1, aspect="auto") + ax0.set_xticks(range(len(model_order)), model_labels, rotation=38, ha="right") + ax0.set_yticks(range(len(class_order)), class_order) + ax0.set_xlabel("Frozen LISBET encoder") + ax0.set_ylabel("CalMS21 behavior") + ax0.set_title("Per-behavior decoding") + annotate_heatmap(ax0, absolute.values, ".2f", threshold=0.55) + fig.colorbar(im0, ax=ax0, fraction=0.046, pad=0.03, label="Test F1") + panel_label(ax0, "c") + + ax1 = fig.add_subplot(gs[0, 1]) + vmax = max(0.01, float(np.nanmax(np.abs(effect.values)))) + norm = TwoSlopeNorm(vmin=-vmax, vcenter=0, vmax=vmax) + im1 = ax1.imshow(effect.values, cmap="coolwarm", norm=norm, aspect="auto") + ax1.set_xticks(range(len(TASKS)), TASKS) + ax1.set_yticks(range(len(class_order)), class_order) + ax1.set_xlabel("Removed task") + ax1.set_ylabel("CalMS21 behavior") + ax1.set_title("Task-removal effect") + for i in range(effect.shape[0]): + for j in range(effect.shape[1]): + ax1.text(j, i, f"{effect.values[i, j]:+.3f}", ha="center", va="center", fontsize=7) + fig.colorbar(im1, ax=ax1, fraction=0.046, pad=0.03, label=r"$F1_{all}-F1_{without}$") + panel_label(ax1, "d") + + ax2 = fig.add_subplot(gs[0, 2]) + summary = summary.set_index("model").loc[model_order].reset_index() + x = np.arange(len(summary)) + y = summary["macro_f1"].to_numpy() + yerr = np.vstack( + [ + y - summary["macro_f1_ci_low"].to_numpy(), + summary["macro_f1_ci_high"].to_numpy() - y, + ] + ) + ax2.errorbar(x, y, yerr=yerr, fmt="o", color="#202020", ecolor="#555555", capsize=3, linewidth=1.1) + ax2.set_xticks(x, model_labels, rotation=38, ha="right") + ax2.set_ylabel("Test macro-F1") + ax2.set_xlabel("Frozen LISBET encoder") + ax2.set_ylim(max(0, float(np.nanmin(yerr[0] * -1 + y)) - 0.05), min(1, float(np.nanmax(yerr[1] + y)) + 0.05)) + ax2.set_title("Overall behavioral decoding") + panel_label(ax2, "e") + + fig.subplots_adjust(left=0.07, right=0.98, top=0.92, bottom=0.28) + save_figure(fig, paths, "Fig_calms21_knn_task_ablation_posthoc") + return fig + + +def plot_all(paths: Paths) -> None: + paths.create_outputs() + training = plot_training(paths) + downstream = plot_downstream(paths) + plt.show() + plt.close(training) + plt.close(downstream) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("stage", choices=("validate", "prepare", "embed", "evaluate", "plot", "all")) + parser.add_argument( + "--lisbet-root", + type=Path, + default=Path.home() / "Dokumente" / "Lisbet", + help="Directory containing the LISBET repository, datasets, and ablation folder", + ) + parser.add_argument("--force", action="store_true", help="Recompute prepared data or embeddings") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--k-grid", type=int, nargs="+", default=[1, 3, 5, 11, 21]) + parser.add_argument( + "--max-train-per-class", + type=int, + default=10000, + help="Balanced cap per class for the kNN reference set; applied identically to every model", + ) + parser.add_argument("--max-validation-per-class", type=int, default=5000) + parser.add_argument("--prediction-chunk-size", type=int, default=5000) + parser.add_argument("--bootstrap-replicates", type=int, default=5000) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + paths = Paths.from_root(args.lisbet_root) + paths.create_outputs() + if args.stage in ("validate", "all"): + validate_runs(paths) + if args.stage in ("prepare", "all"): + prepare_calms21(paths, force=args.force) + if args.stage in ("embed", "all"): + compute_embeddings(paths, force=args.force) + if args.stage in ("evaluate", "all"): + evaluate(paths, args) + if args.stage in ("plot", "all"): + plot_all(paths) + + +if __name__ == "__main__": + main() diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed1.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed1.py new file mode 100644 index 0000000..770663b --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed1.py @@ -0,0 +1,994 @@ +#!/usr/bin/env python3 +"""Reviewer 2.2: LISBET leave-one-task-out analysis on CalMS21. + +Stages +------ +validate Validate the five matched checkpoints and training histories. +prepare Convert CalMS21 task 1 JSON files to multi-animal DLC CSV files. +embed Compute frozen embeddings for train/test videos and all five models. +evaluate Tune a fixed kNN probe on training videos and evaluate official test videos. +plot Generate training, downstream, and combined figures. +all Run all stages in sequence. + +The downstream comparison uses the official CalMS21 task 1 train/test separation. +The k value is selected using only the all-task model and training videos, then held +fixed for every leave-one-task-out model. Confidence intervals resample test videos, +not frames. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import yaml +from matplotlib.colors import TwoSlopeNorm +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import GroupKFold +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import StandardScaler + + +TASKS = ("cons", "order", "shift", "warp") +RUNS = { + "all_tasks": { + "folder": "all_tasks_seed1_75ep", + "model_id": "all_tasks_seed1_75ep", + "label": "All tasks", + "removed": None, + }, + "without_cons": { + "folder": "triple_order_shift_warp_seed1_75ep", + "model_id": "triple_order_shift_warp_seed1_75ep", + "label": "Without cons", + "removed": "cons", + }, + "without_order": { + "folder": "triple_cons_shift_warp_seed1_75ep", + "model_id": "triple_cons_shift_warp_seed1_75ep", + "label": "Without order", + "removed": "order", + }, + "without_shift": { + "folder": "triple_cons_order_warp_seed1_75ep", + "model_id": "triple_cons_order_warp_seed1_75ep", + "label": "Without shift", + "removed": "shift", + }, + "without_warp": { + "folder": "triple_cons_order_shift_seed1_75ep", + "model_id": "triple_cons_order_shift_seed1_75ep", + "label": "Without warp", + "removed": "warp", + }, +} + +RUN_COLORS = { + "all_tasks": "#202020", + "without_cons": "#4477AA", + "without_order": "#EE6677", + "without_shift": "#228833", + "without_warp": "#CCBB44", +} + + +@dataclass(frozen=True) +class Paths: + lisbet_root: Path + ablation_root: Path + calms_root: Path + work: Path + prepared: Path + embedders: Path + embeddings: Path + results: Path + figures: Path + logs: Path + + @classmethod + def from_root(cls, lisbet_root: Path) -> "Paths": + lisbet_root = lisbet_root.expanduser().resolve() + ablation_root = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "results" / "auxiliary_task_assessment" / "task_full_combinations_w200_75ep_train1600" + work = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "reviewer_2_2_calms21_posthoc_seed1" + return cls( + lisbet_root=lisbet_root, + ablation_root=ablation_root, + calms_root=lisbet_root / "lisbet_datasets" / "datasets" / "CalMS21", + work=work, + prepared=work / "prepared_calms21_task1_dlc", + embedders=work / "exported_embedders", + embeddings=work / "embeddings", + results=work / "results", + figures=work / "figures", + logs=work / "logs", + ) + + def create_outputs(self) -> None: + for path in ( + self.work, + self.prepared, + self.embedders, + self.embeddings, + self.results, + self.figures, + self.logs, + ): + path.mkdir(parents=True, exist_ok=True) + + +def model_paths(paths: Paths, run_key: str) -> tuple[Path, Path, Path]: + run = RUNS[run_key] + base = paths.ablation_root / run["folder"] / "models" / run["model_id"] + return ( + base / "model_config.yml", + base / "weights" / "weights_last.pt", + base / "training_history" / "version_0" / "metrics.csv", + ) + + +def validate_runs(paths: Paths) -> pd.DataFrame: + """Check that encoder settings match and only the expected head is removed.""" + rows = [] + backbone_reference = None + input_reference = None + for run_key, run in RUNS.items(): + config_path, weights_path, metrics_path = model_paths(paths, run_key) + for required in (config_path, weights_path, metrics_path): + if not required.exists(): + raise FileNotFoundError(f"Missing required file: {required}") + + config = yaml.safe_load(config_path.read_text()) + metrics = pd.read_csv(metrics_path) + backbone = config.get("backbone") + input_features = config.get("input_features") + if backbone_reference is None: + backbone_reference = backbone + input_reference = input_features + if backbone != backbone_reference: + raise ValueError(f"Backbone mismatch in {run_key}") + if input_features != input_reference: + raise ValueError(f"Input-feature mismatch in {run_key}") + + observed_heads = set(config.get("out_heads", {})) + expected_heads = set(TASKS) + if run["removed"] is not None: + expected_heads.remove(run["removed"]) + if observed_heads != expected_heads: + raise ValueError( + f"Unexpected heads for {run_key}: observed={sorted(observed_heads)}, " + f"expected={sorted(expected_heads)}" + ) + if config.get("window_size") != 200 or config.get("backbone", {}).get("max_length") != 200: + raise ValueError(f"{run_key} is not a matched window-200 model") + if "epoch" not in metrics or int(metrics["epoch"].max()) != 599: + print(f"[warn] {run_key} does not contain all 75 epochs by old validation; continuing for 75ep analysis") + + row = { + "run": run_key, + "label": run["label"], + "removed_task": run["removed"] or "none", + "epochs": int(metrics["epoch"].max()) + 1, + "checkpoint_bytes": weights_path.stat().st_size, + "heads": ",".join(sorted(observed_heads)), + } + for task in TASKS: + for kind in ("score", "loss"): + col = f"{task}_train_{kind}" + row[f"final_{task}_{kind}"] = float(metrics[col].dropna().iloc[-1]) if col in metrics else np.nan + rows.append(row) + + out = pd.DataFrame(rows) + paths.create_outputs() + out.to_csv(paths.results / "validated_run_inventory.csv", index=False) + print(out.to_string(index=False)) + return out + + +def safe_name(value: str) -> str: + value = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value)).strip("_") + return value or "record" + + +def task1_json(paths: Paths, split: str) -> Path: + expected = ( + paths.calms_root + / "task1_classic_classification" + / f"calms21_task1_{split}.json" + ) + if expected.exists(): + return expected + cache_root = paths.lisbet_root / "lisbet_datasets" / "datasets" / ".cache" / "lisbet" + cached = sorted( + cache_root.glob( + "*-task1_classic_classification.zip.unzip/" + f"task1_classic_classification/calms21_task1_{split}.json" + ) + ) + if len(cached) == 1: + print(f"Using cached CalMS21 task 1 {split} data: {cached[0]}") + return cached[0] + if len(cached) > 1: + raise RuntimeError( + f"Found multiple cached CalMS21 task 1 {split} files: {cached}. " + "Remove stale cache copies or materialize the intended dataset path." + ) + raise FileNotFoundError( + f"CalMS21 task 1 {split} JSON was not found at:\n{expected}\n" + "The CalMS21 directory may be an unmaterialized link. Confirm it with " + "`find -L lisbet_datasets/datasets/CalMS21 -maxdepth 3 -type f`." + ) + + +def extract_pose_arrays(record: dict) -> tuple[np.ndarray, np.ndarray]: + """Normalize CalMS21 task 1 arrays to (frames, individuals, keypoints, axes).""" + # Task 1 JSON stores keypoints as + # (frames, individuals, coordinates, keypoints). + positions = np.asarray(record["keypoints"], dtype=np.float32).transpose((0, 1, 3, 2)) + scores = np.asarray(record["scores"], dtype=np.float32) + if positions.ndim != 4 or positions.shape[1:] != (2, 7, 2): + raise ValueError(f"Unexpected CalMS21 position shape after conversion: {positions.shape}") + # CalMS21 task 1 stores confidence as (frames, individuals, keypoints). + # Some converted variants use (frames, keypoints, individuals), so accept + # and normalize either representation for the DLC writer. + if scores.shape == (positions.shape[0], 7, 2): + scores = scores.transpose((0, 2, 1)) + if scores.shape != positions.shape[:3]: + raise ValueError(f"Position/score shape mismatch: {positions.shape}, {scores.shape}") + return positions, scores + + +def write_dlc_csv(path: Path, positions: np.ndarray, scores: np.ndarray) -> None: + individuals = ("resident", "intruder") + keypoints = ("nose", "left_ear", "right_ear", "neck", "left_hip", "right_hip", "tail") + columns = [] + values = [] + for ind_i, individual in enumerate(individuals): + for kp_i, keypoint in enumerate(keypoints): + for coord_i, coord in enumerate(("x", "y")): + columns.append(("calms21", individual, keypoint, coord)) + values.append(positions[:, ind_i, kp_i, coord_i]) + columns.append(("calms21", individual, keypoint, "likelihood")) + values.append(scores[:, ind_i, kp_i]) + frame = pd.DataFrame( + np.column_stack(values), + columns=pd.MultiIndex.from_tuples( + columns, names=("scorer", "individuals", "bodyparts", "coords") + ), + ) + frame.to_csv(path, index=True) + + +def prepare_calms21(paths: Paths, force: bool = False) -> pd.DataFrame: + """Create DLC inputs and exact frame-label tables for official task 1 splits.""" + paths.create_outputs() + manifest_path = paths.prepared / "record_manifest.csv" + if manifest_path.exists() and not force: + manifest = pd.read_csv(manifest_path) + print(f"Using existing prepared dataset: {manifest_path}") + return manifest + + rows = [] + global_vocab = None + seen_ids = set() + for split in ("train", "test"): + pose_dir = paths.prepared / split / "poses" + label_dir = paths.prepared / split / "labels" + pose_dir.mkdir(parents=True, exist_ok=True) + label_dir.mkdir(parents=True, exist_ok=True) + source_path = task1_json(paths, split) + print(f"Loading CalMS21 {split} JSON (this may take several minutes): {source_path}") + # json.load avoids holding an additional full-size text copy of these + # large (approximately 0.6-1.2 GB) source files in memory. + with source_path.open("r", encoding="utf-8") as source: + raw = json.load(source) + + for condition, condition_records in raw.items(): + for original_id, record in condition_records.items(): + stem = safe_name(f"{condition}__{original_id}") + print(f"Preparing {split} record: {stem}") + if stem in seen_ids: + raise ValueError(f"Duplicate generated record ID: {stem}") + seen_ids.add(stem) + + positions, scores = extract_pose_arrays(record) + annotations = np.asarray(record["annotations"], dtype=int) + if len(annotations) != len(positions): + raise ValueError(f"Annotation/pose length mismatch for {stem}") + + vocab_map = record.get("metadata", {}).get("vocab") + if not vocab_map: + raise ValueError(f"Missing behavior vocabulary for {stem}") + vocab = [name for name, idx in sorted(vocab_map.items(), key=lambda item: item[1])] + if global_vocab is None: + global_vocab = vocab + if vocab != global_vocab: + raise ValueError(f"Behavior vocabulary differs in {stem}: {vocab} != {global_vocab}") + if annotations.min() < 0 or annotations.max() >= len(vocab): + raise ValueError(f"Annotation IDs outside vocabulary for {stem}") + + # LISBET's DLC loader scans sequence subdirectories and accepts + # filenames matching `tracking*.csv`. + record_pose_dir = pose_dir / stem + record_pose_dir.mkdir(parents=True, exist_ok=True) + pose_path = record_pose_dir / "tracking.csv" + labels_path = label_dir / f"{stem}.csv" + if force or not pose_path.exists(): + write_dlc_csv(pose_path, positions, scores) + label_df = pd.DataFrame( + { + "record_id": stem, + "frame_idx": np.arange(len(annotations), dtype=int), + "label_id": annotations, + "label_name": [vocab[i] for i in annotations], + } + ) + label_df.to_csv(labels_path, index=False) + rows.append( + { + "split": split, + "condition": condition, + "original_id": original_id, + "record_id": stem, + "n_frames": len(annotations), + "pose_csv": str(pose_path), + "labels_csv": str(labels_path), + } + ) + + manifest = pd.DataFrame(rows).sort_values(["split", "record_id"]) + manifest.to_csv(manifest_path, index=False) + (paths.prepared / "behavior_vocabulary.json").write_text(json.dumps(global_vocab, indent=2)) + print(f"Prepared {len(manifest)} records in {paths.prepared}") + print(manifest.groupby("split")["n_frames"].agg(["count", "sum"])) + return manifest + + +def run_command(command: list[str], stdout_path: Path, stderr_path: Path) -> None: + print("Running:", " ".join(command)) + completed = subprocess.run(command, text=True, capture_output=True) + stdout_path.write_text(completed.stdout) + stderr_path.write_text(completed.stderr) + if completed.returncode != 0: + raise RuntimeError( + f"Command failed with exit code {completed.returncode}. See {stderr_path}\n" + f"Last stderr lines:\n{completed.stderr[-3000:]}" + ) + + +def find_exported_embedder(output_path: Path) -> tuple[Path, Path] | None: + configs = sorted(output_path.rglob("model_config.yml")) + if not configs: + configs = sorted(output_path.rglob("*.yml")) + sorted(output_path.rglob("*.yaml")) + weights = sorted(output_path.rglob("*.pt")) + sorted(output_path.rglob("*.pth")) + if len(configs) == 1 and len(weights) == 1: + return configs[0], weights[0] + if len(configs) == 0 and len(weights) == 0: + return None + raise ValueError( + f"Expected one exported config and one weight file under {output_path}; " + f"found configs={configs}, weights={weights}" + ) + + +def export_embedder(paths: Paths, run_key: str, force: bool = False) -> tuple[Path, Path]: + """Export the shared trained backbone with the embedding inference head.""" + output_path = paths.embedders / run_key + output_path.mkdir(parents=True, exist_ok=True) + existing = find_exported_embedder(output_path) + if existing is not None and not force: + print(f"[skip] {run_key}: using exported embedder {existing[0]}") + return existing + + source_config, source_weights, _ = model_paths(paths, run_key) + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "export_embedder", + str(source_config), + str(source_weights), + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"export_embedder_{run_key}_stdout.txt", + paths.logs / f"export_embedder_{run_key}_stderr.txt", + ) + exported = find_exported_embedder(output_path) + if exported is None: + raise FileNotFoundError(f"export_embedder produced no model files in {output_path}") + print(f"Exported {run_key} embedder: {exported[0]}, {exported[1]}") + return exported + + +def compute_embeddings(paths: Paths, force: bool = False) -> None: + """Run betman for each model and official data split.""" + validate_runs(paths) + manifest = prepare_calms21(paths, force=False) + paths.create_outputs() + for run_key in RUNS: + config_path, weights_path = export_embedder(paths, run_key, force=False) + for split in ("train", "test"): + data_path = paths.prepared / split / "poses" + output_path = paths.embeddings / run_key / split + output_path.mkdir(parents=True, exist_ok=True) + expected = list(output_path.rglob("features_lisbet_embedding.csv")) + expected_records = int((manifest["split"] == split).sum()) + if len(expected) == expected_records and not force: + print(f"[skip] {run_key}/{split}: found {len(expected)} embedding files") + continue + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "compute_embeddings", + str(data_path), + str(config_path), + str(weights_path), + "--data_format", + "maDLC", + "--window_size", + "200", + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"embedding_{run_key}_{split}_stdout.txt", + paths.logs / f"embedding_{run_key}_{split}_stderr.txt", + ) + + +def embedding_columns(frame: pd.DataFrame) -> list[str]: + cols = [c for c in frame.columns if str(c).isdigit()] + if cols: + return sorted(cols, key=lambda c: int(str(c))) + excluded = {"frame_idx", "time", "index"} + cols = [ + c + for c in frame.columns + if c not in excluded + and not str(c).startswith("Unnamed") + and pd.api.types.is_numeric_dtype(frame[c]) + ] + if not cols: + raise ValueError(f"No embedding dimensions found. Columns: {frame.columns.tolist()}") + return cols + + +def index_embedding_files(paths: Paths, run_key: str, split: str, record_ids: list[str]) -> dict[str, Path]: + files = sorted((paths.embeddings / run_key / split).rglob("features_lisbet_embedding.csv")) + mapping = {} + for record_id in record_ids: + matches = [f for f in files if record_id in f.parts or record_id in str(f)] + if len(matches) != 1: + raise ValueError( + f"Expected exactly one embedding file for {run_key}/{split}/{record_id}; " + f"found {len(matches)}. Available examples: {files[:5]}" + ) + mapping[record_id] = matches[0] + return mapping + + +def load_split_embeddings( + paths: Paths, run_key: str, split: str, manifest: pd.DataFrame +) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[str]]: + subset = manifest[manifest["split"] == split].sort_values("record_id") + record_ids = subset["record_id"].tolist() + file_map = index_embedding_files(paths, run_key, split, record_ids) + x_parts, y_parts, group_parts = [], [], [] + class_names = json.loads((paths.prepared / "behavior_vocabulary.json").read_text()) + + for row in subset.itertuples(index=False): + labels = pd.read_csv(row.labels_csv) + emb = pd.read_csv(file_map[row.record_id]) + cols = embedding_columns(emb) + if "frame_idx" in emb.columns: + merged = labels.merge(emb[["frame_idx"] + cols], on="frame_idx", how="inner", validate="one_to_one") + if len(merged) != len(labels): + raise ValueError( + f"Frame-index alignment lost rows for {run_key}/{row.record_id}: " + f"labels={len(labels)}, merged={len(merged)}" + ) + x = merged[cols].to_numpy(dtype=np.float32) + y = merged["label_id"].to_numpy(dtype=int) + else: + if len(emb) != len(labels): + raise ValueError( + f"Embedding/label length mismatch for {run_key}/{row.record_id}: " + f"embeddings={len(emb)}, labels={len(labels)}. No silent truncation is performed." + ) + x = emb[cols].to_numpy(dtype=np.float32) + y = labels["label_id"].to_numpy(dtype=int) + if not np.isfinite(x).all(): + raise ValueError(f"Non-finite embeddings in {run_key}/{row.record_id}") + x_parts.append(x) + y_parts.append(y) + group_parts.append(np.repeat(row.record_id, len(y))) + + return ( + np.concatenate(x_parts), + np.concatenate(y_parts), + np.concatenate(group_parts), + class_names, + ) + + +def balanced_sample(y: np.ndarray, max_per_class: int, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + selected = [] + for cls in np.unique(y): + idx = np.flatnonzero(y == cls) + if len(idx) > max_per_class: + idx = rng.choice(idx, size=max_per_class, replace=False) + selected.append(np.sort(idx)) + return np.sort(np.concatenate(selected)) + + +def metrics_from_cm(cm: np.ndarray) -> dict[str, np.ndarray | float]: + cm = np.asarray(cm, dtype=float) + tp = np.diag(cm) + support = cm.sum(axis=1) + predicted = cm.sum(axis=0) + recall = np.divide(tp, support, out=np.zeros_like(tp), where=support > 0) + precision = np.divide(tp, predicted, out=np.zeros_like(tp), where=predicted > 0) + f1 = np.divide(2 * precision * recall, precision + recall, out=np.zeros_like(tp), where=(precision + recall) > 0) + return { + "accuracy": float(tp.sum() / cm.sum()), + "balanced_accuracy": float(np.mean(recall)), + "macro_f1": float(np.mean(f1)), + "precision": precision, + "recall": recall, + "f1": f1, + "support": support, + } + + +def predict_in_chunks(clf: KNeighborsClassifier, x: np.ndarray, chunk_size: int) -> np.ndarray: + predictions = [] + for start in range(0, len(x), chunk_size): + predictions.append(clf.predict(x[start : start + chunk_size])) + return np.concatenate(predictions) + + +def tune_k_on_all_task_training( + x: np.ndarray, + y: np.ndarray, + groups: np.ndarray, + k_grid: list[int], + max_train_per_class: int, + max_val_per_class: int, + seed: int, +) -> tuple[int, pd.DataFrame]: + n_groups = len(np.unique(groups)) + if n_groups < 3: + raise ValueError("At least three training videos are required for grouped k selection") + splitter = GroupKFold(n_splits=min(5, n_groups)) + rows = [] + labels = np.arange(len(np.unique(y))) + for fold, (train_idx, val_idx) in enumerate(splitter.split(x, y, groups)): + train_keep = train_idx[balanced_sample(y[train_idx], max_train_per_class, seed + fold)] + val_keep = val_idx[balanced_sample(y[val_idx], max_val_per_class, seed + 100 + fold)] + scaler = StandardScaler() + x_train = scaler.fit_transform(x[train_keep]).astype(np.float32) + x_val = scaler.transform(x[val_keep]).astype(np.float32) + for k in k_grid: + clf = KNeighborsClassifier(n_neighbors=k, weights="distance", metric="euclidean", n_jobs=-1) + clf.fit(x_train, y[train_keep]) + pred = predict_in_chunks(clf, x_val, chunk_size=5000) + cm = confusion_matrix(y[val_keep], pred, labels=labels) + metric = metrics_from_cm(cm) + rows.append({"fold": fold, "k": k, "macro_f1": metric["macro_f1"]}) + results = pd.DataFrame(rows) + summary = results.groupby("k", as_index=False)["macro_f1"].agg(["mean", "std"]).reset_index() + best_k = int(summary.sort_values(["mean", "k"], ascending=[False, True]).iloc[0]["k"]) + print("Grouped training-only k selection:") + print(summary.to_string(index=False)) + print("Selected k:", best_k) + return best_k, results + + +def evaluate(paths: Paths, args: argparse.Namespace) -> None: + """Evaluate frozen representations on the untouched official task 1 test videos.""" + paths.create_outputs() + manifest = pd.read_csv(paths.prepared / "record_manifest.csv") + labels = None + + x_all, y_train, groups_train, class_names = load_split_embeddings( + paths, "all_tasks", "train", manifest + ) + labels = np.arange(len(class_names)) + best_k, tuning = tune_k_on_all_task_training( + x_all, + y_train, + groups_train, + args.k_grid, + args.max_train_per_class, + args.max_validation_per_class, + args.seed, + ) + tuning.to_csv(paths.results / "knn_k_selection_grouped_training.csv", index=False) + (paths.results / "selected_knn_k.json").write_text(json.dumps({"k": best_k}, indent=2)) + del x_all, y_train, groups_train + + model_cms: dict[str, dict[str, np.ndarray]] = {} + summary_rows = [] + per_class_rows = [] + cm_rows = [] + + for model_i, run_key in enumerate(RUNS): + print(f"Evaluating {RUNS[run_key]['label']}...") + x_train, y_train, _, names_train = load_split_embeddings(paths, run_key, "train", manifest) + if names_train != class_names: + raise ValueError("Class-name mismatch across models") + train_keep = balanced_sample(y_train, args.max_train_per_class, args.seed) + scaler = StandardScaler() + x_train_scaled = scaler.fit_transform(x_train[train_keep]).astype(np.float32) + clf = KNeighborsClassifier( + n_neighbors=best_k, weights="distance", metric="euclidean", n_jobs=-1 + ) + clf.fit(x_train_scaled, y_train[train_keep]) + del x_train, x_train_scaled, y_train + + x_test, y_test, groups_test, names_test = load_split_embeddings(paths, run_key, "test", manifest) + if names_test != class_names: + raise ValueError("Class-name mismatch across splits") + model_cms[run_key] = {} + for record_id in sorted(np.unique(groups_test)): + idx = np.flatnonzero(groups_test == record_id) + x_record = scaler.transform(x_test[idx]).astype(np.float32) + pred = predict_in_chunks(clf, x_record, args.prediction_chunk_size) + cm = confusion_matrix(y_test[idx], pred, labels=labels) + model_cms[run_key][record_id] = cm + for true_i in labels: + for pred_i in labels: + cm_rows.append( + { + "model": run_key, + "record_id": record_id, + "true_class": class_names[true_i], + "predicted_class": class_names[pred_i], + "count": int(cm[true_i, pred_i]), + } + ) + total_cm = sum(model_cms[run_key].values()) + metric = metrics_from_cm(total_cm) + summary_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "removed_task": RUNS[run_key]["removed"] or "none", + "k": best_k, + "n_train_probe": len(train_keep), + "n_test_frames": int(total_cm.sum()), + "accuracy": metric["accuracy"], + "balanced_accuracy": metric["balanced_accuracy"], + "macro_f1": metric["macro_f1"], + } + ) + for class_i, class_name in enumerate(class_names): + per_class_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "class_id": class_i, + "class_name": class_name, + "precision": metric["precision"][class_i], + "recall": metric["recall"][class_i], + "f1": metric["f1"][class_i], + "support": int(metric["support"][class_i]), + } + ) + del x_test, y_test, groups_test + + pd.DataFrame(cm_rows).to_csv(paths.results / "test_video_confusion_matrices.csv", index=False) + per_class = pd.DataFrame(per_class_rows) + per_class.to_csv(paths.results / "calms21_per_class_metrics.csv", index=False) + + rng = np.random.default_rng(args.seed) + test_videos = sorted(next(iter(model_cms.values())).keys()) + bootstrap_rows = [] + delta_rows = [] + for bootstrap_i in range(args.bootstrap_replicates): + draw = rng.choice(test_videos, size=len(test_videos), replace=True) + boot_metrics = {} + for run_key in RUNS: + cm = sum(model_cms[run_key][video] for video in draw) + boot_metrics[run_key] = metrics_from_cm(cm) + bootstrap_rows.append( + { + "bootstrap": bootstrap_i, + "model": run_key, + "macro_f1": boot_metrics[run_key]["macro_f1"], + "balanced_accuracy": boot_metrics[run_key]["balanced_accuracy"], + } + ) + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + delta_rows.append( + { + "bootstrap": bootstrap_i, + "removed_task": run["removed"], + "delta_macro_f1_all_minus_without": ( + boot_metrics["all_tasks"]["macro_f1"] - boot_metrics[run_key]["macro_f1"] + ), + "delta_per_class_f1_all_minus_without": ( + boot_metrics["all_tasks"]["f1"] - boot_metrics[run_key]["f1"] + ).tolist(), + } + ) + + bootstrap = pd.DataFrame(bootstrap_rows) + bootstrap.to_csv(paths.results / "test_video_bootstrap_metrics.csv", index=False) + delta_long = [] + for row in delta_rows: + for class_i, class_name in enumerate(class_names): + delta_long.append( + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "class_name": class_name, + "delta_f1_all_minus_without": row["delta_per_class_f1_all_minus_without"][class_i], + } + ) + pd.DataFrame(delta_long).to_csv(paths.results / "paired_bootstrap_per_class_task_effects.csv", index=False) + delta_overall = pd.DataFrame( + [ + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "delta_macro_f1_all_minus_without": row["delta_macro_f1_all_minus_without"], + } + for row in delta_rows + ] + ) + delta_overall.to_csv(paths.results / "paired_bootstrap_macro_f1_task_effects.csv", index=False) + + summary = pd.DataFrame(summary_rows) + ci = ( + bootstrap.groupby("model")["macro_f1"] + .quantile([0.025, 0.975]) + .unstack() + .rename(columns={0.025: "macro_f1_ci_low", 0.975: "macro_f1_ci_high"}) + .reset_index() + ) + summary = summary.merge(ci, on="model", how="left") + summary.to_csv(paths.results / "calms21_model_summary.csv", index=False) + + contribution_rows = [] + all_pc = per_class[per_class["model"] == "all_tasks"].set_index("class_name") + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + ablated_pc = per_class[per_class["model"] == run_key].set_index("class_name") + for class_name in class_names: + contribution_rows.append( + { + "removed_task": run["removed"], + "class_name": class_name, + "delta_f1_all_minus_without": ( + all_pc.loc[class_name, "f1"] - ablated_pc.loc[class_name, "f1"] + ), + } + ) + pd.DataFrame(contribution_rows).to_csv(paths.results / "per_class_task_contribution.csv", index=False) + print(summary.to_string(index=False)) + + +def set_plot_style() -> None: + mpl.rcParams.update( + { + "font.family": "sans-serif", + "font.sans-serif": ["DejaVu Sans"], + "font.size": 8, + "axes.titlesize": 9, + "axes.labelsize": 8, + "xtick.labelsize": 7, + "ytick.labelsize": 7, + "legend.fontsize": 7, + "axes.linewidth": 0.7, + "pdf.fonttype": 42, + "ps.fonttype": 42, + "svg.fonttype": "none", + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.facecolor": "white", + "axes.spines.top": False, + "axes.spines.right": False, + } + ) + + +def panel_label(ax: plt.Axes, label: str, x: float = -0.14, y: float = 1.08) -> None: + ax.text(x, y, label, transform=ax.transAxes, ha="left", va="top", fontsize=12, fontweight="normal") + + +def save_figure(fig: plt.Figure, paths: Paths, stem: str) -> None: + for suffix in ("pdf", "png", "svg"): + kwargs = {"dpi": 600} if suffix == "png" else {} + fig.savefig(paths.figures / f"{stem}.{suffix}", bbox_inches="tight", **kwargs) + print("Saved figure:", paths.figures / stem) + + +def plot_training(paths: Paths) -> plt.Figure: + """Plot raw and lightly smoothed trajectories; no false replicate bands.""" + set_plot_style() + histories = {} + for run_key in RUNS: + _, _, metrics_path = model_paths(paths, run_key) + histories[run_key] = pd.read_csv(metrics_path) + + fig, axes = plt.subplots(4, 2, figsize=(8.2, 8.8), sharex=True) + for task_i, task in enumerate(TASKS): + for col_i, kind in enumerate(("score", "loss")): + ax = axes[task_i, col_i] + metric_col = f"{task}_train_{kind}" + for run_key, history in histories.items(): + if metric_col not in history: + continue + x = history["epoch"].to_numpy() + raw = history[metric_col].to_numpy() + smooth = pd.Series(raw).rolling(15, center=True, min_periods=1).mean().to_numpy() + ax.plot(x, raw, color=RUN_COLORS[run_key], alpha=0.16, linewidth=0.45) + ax.plot(x, smooth, color=RUN_COLORS[run_key], linewidth=1.25, label=RUNS[run_key]["label"]) + ax.set_title(task) + if col_i == 0: + ax.set_ylabel("Training score") + ax.set_ylim(0.4, 1.02) + else: + ax.set_ylabel("Training loss") + ax.set_ylim(bottom=0) + if task_i == len(TASKS) - 1: + ax.set_xlabel("Epoch") + panel_label(axes[0, 0], "a") + panel_label(axes[0, 1], "b") + handles = [ + mpl.lines.Line2D([0], [0], color=RUN_COLORS[k], lw=1.5, label=RUNS[k]["label"]) + for k in RUNS + ] + fig.legend(handles=handles, loc="lower center", ncol=5, frameon=False, bbox_to_anchor=(0.5, 0.005)) + fig.subplots_adjust(left=0.10, right=0.98, top=0.97, bottom=0.08, hspace=0.42, wspace=0.28) + save_figure(fig, paths, "Fig_training_curves_75ep_leave_one_task_out") + return fig + + +def annotate_heatmap(ax: plt.Axes, values: np.ndarray, fmt: str, threshold: float | None = None) -> None: + if threshold is None: + threshold = float(np.nanmedian(values)) + for i in range(values.shape[0]): + for j in range(values.shape[1]): + value = values[i, j] + ax.text(j, i, format(value, fmt), ha="center", va="center", fontsize=7, color="white" if value < threshold else "black") + + +def plot_downstream(paths: Paths) -> plt.Figure: + set_plot_style() + summary = pd.read_csv(paths.results / "calms21_model_summary.csv") + per_class = pd.read_csv(paths.results / "calms21_per_class_metrics.csv") + contribution = pd.read_csv(paths.results / "per_class_task_contribution.csv") + model_order = list(RUNS) + model_labels = [RUNS[k]["label"] for k in model_order] + class_order = per_class.sort_values("class_id")["class_name"].drop_duplicates().tolist() + + absolute = ( + per_class.pivot(index="class_name", columns="model", values="f1") + .loc[class_order, model_order] + ) + effect = ( + contribution.pivot(index="class_name", columns="removed_task", values="delta_f1_all_minus_without") + .loc[class_order, list(TASKS)] + ) + + fig = plt.figure(figsize=(11.2, 3.65)) + gs = fig.add_gridspec(1, 3, width_ratios=(1.35, 1.15, 1.0), wspace=0.48) + ax0 = fig.add_subplot(gs[0, 0]) + im0 = ax0.imshow(absolute.values, cmap="viridis", vmin=0, vmax=1, aspect="auto") + ax0.set_xticks(range(len(model_order)), model_labels, rotation=38, ha="right") + ax0.set_yticks(range(len(class_order)), class_order) + ax0.set_xlabel("Frozen LISBET encoder") + ax0.set_ylabel("CalMS21 behavior") + ax0.set_title("Per-behavior decoding") + annotate_heatmap(ax0, absolute.values, ".2f", threshold=0.55) + fig.colorbar(im0, ax=ax0, fraction=0.046, pad=0.03, label="Test F1") + panel_label(ax0, "c") + + ax1 = fig.add_subplot(gs[0, 1]) + vmax = max(0.01, float(np.nanmax(np.abs(effect.values)))) + norm = TwoSlopeNorm(vmin=-vmax, vcenter=0, vmax=vmax) + im1 = ax1.imshow(effect.values, cmap="coolwarm", norm=norm, aspect="auto") + ax1.set_xticks(range(len(TASKS)), TASKS) + ax1.set_yticks(range(len(class_order)), class_order) + ax1.set_xlabel("Removed task") + ax1.set_ylabel("CalMS21 behavior") + ax1.set_title("Task-removal effect") + for i in range(effect.shape[0]): + for j in range(effect.shape[1]): + ax1.text(j, i, f"{effect.values[i, j]:+.3f}", ha="center", va="center", fontsize=7) + fig.colorbar(im1, ax=ax1, fraction=0.046, pad=0.03, label=r"$F1_{all}-F1_{without}$") + panel_label(ax1, "d") + + ax2 = fig.add_subplot(gs[0, 2]) + summary = summary.set_index("model").loc[model_order].reset_index() + x = np.arange(len(summary)) + y = summary["macro_f1"].to_numpy() + yerr = np.vstack( + [ + y - summary["macro_f1_ci_low"].to_numpy(), + summary["macro_f1_ci_high"].to_numpy() - y, + ] + ) + ax2.errorbar(x, y, yerr=yerr, fmt="o", color="#202020", ecolor="#555555", capsize=3, linewidth=1.1) + ax2.set_xticks(x, model_labels, rotation=38, ha="right") + ax2.set_ylabel("Test macro-F1") + ax2.set_xlabel("Frozen LISBET encoder") + ax2.set_ylim(max(0, float(np.nanmin(yerr[0] * -1 + y)) - 0.05), min(1, float(np.nanmax(yerr[1] + y)) + 0.05)) + ax2.set_title("Overall behavioral decoding") + panel_label(ax2, "e") + + fig.subplots_adjust(left=0.07, right=0.98, top=0.92, bottom=0.28) + save_figure(fig, paths, "Fig_calms21_knn_task_ablation_posthoc") + return fig + + +def plot_all(paths: Paths) -> None: + paths.create_outputs() + training = plot_training(paths) + downstream = plot_downstream(paths) + plt.show() + plt.close(training) + plt.close(downstream) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("stage", choices=("validate", "prepare", "embed", "evaluate", "plot", "all")) + parser.add_argument( + "--lisbet-root", + type=Path, + default=Path.home() / "Dokumente" / "Lisbet", + help="Directory containing the LISBET repository, datasets, and ablation folder", + ) + parser.add_argument("--force", action="store_true", help="Recompute prepared data or embeddings") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--k-grid", type=int, nargs="+", default=[1, 3, 5, 11, 21]) + parser.add_argument( + "--max-train-per-class", + type=int, + default=10000, + help="Balanced cap per class for the kNN reference set; applied identically to every model", + ) + parser.add_argument("--max-validation-per-class", type=int, default=5000) + parser.add_argument("--prediction-chunk-size", type=int, default=5000) + parser.add_argument("--bootstrap-replicates", type=int, default=5000) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + paths = Paths.from_root(args.lisbet_root) + paths.create_outputs() + if args.stage in ("validate", "all"): + validate_runs(paths) + if args.stage in ("prepare", "all"): + prepare_calms21(paths, force=args.force) + if args.stage in ("embed", "all"): + compute_embeddings(paths, force=args.force) + if args.stage in ("evaluate", "all"): + evaluate(paths, args) + if args.stage in ("plot", "all"): + plot_all(paths) + + +if __name__ == "__main__": + main() diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed2.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed2.py new file mode 100644 index 0000000..90a84f2 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed2.py @@ -0,0 +1,994 @@ +#!/usr/bin/env python3 +"""Reviewer 2.2: LISBET leave-one-task-out analysis on CalMS21. + +Stages +------ +validate Validate the five matched checkpoints and training histories. +prepare Convert CalMS21 task 1 JSON files to multi-animal DLC CSV files. +embed Compute frozen embeddings for train/test videos and all five models. +evaluate Tune a fixed kNN probe on training videos and evaluate official test videos. +plot Generate training, downstream, and combined figures. +all Run all stages in sequence. + +The downstream comparison uses the official CalMS21 task 1 train/test separation. +The k value is selected using only the all-task model and training videos, then held +fixed for every leave-one-task-out model. Confidence intervals resample test videos, +not frames. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import yaml +from matplotlib.colors import TwoSlopeNorm +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import GroupKFold +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import StandardScaler + + +TASKS = ("cons", "order", "shift", "warp") +RUNS = { + "all_tasks": { + "folder": "all_tasks_seed2_75ep", + "model_id": "all_tasks_seed2_75ep", + "label": "All tasks", + "removed": None, + }, + "without_cons": { + "folder": "triple_order_shift_warp_seed2_75ep", + "model_id": "triple_order_shift_warp_seed2_75ep", + "label": "Without cons", + "removed": "cons", + }, + "without_order": { + "folder": "triple_cons_shift_warp_seed2_75ep", + "model_id": "triple_cons_shift_warp_seed2_75ep", + "label": "Without order", + "removed": "order", + }, + "without_shift": { + "folder": "triple_cons_order_warp_seed2_75ep", + "model_id": "triple_cons_order_warp_seed2_75ep", + "label": "Without shift", + "removed": "shift", + }, + "without_warp": { + "folder": "triple_cons_order_shift_seed2_75ep", + "model_id": "triple_cons_order_shift_seed2_75ep", + "label": "Without warp", + "removed": "warp", + }, +} + +RUN_COLORS = { + "all_tasks": "#202020", + "without_cons": "#4477AA", + "without_order": "#EE6677", + "without_shift": "#228833", + "without_warp": "#CCBB44", +} + + +@dataclass(frozen=True) +class Paths: + lisbet_root: Path + ablation_root: Path + calms_root: Path + work: Path + prepared: Path + embedders: Path + embeddings: Path + results: Path + figures: Path + logs: Path + + @classmethod + def from_root(cls, lisbet_root: Path) -> "Paths": + lisbet_root = lisbet_root.expanduser().resolve() + ablation_root = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "results" / "auxiliary_task_assessment" / "task_full_combinations_w200_75ep_train1600" + work = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "reviewer_2_2_calms21_posthoc_seed2" + return cls( + lisbet_root=lisbet_root, + ablation_root=ablation_root, + calms_root=lisbet_root / "lisbet_datasets" / "datasets" / "CalMS21", + work=work, + prepared=work / "prepared_calms21_task1_dlc", + embedders=work / "exported_embedders", + embeddings=work / "embeddings", + results=work / "results", + figures=work / "figures", + logs=work / "logs", + ) + + def create_outputs(self) -> None: + for path in ( + self.work, + self.prepared, + self.embedders, + self.embeddings, + self.results, + self.figures, + self.logs, + ): + path.mkdir(parents=True, exist_ok=True) + + +def model_paths(paths: Paths, run_key: str) -> tuple[Path, Path, Path]: + run = RUNS[run_key] + base = paths.ablation_root / run["folder"] / "models" / run["model_id"] + return ( + base / "model_config.yml", + base / "weights" / "weights_last.pt", + base / "training_history" / "version_0" / "metrics.csv", + ) + + +def validate_runs(paths: Paths) -> pd.DataFrame: + """Check that encoder settings match and only the expected head is removed.""" + rows = [] + backbone_reference = None + input_reference = None + for run_key, run in RUNS.items(): + config_path, weights_path, metrics_path = model_paths(paths, run_key) + for required in (config_path, weights_path, metrics_path): + if not required.exists(): + raise FileNotFoundError(f"Missing required file: {required}") + + config = yaml.safe_load(config_path.read_text()) + metrics = pd.read_csv(metrics_path) + backbone = config.get("backbone") + input_features = config.get("input_features") + if backbone_reference is None: + backbone_reference = backbone + input_reference = input_features + if backbone != backbone_reference: + raise ValueError(f"Backbone mismatch in {run_key}") + if input_features != input_reference: + raise ValueError(f"Input-feature mismatch in {run_key}") + + observed_heads = set(config.get("out_heads", {})) + expected_heads = set(TASKS) + if run["removed"] is not None: + expected_heads.remove(run["removed"]) + if observed_heads != expected_heads: + raise ValueError( + f"Unexpected heads for {run_key}: observed={sorted(observed_heads)}, " + f"expected={sorted(expected_heads)}" + ) + if config.get("window_size") != 200 or config.get("backbone", {}).get("max_length") != 200: + raise ValueError(f"{run_key} is not a matched window-200 model") + if "epoch" not in metrics or int(metrics["epoch"].max()) != 599: + print(f"[warn] {run_key} does not contain all 75 epochs by old validation; continuing for 75ep analysis") + + row = { + "run": run_key, + "label": run["label"], + "removed_task": run["removed"] or "none", + "epochs": int(metrics["epoch"].max()) + 1, + "checkpoint_bytes": weights_path.stat().st_size, + "heads": ",".join(sorted(observed_heads)), + } + for task in TASKS: + for kind in ("score", "loss"): + col = f"{task}_train_{kind}" + row[f"final_{task}_{kind}"] = float(metrics[col].dropna().iloc[-1]) if col in metrics else np.nan + rows.append(row) + + out = pd.DataFrame(rows) + paths.create_outputs() + out.to_csv(paths.results / "validated_run_inventory.csv", index=False) + print(out.to_string(index=False)) + return out + + +def safe_name(value: str) -> str: + value = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value)).strip("_") + return value or "record" + + +def task1_json(paths: Paths, split: str) -> Path: + expected = ( + paths.calms_root + / "task1_classic_classification" + / f"calms21_task1_{split}.json" + ) + if expected.exists(): + return expected + cache_root = paths.lisbet_root / "lisbet_datasets" / "datasets" / ".cache" / "lisbet" + cached = sorted( + cache_root.glob( + "*-task1_classic_classification.zip.unzip/" + f"task1_classic_classification/calms21_task1_{split}.json" + ) + ) + if len(cached) == 1: + print(f"Using cached CalMS21 task 1 {split} data: {cached[0]}") + return cached[0] + if len(cached) > 1: + raise RuntimeError( + f"Found multiple cached CalMS21 task 1 {split} files: {cached}. " + "Remove stale cache copies or materialize the intended dataset path." + ) + raise FileNotFoundError( + f"CalMS21 task 1 {split} JSON was not found at:\n{expected}\n" + "The CalMS21 directory may be an unmaterialized link. Confirm it with " + "`find -L lisbet_datasets/datasets/CalMS21 -maxdepth 3 -type f`." + ) + + +def extract_pose_arrays(record: dict) -> tuple[np.ndarray, np.ndarray]: + """Normalize CalMS21 task 1 arrays to (frames, individuals, keypoints, axes).""" + # Task 1 JSON stores keypoints as + # (frames, individuals, coordinates, keypoints). + positions = np.asarray(record["keypoints"], dtype=np.float32).transpose((0, 1, 3, 2)) + scores = np.asarray(record["scores"], dtype=np.float32) + if positions.ndim != 4 or positions.shape[1:] != (2, 7, 2): + raise ValueError(f"Unexpected CalMS21 position shape after conversion: {positions.shape}") + # CalMS21 task 1 stores confidence as (frames, individuals, keypoints). + # Some converted variants use (frames, keypoints, individuals), so accept + # and normalize either representation for the DLC writer. + if scores.shape == (positions.shape[0], 7, 2): + scores = scores.transpose((0, 2, 1)) + if scores.shape != positions.shape[:3]: + raise ValueError(f"Position/score shape mismatch: {positions.shape}, {scores.shape}") + return positions, scores + + +def write_dlc_csv(path: Path, positions: np.ndarray, scores: np.ndarray) -> None: + individuals = ("resident", "intruder") + keypoints = ("nose", "left_ear", "right_ear", "neck", "left_hip", "right_hip", "tail") + columns = [] + values = [] + for ind_i, individual in enumerate(individuals): + for kp_i, keypoint in enumerate(keypoints): + for coord_i, coord in enumerate(("x", "y")): + columns.append(("calms21", individual, keypoint, coord)) + values.append(positions[:, ind_i, kp_i, coord_i]) + columns.append(("calms21", individual, keypoint, "likelihood")) + values.append(scores[:, ind_i, kp_i]) + frame = pd.DataFrame( + np.column_stack(values), + columns=pd.MultiIndex.from_tuples( + columns, names=("scorer", "individuals", "bodyparts", "coords") + ), + ) + frame.to_csv(path, index=True) + + +def prepare_calms21(paths: Paths, force: bool = False) -> pd.DataFrame: + """Create DLC inputs and exact frame-label tables for official task 1 splits.""" + paths.create_outputs() + manifest_path = paths.prepared / "record_manifest.csv" + if manifest_path.exists() and not force: + manifest = pd.read_csv(manifest_path) + print(f"Using existing prepared dataset: {manifest_path}") + return manifest + + rows = [] + global_vocab = None + seen_ids = set() + for split in ("train", "test"): + pose_dir = paths.prepared / split / "poses" + label_dir = paths.prepared / split / "labels" + pose_dir.mkdir(parents=True, exist_ok=True) + label_dir.mkdir(parents=True, exist_ok=True) + source_path = task1_json(paths, split) + print(f"Loading CalMS21 {split} JSON (this may take several minutes): {source_path}") + # json.load avoids holding an additional full-size text copy of these + # large (approximately 0.6-1.2 GB) source files in memory. + with source_path.open("r", encoding="utf-8") as source: + raw = json.load(source) + + for condition, condition_records in raw.items(): + for original_id, record in condition_records.items(): + stem = safe_name(f"{condition}__{original_id}") + print(f"Preparing {split} record: {stem}") + if stem in seen_ids: + raise ValueError(f"Duplicate generated record ID: {stem}") + seen_ids.add(stem) + + positions, scores = extract_pose_arrays(record) + annotations = np.asarray(record["annotations"], dtype=int) + if len(annotations) != len(positions): + raise ValueError(f"Annotation/pose length mismatch for {stem}") + + vocab_map = record.get("metadata", {}).get("vocab") + if not vocab_map: + raise ValueError(f"Missing behavior vocabulary for {stem}") + vocab = [name for name, idx in sorted(vocab_map.items(), key=lambda item: item[1])] + if global_vocab is None: + global_vocab = vocab + if vocab != global_vocab: + raise ValueError(f"Behavior vocabulary differs in {stem}: {vocab} != {global_vocab}") + if annotations.min() < 0 or annotations.max() >= len(vocab): + raise ValueError(f"Annotation IDs outside vocabulary for {stem}") + + # LISBET's DLC loader scans sequence subdirectories and accepts + # filenames matching `tracking*.csv`. + record_pose_dir = pose_dir / stem + record_pose_dir.mkdir(parents=True, exist_ok=True) + pose_path = record_pose_dir / "tracking.csv" + labels_path = label_dir / f"{stem}.csv" + if force or not pose_path.exists(): + write_dlc_csv(pose_path, positions, scores) + label_df = pd.DataFrame( + { + "record_id": stem, + "frame_idx": np.arange(len(annotations), dtype=int), + "label_id": annotations, + "label_name": [vocab[i] for i in annotations], + } + ) + label_df.to_csv(labels_path, index=False) + rows.append( + { + "split": split, + "condition": condition, + "original_id": original_id, + "record_id": stem, + "n_frames": len(annotations), + "pose_csv": str(pose_path), + "labels_csv": str(labels_path), + } + ) + + manifest = pd.DataFrame(rows).sort_values(["split", "record_id"]) + manifest.to_csv(manifest_path, index=False) + (paths.prepared / "behavior_vocabulary.json").write_text(json.dumps(global_vocab, indent=2)) + print(f"Prepared {len(manifest)} records in {paths.prepared}") + print(manifest.groupby("split")["n_frames"].agg(["count", "sum"])) + return manifest + + +def run_command(command: list[str], stdout_path: Path, stderr_path: Path) -> None: + print("Running:", " ".join(command)) + completed = subprocess.run(command, text=True, capture_output=True) + stdout_path.write_text(completed.stdout) + stderr_path.write_text(completed.stderr) + if completed.returncode != 0: + raise RuntimeError( + f"Command failed with exit code {completed.returncode}. See {stderr_path}\n" + f"Last stderr lines:\n{completed.stderr[-3000:]}" + ) + + +def find_exported_embedder(output_path: Path) -> tuple[Path, Path] | None: + configs = sorted(output_path.rglob("model_config.yml")) + if not configs: + configs = sorted(output_path.rglob("*.yml")) + sorted(output_path.rglob("*.yaml")) + weights = sorted(output_path.rglob("*.pt")) + sorted(output_path.rglob("*.pth")) + if len(configs) == 1 and len(weights) == 1: + return configs[0], weights[0] + if len(configs) == 0 and len(weights) == 0: + return None + raise ValueError( + f"Expected one exported config and one weight file under {output_path}; " + f"found configs={configs}, weights={weights}" + ) + + +def export_embedder(paths: Paths, run_key: str, force: bool = False) -> tuple[Path, Path]: + """Export the shared trained backbone with the embedding inference head.""" + output_path = paths.embedders / run_key + output_path.mkdir(parents=True, exist_ok=True) + existing = find_exported_embedder(output_path) + if existing is not None and not force: + print(f"[skip] {run_key}: using exported embedder {existing[0]}") + return existing + + source_config, source_weights, _ = model_paths(paths, run_key) + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "export_embedder", + str(source_config), + str(source_weights), + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"export_embedder_{run_key}_stdout.txt", + paths.logs / f"export_embedder_{run_key}_stderr.txt", + ) + exported = find_exported_embedder(output_path) + if exported is None: + raise FileNotFoundError(f"export_embedder produced no model files in {output_path}") + print(f"Exported {run_key} embedder: {exported[0]}, {exported[1]}") + return exported + + +def compute_embeddings(paths: Paths, force: bool = False) -> None: + """Run betman for each model and official data split.""" + validate_runs(paths) + manifest = prepare_calms21(paths, force=False) + paths.create_outputs() + for run_key in RUNS: + config_path, weights_path = export_embedder(paths, run_key, force=False) + for split in ("train", "test"): + data_path = paths.prepared / split / "poses" + output_path = paths.embeddings / run_key / split + output_path.mkdir(parents=True, exist_ok=True) + expected = list(output_path.rglob("features_lisbet_embedding.csv")) + expected_records = int((manifest["split"] == split).sum()) + if len(expected) == expected_records and not force: + print(f"[skip] {run_key}/{split}: found {len(expected)} embedding files") + continue + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "compute_embeddings", + str(data_path), + str(config_path), + str(weights_path), + "--data_format", + "maDLC", + "--window_size", + "200", + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"embedding_{run_key}_{split}_stdout.txt", + paths.logs / f"embedding_{run_key}_{split}_stderr.txt", + ) + + +def embedding_columns(frame: pd.DataFrame) -> list[str]: + cols = [c for c in frame.columns if str(c).isdigit()] + if cols: + return sorted(cols, key=lambda c: int(str(c))) + excluded = {"frame_idx", "time", "index"} + cols = [ + c + for c in frame.columns + if c not in excluded + and not str(c).startswith("Unnamed") + and pd.api.types.is_numeric_dtype(frame[c]) + ] + if not cols: + raise ValueError(f"No embedding dimensions found. Columns: {frame.columns.tolist()}") + return cols + + +def index_embedding_files(paths: Paths, run_key: str, split: str, record_ids: list[str]) -> dict[str, Path]: + files = sorted((paths.embeddings / run_key / split).rglob("features_lisbet_embedding.csv")) + mapping = {} + for record_id in record_ids: + matches = [f for f in files if record_id in f.parts or record_id in str(f)] + if len(matches) != 1: + raise ValueError( + f"Expected exactly one embedding file for {run_key}/{split}/{record_id}; " + f"found {len(matches)}. Available examples: {files[:5]}" + ) + mapping[record_id] = matches[0] + return mapping + + +def load_split_embeddings( + paths: Paths, run_key: str, split: str, manifest: pd.DataFrame +) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[str]]: + subset = manifest[manifest["split"] == split].sort_values("record_id") + record_ids = subset["record_id"].tolist() + file_map = index_embedding_files(paths, run_key, split, record_ids) + x_parts, y_parts, group_parts = [], [], [] + class_names = json.loads((paths.prepared / "behavior_vocabulary.json").read_text()) + + for row in subset.itertuples(index=False): + labels = pd.read_csv(row.labels_csv) + emb = pd.read_csv(file_map[row.record_id]) + cols = embedding_columns(emb) + if "frame_idx" in emb.columns: + merged = labels.merge(emb[["frame_idx"] + cols], on="frame_idx", how="inner", validate="one_to_one") + if len(merged) != len(labels): + raise ValueError( + f"Frame-index alignment lost rows for {run_key}/{row.record_id}: " + f"labels={len(labels)}, merged={len(merged)}" + ) + x = merged[cols].to_numpy(dtype=np.float32) + y = merged["label_id"].to_numpy(dtype=int) + else: + if len(emb) != len(labels): + raise ValueError( + f"Embedding/label length mismatch for {run_key}/{row.record_id}: " + f"embeddings={len(emb)}, labels={len(labels)}. No silent truncation is performed." + ) + x = emb[cols].to_numpy(dtype=np.float32) + y = labels["label_id"].to_numpy(dtype=int) + if not np.isfinite(x).all(): + raise ValueError(f"Non-finite embeddings in {run_key}/{row.record_id}") + x_parts.append(x) + y_parts.append(y) + group_parts.append(np.repeat(row.record_id, len(y))) + + return ( + np.concatenate(x_parts), + np.concatenate(y_parts), + np.concatenate(group_parts), + class_names, + ) + + +def balanced_sample(y: np.ndarray, max_per_class: int, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + selected = [] + for cls in np.unique(y): + idx = np.flatnonzero(y == cls) + if len(idx) > max_per_class: + idx = rng.choice(idx, size=max_per_class, replace=False) + selected.append(np.sort(idx)) + return np.sort(np.concatenate(selected)) + + +def metrics_from_cm(cm: np.ndarray) -> dict[str, np.ndarray | float]: + cm = np.asarray(cm, dtype=float) + tp = np.diag(cm) + support = cm.sum(axis=1) + predicted = cm.sum(axis=0) + recall = np.divide(tp, support, out=np.zeros_like(tp), where=support > 0) + precision = np.divide(tp, predicted, out=np.zeros_like(tp), where=predicted > 0) + f1 = np.divide(2 * precision * recall, precision + recall, out=np.zeros_like(tp), where=(precision + recall) > 0) + return { + "accuracy": float(tp.sum() / cm.sum()), + "balanced_accuracy": float(np.mean(recall)), + "macro_f1": float(np.mean(f1)), + "precision": precision, + "recall": recall, + "f1": f1, + "support": support, + } + + +def predict_in_chunks(clf: KNeighborsClassifier, x: np.ndarray, chunk_size: int) -> np.ndarray: + predictions = [] + for start in range(0, len(x), chunk_size): + predictions.append(clf.predict(x[start : start + chunk_size])) + return np.concatenate(predictions) + + +def tune_k_on_all_task_training( + x: np.ndarray, + y: np.ndarray, + groups: np.ndarray, + k_grid: list[int], + max_train_per_class: int, + max_val_per_class: int, + seed: int, +) -> tuple[int, pd.DataFrame]: + n_groups = len(np.unique(groups)) + if n_groups < 3: + raise ValueError("At least three training videos are required for grouped k selection") + splitter = GroupKFold(n_splits=min(5, n_groups)) + rows = [] + labels = np.arange(len(np.unique(y))) + for fold, (train_idx, val_idx) in enumerate(splitter.split(x, y, groups)): + train_keep = train_idx[balanced_sample(y[train_idx], max_train_per_class, seed + fold)] + val_keep = val_idx[balanced_sample(y[val_idx], max_val_per_class, seed + 100 + fold)] + scaler = StandardScaler() + x_train = scaler.fit_transform(x[train_keep]).astype(np.float32) + x_val = scaler.transform(x[val_keep]).astype(np.float32) + for k in k_grid: + clf = KNeighborsClassifier(n_neighbors=k, weights="distance", metric="euclidean", n_jobs=-1) + clf.fit(x_train, y[train_keep]) + pred = predict_in_chunks(clf, x_val, chunk_size=5000) + cm = confusion_matrix(y[val_keep], pred, labels=labels) + metric = metrics_from_cm(cm) + rows.append({"fold": fold, "k": k, "macro_f1": metric["macro_f1"]}) + results = pd.DataFrame(rows) + summary = results.groupby("k", as_index=False)["macro_f1"].agg(["mean", "std"]).reset_index() + best_k = int(summary.sort_values(["mean", "k"], ascending=[False, True]).iloc[0]["k"]) + print("Grouped training-only k selection:") + print(summary.to_string(index=False)) + print("Selected k:", best_k) + return best_k, results + + +def evaluate(paths: Paths, args: argparse.Namespace) -> None: + """Evaluate frozen representations on the untouched official task 1 test videos.""" + paths.create_outputs() + manifest = pd.read_csv(paths.prepared / "record_manifest.csv") + labels = None + + x_all, y_train, groups_train, class_names = load_split_embeddings( + paths, "all_tasks", "train", manifest + ) + labels = np.arange(len(class_names)) + best_k, tuning = tune_k_on_all_task_training( + x_all, + y_train, + groups_train, + args.k_grid, + args.max_train_per_class, + args.max_validation_per_class, + args.seed, + ) + tuning.to_csv(paths.results / "knn_k_selection_grouped_training.csv", index=False) + (paths.results / "selected_knn_k.json").write_text(json.dumps({"k": best_k}, indent=2)) + del x_all, y_train, groups_train + + model_cms: dict[str, dict[str, np.ndarray]] = {} + summary_rows = [] + per_class_rows = [] + cm_rows = [] + + for model_i, run_key in enumerate(RUNS): + print(f"Evaluating {RUNS[run_key]['label']}...") + x_train, y_train, _, names_train = load_split_embeddings(paths, run_key, "train", manifest) + if names_train != class_names: + raise ValueError("Class-name mismatch across models") + train_keep = balanced_sample(y_train, args.max_train_per_class, args.seed) + scaler = StandardScaler() + x_train_scaled = scaler.fit_transform(x_train[train_keep]).astype(np.float32) + clf = KNeighborsClassifier( + n_neighbors=best_k, weights="distance", metric="euclidean", n_jobs=-1 + ) + clf.fit(x_train_scaled, y_train[train_keep]) + del x_train, x_train_scaled, y_train + + x_test, y_test, groups_test, names_test = load_split_embeddings(paths, run_key, "test", manifest) + if names_test != class_names: + raise ValueError("Class-name mismatch across splits") + model_cms[run_key] = {} + for record_id in sorted(np.unique(groups_test)): + idx = np.flatnonzero(groups_test == record_id) + x_record = scaler.transform(x_test[idx]).astype(np.float32) + pred = predict_in_chunks(clf, x_record, args.prediction_chunk_size) + cm = confusion_matrix(y_test[idx], pred, labels=labels) + model_cms[run_key][record_id] = cm + for true_i in labels: + for pred_i in labels: + cm_rows.append( + { + "model": run_key, + "record_id": record_id, + "true_class": class_names[true_i], + "predicted_class": class_names[pred_i], + "count": int(cm[true_i, pred_i]), + } + ) + total_cm = sum(model_cms[run_key].values()) + metric = metrics_from_cm(total_cm) + summary_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "removed_task": RUNS[run_key]["removed"] or "none", + "k": best_k, + "n_train_probe": len(train_keep), + "n_test_frames": int(total_cm.sum()), + "accuracy": metric["accuracy"], + "balanced_accuracy": metric["balanced_accuracy"], + "macro_f1": metric["macro_f1"], + } + ) + for class_i, class_name in enumerate(class_names): + per_class_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "class_id": class_i, + "class_name": class_name, + "precision": metric["precision"][class_i], + "recall": metric["recall"][class_i], + "f1": metric["f1"][class_i], + "support": int(metric["support"][class_i]), + } + ) + del x_test, y_test, groups_test + + pd.DataFrame(cm_rows).to_csv(paths.results / "test_video_confusion_matrices.csv", index=False) + per_class = pd.DataFrame(per_class_rows) + per_class.to_csv(paths.results / "calms21_per_class_metrics.csv", index=False) + + rng = np.random.default_rng(args.seed) + test_videos = sorted(next(iter(model_cms.values())).keys()) + bootstrap_rows = [] + delta_rows = [] + for bootstrap_i in range(args.bootstrap_replicates): + draw = rng.choice(test_videos, size=len(test_videos), replace=True) + boot_metrics = {} + for run_key in RUNS: + cm = sum(model_cms[run_key][video] for video in draw) + boot_metrics[run_key] = metrics_from_cm(cm) + bootstrap_rows.append( + { + "bootstrap": bootstrap_i, + "model": run_key, + "macro_f1": boot_metrics[run_key]["macro_f1"], + "balanced_accuracy": boot_metrics[run_key]["balanced_accuracy"], + } + ) + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + delta_rows.append( + { + "bootstrap": bootstrap_i, + "removed_task": run["removed"], + "delta_macro_f1_all_minus_without": ( + boot_metrics["all_tasks"]["macro_f1"] - boot_metrics[run_key]["macro_f1"] + ), + "delta_per_class_f1_all_minus_without": ( + boot_metrics["all_tasks"]["f1"] - boot_metrics[run_key]["f1"] + ).tolist(), + } + ) + + bootstrap = pd.DataFrame(bootstrap_rows) + bootstrap.to_csv(paths.results / "test_video_bootstrap_metrics.csv", index=False) + delta_long = [] + for row in delta_rows: + for class_i, class_name in enumerate(class_names): + delta_long.append( + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "class_name": class_name, + "delta_f1_all_minus_without": row["delta_per_class_f1_all_minus_without"][class_i], + } + ) + pd.DataFrame(delta_long).to_csv(paths.results / "paired_bootstrap_per_class_task_effects.csv", index=False) + delta_overall = pd.DataFrame( + [ + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "delta_macro_f1_all_minus_without": row["delta_macro_f1_all_minus_without"], + } + for row in delta_rows + ] + ) + delta_overall.to_csv(paths.results / "paired_bootstrap_macro_f1_task_effects.csv", index=False) + + summary = pd.DataFrame(summary_rows) + ci = ( + bootstrap.groupby("model")["macro_f1"] + .quantile([0.025, 0.975]) + .unstack() + .rename(columns={0.025: "macro_f1_ci_low", 0.975: "macro_f1_ci_high"}) + .reset_index() + ) + summary = summary.merge(ci, on="model", how="left") + summary.to_csv(paths.results / "calms21_model_summary.csv", index=False) + + contribution_rows = [] + all_pc = per_class[per_class["model"] == "all_tasks"].set_index("class_name") + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + ablated_pc = per_class[per_class["model"] == run_key].set_index("class_name") + for class_name in class_names: + contribution_rows.append( + { + "removed_task": run["removed"], + "class_name": class_name, + "delta_f1_all_minus_without": ( + all_pc.loc[class_name, "f1"] - ablated_pc.loc[class_name, "f1"] + ), + } + ) + pd.DataFrame(contribution_rows).to_csv(paths.results / "per_class_task_contribution.csv", index=False) + print(summary.to_string(index=False)) + + +def set_plot_style() -> None: + mpl.rcParams.update( + { + "font.family": "sans-serif", + "font.sans-serif": ["DejaVu Sans"], + "font.size": 8, + "axes.titlesize": 9, + "axes.labelsize": 8, + "xtick.labelsize": 7, + "ytick.labelsize": 7, + "legend.fontsize": 7, + "axes.linewidth": 0.7, + "pdf.fonttype": 42, + "ps.fonttype": 42, + "svg.fonttype": "none", + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.facecolor": "white", + "axes.spines.top": False, + "axes.spines.right": False, + } + ) + + +def panel_label(ax: plt.Axes, label: str, x: float = -0.14, y: float = 1.08) -> None: + ax.text(x, y, label, transform=ax.transAxes, ha="left", va="top", fontsize=12, fontweight="normal") + + +def save_figure(fig: plt.Figure, paths: Paths, stem: str) -> None: + for suffix in ("pdf", "png", "svg"): + kwargs = {"dpi": 600} if suffix == "png" else {} + fig.savefig(paths.figures / f"{stem}.{suffix}", bbox_inches="tight", **kwargs) + print("Saved figure:", paths.figures / stem) + + +def plot_training(paths: Paths) -> plt.Figure: + """Plot raw and lightly smoothed trajectories; no false replicate bands.""" + set_plot_style() + histories = {} + for run_key in RUNS: + _, _, metrics_path = model_paths(paths, run_key) + histories[run_key] = pd.read_csv(metrics_path) + + fig, axes = plt.subplots(4, 2, figsize=(8.2, 8.8), sharex=True) + for task_i, task in enumerate(TASKS): + for col_i, kind in enumerate(("score", "loss")): + ax = axes[task_i, col_i] + metric_col = f"{task}_train_{kind}" + for run_key, history in histories.items(): + if metric_col not in history: + continue + x = history["epoch"].to_numpy() + raw = history[metric_col].to_numpy() + smooth = pd.Series(raw).rolling(15, center=True, min_periods=1).mean().to_numpy() + ax.plot(x, raw, color=RUN_COLORS[run_key], alpha=0.16, linewidth=0.45) + ax.plot(x, smooth, color=RUN_COLORS[run_key], linewidth=1.25, label=RUNS[run_key]["label"]) + ax.set_title(task) + if col_i == 0: + ax.set_ylabel("Training score") + ax.set_ylim(0.4, 1.02) + else: + ax.set_ylabel("Training loss") + ax.set_ylim(bottom=0) + if task_i == len(TASKS) - 1: + ax.set_xlabel("Epoch") + panel_label(axes[0, 0], "a") + panel_label(axes[0, 1], "b") + handles = [ + mpl.lines.Line2D([0], [0], color=RUN_COLORS[k], lw=1.5, label=RUNS[k]["label"]) + for k in RUNS + ] + fig.legend(handles=handles, loc="lower center", ncol=5, frameon=False, bbox_to_anchor=(0.5, 0.005)) + fig.subplots_adjust(left=0.10, right=0.98, top=0.97, bottom=0.08, hspace=0.42, wspace=0.28) + save_figure(fig, paths, "Fig_training_curves_75ep_leave_one_task_out") + return fig + + +def annotate_heatmap(ax: plt.Axes, values: np.ndarray, fmt: str, threshold: float | None = None) -> None: + if threshold is None: + threshold = float(np.nanmedian(values)) + for i in range(values.shape[0]): + for j in range(values.shape[1]): + value = values[i, j] + ax.text(j, i, format(value, fmt), ha="center", va="center", fontsize=7, color="white" if value < threshold else "black") + + +def plot_downstream(paths: Paths) -> plt.Figure: + set_plot_style() + summary = pd.read_csv(paths.results / "calms21_model_summary.csv") + per_class = pd.read_csv(paths.results / "calms21_per_class_metrics.csv") + contribution = pd.read_csv(paths.results / "per_class_task_contribution.csv") + model_order = list(RUNS) + model_labels = [RUNS[k]["label"] for k in model_order] + class_order = per_class.sort_values("class_id")["class_name"].drop_duplicates().tolist() + + absolute = ( + per_class.pivot(index="class_name", columns="model", values="f1") + .loc[class_order, model_order] + ) + effect = ( + contribution.pivot(index="class_name", columns="removed_task", values="delta_f1_all_minus_without") + .loc[class_order, list(TASKS)] + ) + + fig = plt.figure(figsize=(11.2, 3.65)) + gs = fig.add_gridspec(1, 3, width_ratios=(1.35, 1.15, 1.0), wspace=0.48) + ax0 = fig.add_subplot(gs[0, 0]) + im0 = ax0.imshow(absolute.values, cmap="viridis", vmin=0, vmax=1, aspect="auto") + ax0.set_xticks(range(len(model_order)), model_labels, rotation=38, ha="right") + ax0.set_yticks(range(len(class_order)), class_order) + ax0.set_xlabel("Frozen LISBET encoder") + ax0.set_ylabel("CalMS21 behavior") + ax0.set_title("Per-behavior decoding") + annotate_heatmap(ax0, absolute.values, ".2f", threshold=0.55) + fig.colorbar(im0, ax=ax0, fraction=0.046, pad=0.03, label="Test F1") + panel_label(ax0, "c") + + ax1 = fig.add_subplot(gs[0, 1]) + vmax = max(0.01, float(np.nanmax(np.abs(effect.values)))) + norm = TwoSlopeNorm(vmin=-vmax, vcenter=0, vmax=vmax) + im1 = ax1.imshow(effect.values, cmap="coolwarm", norm=norm, aspect="auto") + ax1.set_xticks(range(len(TASKS)), TASKS) + ax1.set_yticks(range(len(class_order)), class_order) + ax1.set_xlabel("Removed task") + ax1.set_ylabel("CalMS21 behavior") + ax1.set_title("Task-removal effect") + for i in range(effect.shape[0]): + for j in range(effect.shape[1]): + ax1.text(j, i, f"{effect.values[i, j]:+.3f}", ha="center", va="center", fontsize=7) + fig.colorbar(im1, ax=ax1, fraction=0.046, pad=0.03, label=r"$F1_{all}-F1_{without}$") + panel_label(ax1, "d") + + ax2 = fig.add_subplot(gs[0, 2]) + summary = summary.set_index("model").loc[model_order].reset_index() + x = np.arange(len(summary)) + y = summary["macro_f1"].to_numpy() + yerr = np.vstack( + [ + y - summary["macro_f1_ci_low"].to_numpy(), + summary["macro_f1_ci_high"].to_numpy() - y, + ] + ) + ax2.errorbar(x, y, yerr=yerr, fmt="o", color="#202020", ecolor="#555555", capsize=3, linewidth=1.1) + ax2.set_xticks(x, model_labels, rotation=38, ha="right") + ax2.set_ylabel("Test macro-F1") + ax2.set_xlabel("Frozen LISBET encoder") + ax2.set_ylim(max(0, float(np.nanmin(yerr[0] * -1 + y)) - 0.05), min(1, float(np.nanmax(yerr[1] + y)) + 0.05)) + ax2.set_title("Overall behavioral decoding") + panel_label(ax2, "e") + + fig.subplots_adjust(left=0.07, right=0.98, top=0.92, bottom=0.28) + save_figure(fig, paths, "Fig_calms21_knn_task_ablation_posthoc") + return fig + + +def plot_all(paths: Paths) -> None: + paths.create_outputs() + training = plot_training(paths) + downstream = plot_downstream(paths) + plt.show() + plt.close(training) + plt.close(downstream) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("stage", choices=("validate", "prepare", "embed", "evaluate", "plot", "all")) + parser.add_argument( + "--lisbet-root", + type=Path, + default=Path.home() / "Dokumente" / "Lisbet", + help="Directory containing the LISBET repository, datasets, and ablation folder", + ) + parser.add_argument("--force", action="store_true", help="Recompute prepared data or embeddings") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--k-grid", type=int, nargs="+", default=[1, 3, 5, 11, 21]) + parser.add_argument( + "--max-train-per-class", + type=int, + default=10000, + help="Balanced cap per class for the kNN reference set; applied identically to every model", + ) + parser.add_argument("--max-validation-per-class", type=int, default=5000) + parser.add_argument("--prediction-chunk-size", type=int, default=5000) + parser.add_argument("--bootstrap-replicates", type=int, default=5000) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + paths = Paths.from_root(args.lisbet_root) + paths.create_outputs() + if args.stage in ("validate", "all"): + validate_runs(paths) + if args.stage in ("prepare", "all"): + prepare_calms21(paths, force=args.force) + if args.stage in ("embed", "all"): + compute_embeddings(paths, force=args.force) + if args.stage in ("evaluate", "all"): + evaluate(paths, args) + if args.stage in ("plot", "all"): + plot_all(paths) + + +if __name__ == "__main__": + main() diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed3.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed3.py new file mode 100644 index 0000000..1b8cf37 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed3.py @@ -0,0 +1,994 @@ +#!/usr/bin/env python3 +"""Reviewer 2.2: LISBET leave-one-task-out analysis on CalMS21. + +Stages +------ +validate Validate the five matched checkpoints and training histories. +prepare Convert CalMS21 task 1 JSON files to multi-animal DLC CSV files. +embed Compute frozen embeddings for train/test videos and all five models. +evaluate Tune a fixed kNN probe on training videos and evaluate official test videos. +plot Generate training, downstream, and combined figures. +all Run all stages in sequence. + +The downstream comparison uses the official CalMS21 task 1 train/test separation. +The k value is selected using only the all-task model and training videos, then held +fixed for every leave-one-task-out model. Confidence intervals resample test videos, +not frames. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import yaml +from matplotlib.colors import TwoSlopeNorm +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import GroupKFold +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import StandardScaler + + +TASKS = ("cons", "order", "shift", "warp") +RUNS = { + "all_tasks": { + "folder": "all_tasks_seed3_75ep", + "model_id": "all_tasks_seed3_75ep", + "label": "All tasks", + "removed": None, + }, + "without_cons": { + "folder": "triple_order_shift_warp_seed3_75ep", + "model_id": "triple_order_shift_warp_seed3_75ep", + "label": "Without cons", + "removed": "cons", + }, + "without_order": { + "folder": "triple_cons_shift_warp_seed3_75ep", + "model_id": "triple_cons_shift_warp_seed3_75ep", + "label": "Without order", + "removed": "order", + }, + "without_shift": { + "folder": "triple_cons_order_warp_seed3_75ep", + "model_id": "triple_cons_order_warp_seed3_75ep", + "label": "Without shift", + "removed": "shift", + }, + "without_warp": { + "folder": "triple_cons_order_shift_seed3_75ep", + "model_id": "triple_cons_order_shift_seed3_75ep", + "label": "Without warp", + "removed": "warp", + }, +} + +RUN_COLORS = { + "all_tasks": "#202020", + "without_cons": "#4477AA", + "without_order": "#EE6677", + "without_shift": "#228833", + "without_warp": "#CCBB44", +} + + +@dataclass(frozen=True) +class Paths: + lisbet_root: Path + ablation_root: Path + calms_root: Path + work: Path + prepared: Path + embedders: Path + embeddings: Path + results: Path + figures: Path + logs: Path + + @classmethod + def from_root(cls, lisbet_root: Path) -> "Paths": + lisbet_root = lisbet_root.expanduser().resolve() + ablation_root = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "results" / "auxiliary_task_assessment" / "task_full_combinations_w200_75ep_train1600" + work = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "reviewer_2_2_calms21_posthoc_seed3" + return cls( + lisbet_root=lisbet_root, + ablation_root=ablation_root, + calms_root=lisbet_root / "lisbet_datasets" / "datasets" / "CalMS21", + work=work, + prepared=work / "prepared_calms21_task1_dlc", + embedders=work / "exported_embedders", + embeddings=work / "embeddings", + results=work / "results", + figures=work / "figures", + logs=work / "logs", + ) + + def create_outputs(self) -> None: + for path in ( + self.work, + self.prepared, + self.embedders, + self.embeddings, + self.results, + self.figures, + self.logs, + ): + path.mkdir(parents=True, exist_ok=True) + + +def model_paths(paths: Paths, run_key: str) -> tuple[Path, Path, Path]: + run = RUNS[run_key] + base = paths.ablation_root / run["folder"] / "models" / run["model_id"] + return ( + base / "model_config.yml", + base / "weights" / "weights_last.pt", + base / "training_history" / "version_0" / "metrics.csv", + ) + + +def validate_runs(paths: Paths) -> pd.DataFrame: + """Check that encoder settings match and only the expected head is removed.""" + rows = [] + backbone_reference = None + input_reference = None + for run_key, run in RUNS.items(): + config_path, weights_path, metrics_path = model_paths(paths, run_key) + for required in (config_path, weights_path, metrics_path): + if not required.exists(): + raise FileNotFoundError(f"Missing required file: {required}") + + config = yaml.safe_load(config_path.read_text()) + metrics = pd.read_csv(metrics_path) + backbone = config.get("backbone") + input_features = config.get("input_features") + if backbone_reference is None: + backbone_reference = backbone + input_reference = input_features + if backbone != backbone_reference: + raise ValueError(f"Backbone mismatch in {run_key}") + if input_features != input_reference: + raise ValueError(f"Input-feature mismatch in {run_key}") + + observed_heads = set(config.get("out_heads", {})) + expected_heads = set(TASKS) + if run["removed"] is not None: + expected_heads.remove(run["removed"]) + if observed_heads != expected_heads: + raise ValueError( + f"Unexpected heads for {run_key}: observed={sorted(observed_heads)}, " + f"expected={sorted(expected_heads)}" + ) + if config.get("window_size") != 200 or config.get("backbone", {}).get("max_length") != 200: + raise ValueError(f"{run_key} is not a matched window-200 model") + if "epoch" not in metrics or int(metrics["epoch"].max()) != 599: + print(f"[warn] {run_key} does not contain all 75 epochs by old validation; continuing for 75ep analysis") + + row = { + "run": run_key, + "label": run["label"], + "removed_task": run["removed"] or "none", + "epochs": int(metrics["epoch"].max()) + 1, + "checkpoint_bytes": weights_path.stat().st_size, + "heads": ",".join(sorted(observed_heads)), + } + for task in TASKS: + for kind in ("score", "loss"): + col = f"{task}_train_{kind}" + row[f"final_{task}_{kind}"] = float(metrics[col].dropna().iloc[-1]) if col in metrics else np.nan + rows.append(row) + + out = pd.DataFrame(rows) + paths.create_outputs() + out.to_csv(paths.results / "validated_run_inventory.csv", index=False) + print(out.to_string(index=False)) + return out + + +def safe_name(value: str) -> str: + value = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value)).strip("_") + return value or "record" + + +def task1_json(paths: Paths, split: str) -> Path: + expected = ( + paths.calms_root + / "task1_classic_classification" + / f"calms21_task1_{split}.json" + ) + if expected.exists(): + return expected + cache_root = paths.lisbet_root / "lisbet_datasets" / "datasets" / ".cache" / "lisbet" + cached = sorted( + cache_root.glob( + "*-task1_classic_classification.zip.unzip/" + f"task1_classic_classification/calms21_task1_{split}.json" + ) + ) + if len(cached) == 1: + print(f"Using cached CalMS21 task 1 {split} data: {cached[0]}") + return cached[0] + if len(cached) > 1: + raise RuntimeError( + f"Found multiple cached CalMS21 task 1 {split} files: {cached}. " + "Remove stale cache copies or materialize the intended dataset path." + ) + raise FileNotFoundError( + f"CalMS21 task 1 {split} JSON was not found at:\n{expected}\n" + "The CalMS21 directory may be an unmaterialized link. Confirm it with " + "`find -L lisbet_datasets/datasets/CalMS21 -maxdepth 3 -type f`." + ) + + +def extract_pose_arrays(record: dict) -> tuple[np.ndarray, np.ndarray]: + """Normalize CalMS21 task 1 arrays to (frames, individuals, keypoints, axes).""" + # Task 1 JSON stores keypoints as + # (frames, individuals, coordinates, keypoints). + positions = np.asarray(record["keypoints"], dtype=np.float32).transpose((0, 1, 3, 2)) + scores = np.asarray(record["scores"], dtype=np.float32) + if positions.ndim != 4 or positions.shape[1:] != (2, 7, 2): + raise ValueError(f"Unexpected CalMS21 position shape after conversion: {positions.shape}") + # CalMS21 task 1 stores confidence as (frames, individuals, keypoints). + # Some converted variants use (frames, keypoints, individuals), so accept + # and normalize either representation for the DLC writer. + if scores.shape == (positions.shape[0], 7, 2): + scores = scores.transpose((0, 2, 1)) + if scores.shape != positions.shape[:3]: + raise ValueError(f"Position/score shape mismatch: {positions.shape}, {scores.shape}") + return positions, scores + + +def write_dlc_csv(path: Path, positions: np.ndarray, scores: np.ndarray) -> None: + individuals = ("resident", "intruder") + keypoints = ("nose", "left_ear", "right_ear", "neck", "left_hip", "right_hip", "tail") + columns = [] + values = [] + for ind_i, individual in enumerate(individuals): + for kp_i, keypoint in enumerate(keypoints): + for coord_i, coord in enumerate(("x", "y")): + columns.append(("calms21", individual, keypoint, coord)) + values.append(positions[:, ind_i, kp_i, coord_i]) + columns.append(("calms21", individual, keypoint, "likelihood")) + values.append(scores[:, ind_i, kp_i]) + frame = pd.DataFrame( + np.column_stack(values), + columns=pd.MultiIndex.from_tuples( + columns, names=("scorer", "individuals", "bodyparts", "coords") + ), + ) + frame.to_csv(path, index=True) + + +def prepare_calms21(paths: Paths, force: bool = False) -> pd.DataFrame: + """Create DLC inputs and exact frame-label tables for official task 1 splits.""" + paths.create_outputs() + manifest_path = paths.prepared / "record_manifest.csv" + if manifest_path.exists() and not force: + manifest = pd.read_csv(manifest_path) + print(f"Using existing prepared dataset: {manifest_path}") + return manifest + + rows = [] + global_vocab = None + seen_ids = set() + for split in ("train", "test"): + pose_dir = paths.prepared / split / "poses" + label_dir = paths.prepared / split / "labels" + pose_dir.mkdir(parents=True, exist_ok=True) + label_dir.mkdir(parents=True, exist_ok=True) + source_path = task1_json(paths, split) + print(f"Loading CalMS21 {split} JSON (this may take several minutes): {source_path}") + # json.load avoids holding an additional full-size text copy of these + # large (approximately 0.6-1.2 GB) source files in memory. + with source_path.open("r", encoding="utf-8") as source: + raw = json.load(source) + + for condition, condition_records in raw.items(): + for original_id, record in condition_records.items(): + stem = safe_name(f"{condition}__{original_id}") + print(f"Preparing {split} record: {stem}") + if stem in seen_ids: + raise ValueError(f"Duplicate generated record ID: {stem}") + seen_ids.add(stem) + + positions, scores = extract_pose_arrays(record) + annotations = np.asarray(record["annotations"], dtype=int) + if len(annotations) != len(positions): + raise ValueError(f"Annotation/pose length mismatch for {stem}") + + vocab_map = record.get("metadata", {}).get("vocab") + if not vocab_map: + raise ValueError(f"Missing behavior vocabulary for {stem}") + vocab = [name for name, idx in sorted(vocab_map.items(), key=lambda item: item[1])] + if global_vocab is None: + global_vocab = vocab + if vocab != global_vocab: + raise ValueError(f"Behavior vocabulary differs in {stem}: {vocab} != {global_vocab}") + if annotations.min() < 0 or annotations.max() >= len(vocab): + raise ValueError(f"Annotation IDs outside vocabulary for {stem}") + + # LISBET's DLC loader scans sequence subdirectories and accepts + # filenames matching `tracking*.csv`. + record_pose_dir = pose_dir / stem + record_pose_dir.mkdir(parents=True, exist_ok=True) + pose_path = record_pose_dir / "tracking.csv" + labels_path = label_dir / f"{stem}.csv" + if force or not pose_path.exists(): + write_dlc_csv(pose_path, positions, scores) + label_df = pd.DataFrame( + { + "record_id": stem, + "frame_idx": np.arange(len(annotations), dtype=int), + "label_id": annotations, + "label_name": [vocab[i] for i in annotations], + } + ) + label_df.to_csv(labels_path, index=False) + rows.append( + { + "split": split, + "condition": condition, + "original_id": original_id, + "record_id": stem, + "n_frames": len(annotations), + "pose_csv": str(pose_path), + "labels_csv": str(labels_path), + } + ) + + manifest = pd.DataFrame(rows).sort_values(["split", "record_id"]) + manifest.to_csv(manifest_path, index=False) + (paths.prepared / "behavior_vocabulary.json").write_text(json.dumps(global_vocab, indent=2)) + print(f"Prepared {len(manifest)} records in {paths.prepared}") + print(manifest.groupby("split")["n_frames"].agg(["count", "sum"])) + return manifest + + +def run_command(command: list[str], stdout_path: Path, stderr_path: Path) -> None: + print("Running:", " ".join(command)) + completed = subprocess.run(command, text=True, capture_output=True) + stdout_path.write_text(completed.stdout) + stderr_path.write_text(completed.stderr) + if completed.returncode != 0: + raise RuntimeError( + f"Command failed with exit code {completed.returncode}. See {stderr_path}\n" + f"Last stderr lines:\n{completed.stderr[-3000:]}" + ) + + +def find_exported_embedder(output_path: Path) -> tuple[Path, Path] | None: + configs = sorted(output_path.rglob("model_config.yml")) + if not configs: + configs = sorted(output_path.rglob("*.yml")) + sorted(output_path.rglob("*.yaml")) + weights = sorted(output_path.rglob("*.pt")) + sorted(output_path.rglob("*.pth")) + if len(configs) == 1 and len(weights) == 1: + return configs[0], weights[0] + if len(configs) == 0 and len(weights) == 0: + return None + raise ValueError( + f"Expected one exported config and one weight file under {output_path}; " + f"found configs={configs}, weights={weights}" + ) + + +def export_embedder(paths: Paths, run_key: str, force: bool = False) -> tuple[Path, Path]: + """Export the shared trained backbone with the embedding inference head.""" + output_path = paths.embedders / run_key + output_path.mkdir(parents=True, exist_ok=True) + existing = find_exported_embedder(output_path) + if existing is not None and not force: + print(f"[skip] {run_key}: using exported embedder {existing[0]}") + return existing + + source_config, source_weights, _ = model_paths(paths, run_key) + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "export_embedder", + str(source_config), + str(source_weights), + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"export_embedder_{run_key}_stdout.txt", + paths.logs / f"export_embedder_{run_key}_stderr.txt", + ) + exported = find_exported_embedder(output_path) + if exported is None: + raise FileNotFoundError(f"export_embedder produced no model files in {output_path}") + print(f"Exported {run_key} embedder: {exported[0]}, {exported[1]}") + return exported + + +def compute_embeddings(paths: Paths, force: bool = False) -> None: + """Run betman for each model and official data split.""" + validate_runs(paths) + manifest = prepare_calms21(paths, force=False) + paths.create_outputs() + for run_key in RUNS: + config_path, weights_path = export_embedder(paths, run_key, force=False) + for split in ("train", "test"): + data_path = paths.prepared / split / "poses" + output_path = paths.embeddings / run_key / split + output_path.mkdir(parents=True, exist_ok=True) + expected = list(output_path.rglob("features_lisbet_embedding.csv")) + expected_records = int((manifest["split"] == split).sum()) + if len(expected) == expected_records and not force: + print(f"[skip] {run_key}/{split}: found {len(expected)} embedding files") + continue + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "compute_embeddings", + str(data_path), + str(config_path), + str(weights_path), + "--data_format", + "maDLC", + "--window_size", + "200", + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"embedding_{run_key}_{split}_stdout.txt", + paths.logs / f"embedding_{run_key}_{split}_stderr.txt", + ) + + +def embedding_columns(frame: pd.DataFrame) -> list[str]: + cols = [c for c in frame.columns if str(c).isdigit()] + if cols: + return sorted(cols, key=lambda c: int(str(c))) + excluded = {"frame_idx", "time", "index"} + cols = [ + c + for c in frame.columns + if c not in excluded + and not str(c).startswith("Unnamed") + and pd.api.types.is_numeric_dtype(frame[c]) + ] + if not cols: + raise ValueError(f"No embedding dimensions found. Columns: {frame.columns.tolist()}") + return cols + + +def index_embedding_files(paths: Paths, run_key: str, split: str, record_ids: list[str]) -> dict[str, Path]: + files = sorted((paths.embeddings / run_key / split).rglob("features_lisbet_embedding.csv")) + mapping = {} + for record_id in record_ids: + matches = [f for f in files if record_id in f.parts or record_id in str(f)] + if len(matches) != 1: + raise ValueError( + f"Expected exactly one embedding file for {run_key}/{split}/{record_id}; " + f"found {len(matches)}. Available examples: {files[:5]}" + ) + mapping[record_id] = matches[0] + return mapping + + +def load_split_embeddings( + paths: Paths, run_key: str, split: str, manifest: pd.DataFrame +) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[str]]: + subset = manifest[manifest["split"] == split].sort_values("record_id") + record_ids = subset["record_id"].tolist() + file_map = index_embedding_files(paths, run_key, split, record_ids) + x_parts, y_parts, group_parts = [], [], [] + class_names = json.loads((paths.prepared / "behavior_vocabulary.json").read_text()) + + for row in subset.itertuples(index=False): + labels = pd.read_csv(row.labels_csv) + emb = pd.read_csv(file_map[row.record_id]) + cols = embedding_columns(emb) + if "frame_idx" in emb.columns: + merged = labels.merge(emb[["frame_idx"] + cols], on="frame_idx", how="inner", validate="one_to_one") + if len(merged) != len(labels): + raise ValueError( + f"Frame-index alignment lost rows for {run_key}/{row.record_id}: " + f"labels={len(labels)}, merged={len(merged)}" + ) + x = merged[cols].to_numpy(dtype=np.float32) + y = merged["label_id"].to_numpy(dtype=int) + else: + if len(emb) != len(labels): + raise ValueError( + f"Embedding/label length mismatch for {run_key}/{row.record_id}: " + f"embeddings={len(emb)}, labels={len(labels)}. No silent truncation is performed." + ) + x = emb[cols].to_numpy(dtype=np.float32) + y = labels["label_id"].to_numpy(dtype=int) + if not np.isfinite(x).all(): + raise ValueError(f"Non-finite embeddings in {run_key}/{row.record_id}") + x_parts.append(x) + y_parts.append(y) + group_parts.append(np.repeat(row.record_id, len(y))) + + return ( + np.concatenate(x_parts), + np.concatenate(y_parts), + np.concatenate(group_parts), + class_names, + ) + + +def balanced_sample(y: np.ndarray, max_per_class: int, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + selected = [] + for cls in np.unique(y): + idx = np.flatnonzero(y == cls) + if len(idx) > max_per_class: + idx = rng.choice(idx, size=max_per_class, replace=False) + selected.append(np.sort(idx)) + return np.sort(np.concatenate(selected)) + + +def metrics_from_cm(cm: np.ndarray) -> dict[str, np.ndarray | float]: + cm = np.asarray(cm, dtype=float) + tp = np.diag(cm) + support = cm.sum(axis=1) + predicted = cm.sum(axis=0) + recall = np.divide(tp, support, out=np.zeros_like(tp), where=support > 0) + precision = np.divide(tp, predicted, out=np.zeros_like(tp), where=predicted > 0) + f1 = np.divide(2 * precision * recall, precision + recall, out=np.zeros_like(tp), where=(precision + recall) > 0) + return { + "accuracy": float(tp.sum() / cm.sum()), + "balanced_accuracy": float(np.mean(recall)), + "macro_f1": float(np.mean(f1)), + "precision": precision, + "recall": recall, + "f1": f1, + "support": support, + } + + +def predict_in_chunks(clf: KNeighborsClassifier, x: np.ndarray, chunk_size: int) -> np.ndarray: + predictions = [] + for start in range(0, len(x), chunk_size): + predictions.append(clf.predict(x[start : start + chunk_size])) + return np.concatenate(predictions) + + +def tune_k_on_all_task_training( + x: np.ndarray, + y: np.ndarray, + groups: np.ndarray, + k_grid: list[int], + max_train_per_class: int, + max_val_per_class: int, + seed: int, +) -> tuple[int, pd.DataFrame]: + n_groups = len(np.unique(groups)) + if n_groups < 3: + raise ValueError("At least three training videos are required for grouped k selection") + splitter = GroupKFold(n_splits=min(5, n_groups)) + rows = [] + labels = np.arange(len(np.unique(y))) + for fold, (train_idx, val_idx) in enumerate(splitter.split(x, y, groups)): + train_keep = train_idx[balanced_sample(y[train_idx], max_train_per_class, seed + fold)] + val_keep = val_idx[balanced_sample(y[val_idx], max_val_per_class, seed + 100 + fold)] + scaler = StandardScaler() + x_train = scaler.fit_transform(x[train_keep]).astype(np.float32) + x_val = scaler.transform(x[val_keep]).astype(np.float32) + for k in k_grid: + clf = KNeighborsClassifier(n_neighbors=k, weights="distance", metric="euclidean", n_jobs=-1) + clf.fit(x_train, y[train_keep]) + pred = predict_in_chunks(clf, x_val, chunk_size=5000) + cm = confusion_matrix(y[val_keep], pred, labels=labels) + metric = metrics_from_cm(cm) + rows.append({"fold": fold, "k": k, "macro_f1": metric["macro_f1"]}) + results = pd.DataFrame(rows) + summary = results.groupby("k", as_index=False)["macro_f1"].agg(["mean", "std"]).reset_index() + best_k = int(summary.sort_values(["mean", "k"], ascending=[False, True]).iloc[0]["k"]) + print("Grouped training-only k selection:") + print(summary.to_string(index=False)) + print("Selected k:", best_k) + return best_k, results + + +def evaluate(paths: Paths, args: argparse.Namespace) -> None: + """Evaluate frozen representations on the untouched official task 1 test videos.""" + paths.create_outputs() + manifest = pd.read_csv(paths.prepared / "record_manifest.csv") + labels = None + + x_all, y_train, groups_train, class_names = load_split_embeddings( + paths, "all_tasks", "train", manifest + ) + labels = np.arange(len(class_names)) + best_k, tuning = tune_k_on_all_task_training( + x_all, + y_train, + groups_train, + args.k_grid, + args.max_train_per_class, + args.max_validation_per_class, + args.seed, + ) + tuning.to_csv(paths.results / "knn_k_selection_grouped_training.csv", index=False) + (paths.results / "selected_knn_k.json").write_text(json.dumps({"k": best_k}, indent=2)) + del x_all, y_train, groups_train + + model_cms: dict[str, dict[str, np.ndarray]] = {} + summary_rows = [] + per_class_rows = [] + cm_rows = [] + + for model_i, run_key in enumerate(RUNS): + print(f"Evaluating {RUNS[run_key]['label']}...") + x_train, y_train, _, names_train = load_split_embeddings(paths, run_key, "train", manifest) + if names_train != class_names: + raise ValueError("Class-name mismatch across models") + train_keep = balanced_sample(y_train, args.max_train_per_class, args.seed) + scaler = StandardScaler() + x_train_scaled = scaler.fit_transform(x_train[train_keep]).astype(np.float32) + clf = KNeighborsClassifier( + n_neighbors=best_k, weights="distance", metric="euclidean", n_jobs=-1 + ) + clf.fit(x_train_scaled, y_train[train_keep]) + del x_train, x_train_scaled, y_train + + x_test, y_test, groups_test, names_test = load_split_embeddings(paths, run_key, "test", manifest) + if names_test != class_names: + raise ValueError("Class-name mismatch across splits") + model_cms[run_key] = {} + for record_id in sorted(np.unique(groups_test)): + idx = np.flatnonzero(groups_test == record_id) + x_record = scaler.transform(x_test[idx]).astype(np.float32) + pred = predict_in_chunks(clf, x_record, args.prediction_chunk_size) + cm = confusion_matrix(y_test[idx], pred, labels=labels) + model_cms[run_key][record_id] = cm + for true_i in labels: + for pred_i in labels: + cm_rows.append( + { + "model": run_key, + "record_id": record_id, + "true_class": class_names[true_i], + "predicted_class": class_names[pred_i], + "count": int(cm[true_i, pred_i]), + } + ) + total_cm = sum(model_cms[run_key].values()) + metric = metrics_from_cm(total_cm) + summary_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "removed_task": RUNS[run_key]["removed"] or "none", + "k": best_k, + "n_train_probe": len(train_keep), + "n_test_frames": int(total_cm.sum()), + "accuracy": metric["accuracy"], + "balanced_accuracy": metric["balanced_accuracy"], + "macro_f1": metric["macro_f1"], + } + ) + for class_i, class_name in enumerate(class_names): + per_class_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "class_id": class_i, + "class_name": class_name, + "precision": metric["precision"][class_i], + "recall": metric["recall"][class_i], + "f1": metric["f1"][class_i], + "support": int(metric["support"][class_i]), + } + ) + del x_test, y_test, groups_test + + pd.DataFrame(cm_rows).to_csv(paths.results / "test_video_confusion_matrices.csv", index=False) + per_class = pd.DataFrame(per_class_rows) + per_class.to_csv(paths.results / "calms21_per_class_metrics.csv", index=False) + + rng = np.random.default_rng(args.seed) + test_videos = sorted(next(iter(model_cms.values())).keys()) + bootstrap_rows = [] + delta_rows = [] + for bootstrap_i in range(args.bootstrap_replicates): + draw = rng.choice(test_videos, size=len(test_videos), replace=True) + boot_metrics = {} + for run_key in RUNS: + cm = sum(model_cms[run_key][video] for video in draw) + boot_metrics[run_key] = metrics_from_cm(cm) + bootstrap_rows.append( + { + "bootstrap": bootstrap_i, + "model": run_key, + "macro_f1": boot_metrics[run_key]["macro_f1"], + "balanced_accuracy": boot_metrics[run_key]["balanced_accuracy"], + } + ) + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + delta_rows.append( + { + "bootstrap": bootstrap_i, + "removed_task": run["removed"], + "delta_macro_f1_all_minus_without": ( + boot_metrics["all_tasks"]["macro_f1"] - boot_metrics[run_key]["macro_f1"] + ), + "delta_per_class_f1_all_minus_without": ( + boot_metrics["all_tasks"]["f1"] - boot_metrics[run_key]["f1"] + ).tolist(), + } + ) + + bootstrap = pd.DataFrame(bootstrap_rows) + bootstrap.to_csv(paths.results / "test_video_bootstrap_metrics.csv", index=False) + delta_long = [] + for row in delta_rows: + for class_i, class_name in enumerate(class_names): + delta_long.append( + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "class_name": class_name, + "delta_f1_all_minus_without": row["delta_per_class_f1_all_minus_without"][class_i], + } + ) + pd.DataFrame(delta_long).to_csv(paths.results / "paired_bootstrap_per_class_task_effects.csv", index=False) + delta_overall = pd.DataFrame( + [ + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "delta_macro_f1_all_minus_without": row["delta_macro_f1_all_minus_without"], + } + for row in delta_rows + ] + ) + delta_overall.to_csv(paths.results / "paired_bootstrap_macro_f1_task_effects.csv", index=False) + + summary = pd.DataFrame(summary_rows) + ci = ( + bootstrap.groupby("model")["macro_f1"] + .quantile([0.025, 0.975]) + .unstack() + .rename(columns={0.025: "macro_f1_ci_low", 0.975: "macro_f1_ci_high"}) + .reset_index() + ) + summary = summary.merge(ci, on="model", how="left") + summary.to_csv(paths.results / "calms21_model_summary.csv", index=False) + + contribution_rows = [] + all_pc = per_class[per_class["model"] == "all_tasks"].set_index("class_name") + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + ablated_pc = per_class[per_class["model"] == run_key].set_index("class_name") + for class_name in class_names: + contribution_rows.append( + { + "removed_task": run["removed"], + "class_name": class_name, + "delta_f1_all_minus_without": ( + all_pc.loc[class_name, "f1"] - ablated_pc.loc[class_name, "f1"] + ), + } + ) + pd.DataFrame(contribution_rows).to_csv(paths.results / "per_class_task_contribution.csv", index=False) + print(summary.to_string(index=False)) + + +def set_plot_style() -> None: + mpl.rcParams.update( + { + "font.family": "sans-serif", + "font.sans-serif": ["DejaVu Sans"], + "font.size": 8, + "axes.titlesize": 9, + "axes.labelsize": 8, + "xtick.labelsize": 7, + "ytick.labelsize": 7, + "legend.fontsize": 7, + "axes.linewidth": 0.7, + "pdf.fonttype": 42, + "ps.fonttype": 42, + "svg.fonttype": "none", + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.facecolor": "white", + "axes.spines.top": False, + "axes.spines.right": False, + } + ) + + +def panel_label(ax: plt.Axes, label: str, x: float = -0.14, y: float = 1.08) -> None: + ax.text(x, y, label, transform=ax.transAxes, ha="left", va="top", fontsize=12, fontweight="normal") + + +def save_figure(fig: plt.Figure, paths: Paths, stem: str) -> None: + for suffix in ("pdf", "png", "svg"): + kwargs = {"dpi": 600} if suffix == "png" else {} + fig.savefig(paths.figures / f"{stem}.{suffix}", bbox_inches="tight", **kwargs) + print("Saved figure:", paths.figures / stem) + + +def plot_training(paths: Paths) -> plt.Figure: + """Plot raw and lightly smoothed trajectories; no false replicate bands.""" + set_plot_style() + histories = {} + for run_key in RUNS: + _, _, metrics_path = model_paths(paths, run_key) + histories[run_key] = pd.read_csv(metrics_path) + + fig, axes = plt.subplots(4, 2, figsize=(8.2, 8.8), sharex=True) + for task_i, task in enumerate(TASKS): + for col_i, kind in enumerate(("score", "loss")): + ax = axes[task_i, col_i] + metric_col = f"{task}_train_{kind}" + for run_key, history in histories.items(): + if metric_col not in history: + continue + x = history["epoch"].to_numpy() + raw = history[metric_col].to_numpy() + smooth = pd.Series(raw).rolling(15, center=True, min_periods=1).mean().to_numpy() + ax.plot(x, raw, color=RUN_COLORS[run_key], alpha=0.16, linewidth=0.45) + ax.plot(x, smooth, color=RUN_COLORS[run_key], linewidth=1.25, label=RUNS[run_key]["label"]) + ax.set_title(task) + if col_i == 0: + ax.set_ylabel("Training score") + ax.set_ylim(0.4, 1.02) + else: + ax.set_ylabel("Training loss") + ax.set_ylim(bottom=0) + if task_i == len(TASKS) - 1: + ax.set_xlabel("Epoch") + panel_label(axes[0, 0], "a") + panel_label(axes[0, 1], "b") + handles = [ + mpl.lines.Line2D([0], [0], color=RUN_COLORS[k], lw=1.5, label=RUNS[k]["label"]) + for k in RUNS + ] + fig.legend(handles=handles, loc="lower center", ncol=5, frameon=False, bbox_to_anchor=(0.5, 0.005)) + fig.subplots_adjust(left=0.10, right=0.98, top=0.97, bottom=0.08, hspace=0.42, wspace=0.28) + save_figure(fig, paths, "Fig_training_curves_75ep_leave_one_task_out") + return fig + + +def annotate_heatmap(ax: plt.Axes, values: np.ndarray, fmt: str, threshold: float | None = None) -> None: + if threshold is None: + threshold = float(np.nanmedian(values)) + for i in range(values.shape[0]): + for j in range(values.shape[1]): + value = values[i, j] + ax.text(j, i, format(value, fmt), ha="center", va="center", fontsize=7, color="white" if value < threshold else "black") + + +def plot_downstream(paths: Paths) -> plt.Figure: + set_plot_style() + summary = pd.read_csv(paths.results / "calms21_model_summary.csv") + per_class = pd.read_csv(paths.results / "calms21_per_class_metrics.csv") + contribution = pd.read_csv(paths.results / "per_class_task_contribution.csv") + model_order = list(RUNS) + model_labels = [RUNS[k]["label"] for k in model_order] + class_order = per_class.sort_values("class_id")["class_name"].drop_duplicates().tolist() + + absolute = ( + per_class.pivot(index="class_name", columns="model", values="f1") + .loc[class_order, model_order] + ) + effect = ( + contribution.pivot(index="class_name", columns="removed_task", values="delta_f1_all_minus_without") + .loc[class_order, list(TASKS)] + ) + + fig = plt.figure(figsize=(11.2, 3.65)) + gs = fig.add_gridspec(1, 3, width_ratios=(1.35, 1.15, 1.0), wspace=0.48) + ax0 = fig.add_subplot(gs[0, 0]) + im0 = ax0.imshow(absolute.values, cmap="viridis", vmin=0, vmax=1, aspect="auto") + ax0.set_xticks(range(len(model_order)), model_labels, rotation=38, ha="right") + ax0.set_yticks(range(len(class_order)), class_order) + ax0.set_xlabel("Frozen LISBET encoder") + ax0.set_ylabel("CalMS21 behavior") + ax0.set_title("Per-behavior decoding") + annotate_heatmap(ax0, absolute.values, ".2f", threshold=0.55) + fig.colorbar(im0, ax=ax0, fraction=0.046, pad=0.03, label="Test F1") + panel_label(ax0, "c") + + ax1 = fig.add_subplot(gs[0, 1]) + vmax = max(0.01, float(np.nanmax(np.abs(effect.values)))) + norm = TwoSlopeNorm(vmin=-vmax, vcenter=0, vmax=vmax) + im1 = ax1.imshow(effect.values, cmap="coolwarm", norm=norm, aspect="auto") + ax1.set_xticks(range(len(TASKS)), TASKS) + ax1.set_yticks(range(len(class_order)), class_order) + ax1.set_xlabel("Removed task") + ax1.set_ylabel("CalMS21 behavior") + ax1.set_title("Task-removal effect") + for i in range(effect.shape[0]): + for j in range(effect.shape[1]): + ax1.text(j, i, f"{effect.values[i, j]:+.3f}", ha="center", va="center", fontsize=7) + fig.colorbar(im1, ax=ax1, fraction=0.046, pad=0.03, label=r"$F1_{all}-F1_{without}$") + panel_label(ax1, "d") + + ax2 = fig.add_subplot(gs[0, 2]) + summary = summary.set_index("model").loc[model_order].reset_index() + x = np.arange(len(summary)) + y = summary["macro_f1"].to_numpy() + yerr = np.vstack( + [ + y - summary["macro_f1_ci_low"].to_numpy(), + summary["macro_f1_ci_high"].to_numpy() - y, + ] + ) + ax2.errorbar(x, y, yerr=yerr, fmt="o", color="#202020", ecolor="#555555", capsize=3, linewidth=1.1) + ax2.set_xticks(x, model_labels, rotation=38, ha="right") + ax2.set_ylabel("Test macro-F1") + ax2.set_xlabel("Frozen LISBET encoder") + ax2.set_ylim(max(0, float(np.nanmin(yerr[0] * -1 + y)) - 0.05), min(1, float(np.nanmax(yerr[1] + y)) + 0.05)) + ax2.set_title("Overall behavioral decoding") + panel_label(ax2, "e") + + fig.subplots_adjust(left=0.07, right=0.98, top=0.92, bottom=0.28) + save_figure(fig, paths, "Fig_calms21_knn_task_ablation_posthoc") + return fig + + +def plot_all(paths: Paths) -> None: + paths.create_outputs() + training = plot_training(paths) + downstream = plot_downstream(paths) + plt.show() + plt.close(training) + plt.close(downstream) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("stage", choices=("validate", "prepare", "embed", "evaluate", "plot", "all")) + parser.add_argument( + "--lisbet-root", + type=Path, + default=Path.home() / "Dokumente" / "Lisbet", + help="Directory containing the LISBET repository, datasets, and ablation folder", + ) + parser.add_argument("--force", action="store_true", help="Recompute prepared data or embeddings") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--k-grid", type=int, nargs="+", default=[1, 3, 5, 11, 21]) + parser.add_argument( + "--max-train-per-class", + type=int, + default=10000, + help="Balanced cap per class for the kNN reference set; applied identically to every model", + ) + parser.add_argument("--max-validation-per-class", type=int, default=5000) + parser.add_argument("--prediction-chunk-size", type=int, default=5000) + parser.add_argument("--bootstrap-replicates", type=int, default=5000) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + paths = Paths.from_root(args.lisbet_root) + paths.create_outputs() + if args.stage in ("validate", "all"): + validate_runs(paths) + if args.stage in ("prepare", "all"): + prepare_calms21(paths, force=args.force) + if args.stage in ("embed", "all"): + compute_embeddings(paths, force=args.force) + if args.stage in ("evaluate", "all"): + evaluate(paths, args) + if args.stage in ("plot", "all"): + plot_all(paths) + + +if __name__ == "__main__": + main() diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed4.py b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed4.py new file mode 100644 index 0000000..90853e9 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed4.py @@ -0,0 +1,994 @@ +#!/usr/bin/env python3 +"""Reviewer 2.2: LISBET leave-one-task-out analysis on CalMS21. + +Stages +------ +validate Validate the five matched checkpoints and training histories. +prepare Convert CalMS21 task 1 JSON files to multi-animal DLC CSV files. +embed Compute frozen embeddings for train/test videos and all five models. +evaluate Tune a fixed kNN probe on training videos and evaluate official test videos. +plot Generate training, downstream, and combined figures. +all Run all stages in sequence. + +The downstream comparison uses the official CalMS21 task 1 train/test separation. +The k value is selected using only the all-task model and training videos, then held +fixed for every leave-one-task-out model. Confidence intervals resample test videos, +not frames. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import yaml +from matplotlib.colors import TwoSlopeNorm +from sklearn.metrics import confusion_matrix +from sklearn.model_selection import GroupKFold +from sklearn.neighbors import KNeighborsClassifier +from sklearn.preprocessing import StandardScaler + + +TASKS = ("cons", "order", "shift", "warp") +RUNS = { + "all_tasks": { + "folder": "all_tasks_seed4_75ep", + "model_id": "all_tasks_seed4_75ep", + "label": "All tasks", + "removed": None, + }, + "without_cons": { + "folder": "triple_order_shift_warp_seed4_75ep", + "model_id": "triple_order_shift_warp_seed4_75ep", + "label": "Without cons", + "removed": "cons", + }, + "without_order": { + "folder": "triple_cons_shift_warp_seed4_75ep", + "model_id": "triple_cons_shift_warp_seed4_75ep", + "label": "Without order", + "removed": "order", + }, + "without_shift": { + "folder": "triple_cons_order_warp_seed4_75ep", + "model_id": "triple_cons_order_warp_seed4_75ep", + "label": "Without shift", + "removed": "shift", + }, + "without_warp": { + "folder": "triple_cons_order_shift_seed4_75ep", + "model_id": "triple_cons_order_shift_seed4_75ep", + "label": "Without warp", + "removed": "warp", + }, +} + +RUN_COLORS = { + "all_tasks": "#202020", + "without_cons": "#4477AA", + "without_order": "#EE6677", + "without_shift": "#228833", + "without_warp": "#CCBB44", +} + + +@dataclass(frozen=True) +class Paths: + lisbet_root: Path + ablation_root: Path + calms_root: Path + work: Path + prepared: Path + embedders: Path + embeddings: Path + results: Path + figures: Path + logs: Path + + @classmethod + def from_root(cls, lisbet_root: Path) -> "Paths": + lisbet_root = lisbet_root.expanduser().resolve() + ablation_root = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "results" / "auxiliary_task_assessment" / "task_full_combinations_w200_75ep_train1600" + work = lisbet_root / "lisbet_task_ablation_w200_75ep_train1600_FULL_20260625_1624" / "reviewer_2_2_calms21_posthoc_seed4" + return cls( + lisbet_root=lisbet_root, + ablation_root=ablation_root, + calms_root=lisbet_root / "lisbet_datasets" / "datasets" / "CalMS21", + work=work, + prepared=work / "prepared_calms21_task1_dlc", + embedders=work / "exported_embedders", + embeddings=work / "embeddings", + results=work / "results", + figures=work / "figures", + logs=work / "logs", + ) + + def create_outputs(self) -> None: + for path in ( + self.work, + self.prepared, + self.embedders, + self.embeddings, + self.results, + self.figures, + self.logs, + ): + path.mkdir(parents=True, exist_ok=True) + + +def model_paths(paths: Paths, run_key: str) -> tuple[Path, Path, Path]: + run = RUNS[run_key] + base = paths.ablation_root / run["folder"] / "models" / run["model_id"] + return ( + base / "model_config.yml", + base / "weights" / "weights_last.pt", + base / "training_history" / "version_0" / "metrics.csv", + ) + + +def validate_runs(paths: Paths) -> pd.DataFrame: + """Check that encoder settings match and only the expected head is removed.""" + rows = [] + backbone_reference = None + input_reference = None + for run_key, run in RUNS.items(): + config_path, weights_path, metrics_path = model_paths(paths, run_key) + for required in (config_path, weights_path, metrics_path): + if not required.exists(): + raise FileNotFoundError(f"Missing required file: {required}") + + config = yaml.safe_load(config_path.read_text()) + metrics = pd.read_csv(metrics_path) + backbone = config.get("backbone") + input_features = config.get("input_features") + if backbone_reference is None: + backbone_reference = backbone + input_reference = input_features + if backbone != backbone_reference: + raise ValueError(f"Backbone mismatch in {run_key}") + if input_features != input_reference: + raise ValueError(f"Input-feature mismatch in {run_key}") + + observed_heads = set(config.get("out_heads", {})) + expected_heads = set(TASKS) + if run["removed"] is not None: + expected_heads.remove(run["removed"]) + if observed_heads != expected_heads: + raise ValueError( + f"Unexpected heads for {run_key}: observed={sorted(observed_heads)}, " + f"expected={sorted(expected_heads)}" + ) + if config.get("window_size") != 200 or config.get("backbone", {}).get("max_length") != 200: + raise ValueError(f"{run_key} is not a matched window-200 model") + if "epoch" not in metrics or int(metrics["epoch"].max()) != 599: + print(f"[warn] {run_key} does not contain all 75 epochs by old validation; continuing for 75ep analysis") + + row = { + "run": run_key, + "label": run["label"], + "removed_task": run["removed"] or "none", + "epochs": int(metrics["epoch"].max()) + 1, + "checkpoint_bytes": weights_path.stat().st_size, + "heads": ",".join(sorted(observed_heads)), + } + for task in TASKS: + for kind in ("score", "loss"): + col = f"{task}_train_{kind}" + row[f"final_{task}_{kind}"] = float(metrics[col].dropna().iloc[-1]) if col in metrics else np.nan + rows.append(row) + + out = pd.DataFrame(rows) + paths.create_outputs() + out.to_csv(paths.results / "validated_run_inventory.csv", index=False) + print(out.to_string(index=False)) + return out + + +def safe_name(value: str) -> str: + value = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value)).strip("_") + return value or "record" + + +def task1_json(paths: Paths, split: str) -> Path: + expected = ( + paths.calms_root + / "task1_classic_classification" + / f"calms21_task1_{split}.json" + ) + if expected.exists(): + return expected + cache_root = paths.lisbet_root / "lisbet_datasets" / "datasets" / ".cache" / "lisbet" + cached = sorted( + cache_root.glob( + "*-task1_classic_classification.zip.unzip/" + f"task1_classic_classification/calms21_task1_{split}.json" + ) + ) + if len(cached) == 1: + print(f"Using cached CalMS21 task 1 {split} data: {cached[0]}") + return cached[0] + if len(cached) > 1: + raise RuntimeError( + f"Found multiple cached CalMS21 task 1 {split} files: {cached}. " + "Remove stale cache copies or materialize the intended dataset path." + ) + raise FileNotFoundError( + f"CalMS21 task 1 {split} JSON was not found at:\n{expected}\n" + "The CalMS21 directory may be an unmaterialized link. Confirm it with " + "`find -L lisbet_datasets/datasets/CalMS21 -maxdepth 3 -type f`." + ) + + +def extract_pose_arrays(record: dict) -> tuple[np.ndarray, np.ndarray]: + """Normalize CalMS21 task 1 arrays to (frames, individuals, keypoints, axes).""" + # Task 1 JSON stores keypoints as + # (frames, individuals, coordinates, keypoints). + positions = np.asarray(record["keypoints"], dtype=np.float32).transpose((0, 1, 3, 2)) + scores = np.asarray(record["scores"], dtype=np.float32) + if positions.ndim != 4 or positions.shape[1:] != (2, 7, 2): + raise ValueError(f"Unexpected CalMS21 position shape after conversion: {positions.shape}") + # CalMS21 task 1 stores confidence as (frames, individuals, keypoints). + # Some converted variants use (frames, keypoints, individuals), so accept + # and normalize either representation for the DLC writer. + if scores.shape == (positions.shape[0], 7, 2): + scores = scores.transpose((0, 2, 1)) + if scores.shape != positions.shape[:3]: + raise ValueError(f"Position/score shape mismatch: {positions.shape}, {scores.shape}") + return positions, scores + + +def write_dlc_csv(path: Path, positions: np.ndarray, scores: np.ndarray) -> None: + individuals = ("resident", "intruder") + keypoints = ("nose", "left_ear", "right_ear", "neck", "left_hip", "right_hip", "tail") + columns = [] + values = [] + for ind_i, individual in enumerate(individuals): + for kp_i, keypoint in enumerate(keypoints): + for coord_i, coord in enumerate(("x", "y")): + columns.append(("calms21", individual, keypoint, coord)) + values.append(positions[:, ind_i, kp_i, coord_i]) + columns.append(("calms21", individual, keypoint, "likelihood")) + values.append(scores[:, ind_i, kp_i]) + frame = pd.DataFrame( + np.column_stack(values), + columns=pd.MultiIndex.from_tuples( + columns, names=("scorer", "individuals", "bodyparts", "coords") + ), + ) + frame.to_csv(path, index=True) + + +def prepare_calms21(paths: Paths, force: bool = False) -> pd.DataFrame: + """Create DLC inputs and exact frame-label tables for official task 1 splits.""" + paths.create_outputs() + manifest_path = paths.prepared / "record_manifest.csv" + if manifest_path.exists() and not force: + manifest = pd.read_csv(manifest_path) + print(f"Using existing prepared dataset: {manifest_path}") + return manifest + + rows = [] + global_vocab = None + seen_ids = set() + for split in ("train", "test"): + pose_dir = paths.prepared / split / "poses" + label_dir = paths.prepared / split / "labels" + pose_dir.mkdir(parents=True, exist_ok=True) + label_dir.mkdir(parents=True, exist_ok=True) + source_path = task1_json(paths, split) + print(f"Loading CalMS21 {split} JSON (this may take several minutes): {source_path}") + # json.load avoids holding an additional full-size text copy of these + # large (approximately 0.6-1.2 GB) source files in memory. + with source_path.open("r", encoding="utf-8") as source: + raw = json.load(source) + + for condition, condition_records in raw.items(): + for original_id, record in condition_records.items(): + stem = safe_name(f"{condition}__{original_id}") + print(f"Preparing {split} record: {stem}") + if stem in seen_ids: + raise ValueError(f"Duplicate generated record ID: {stem}") + seen_ids.add(stem) + + positions, scores = extract_pose_arrays(record) + annotations = np.asarray(record["annotations"], dtype=int) + if len(annotations) != len(positions): + raise ValueError(f"Annotation/pose length mismatch for {stem}") + + vocab_map = record.get("metadata", {}).get("vocab") + if not vocab_map: + raise ValueError(f"Missing behavior vocabulary for {stem}") + vocab = [name for name, idx in sorted(vocab_map.items(), key=lambda item: item[1])] + if global_vocab is None: + global_vocab = vocab + if vocab != global_vocab: + raise ValueError(f"Behavior vocabulary differs in {stem}: {vocab} != {global_vocab}") + if annotations.min() < 0 or annotations.max() >= len(vocab): + raise ValueError(f"Annotation IDs outside vocabulary for {stem}") + + # LISBET's DLC loader scans sequence subdirectories and accepts + # filenames matching `tracking*.csv`. + record_pose_dir = pose_dir / stem + record_pose_dir.mkdir(parents=True, exist_ok=True) + pose_path = record_pose_dir / "tracking.csv" + labels_path = label_dir / f"{stem}.csv" + if force or not pose_path.exists(): + write_dlc_csv(pose_path, positions, scores) + label_df = pd.DataFrame( + { + "record_id": stem, + "frame_idx": np.arange(len(annotations), dtype=int), + "label_id": annotations, + "label_name": [vocab[i] for i in annotations], + } + ) + label_df.to_csv(labels_path, index=False) + rows.append( + { + "split": split, + "condition": condition, + "original_id": original_id, + "record_id": stem, + "n_frames": len(annotations), + "pose_csv": str(pose_path), + "labels_csv": str(labels_path), + } + ) + + manifest = pd.DataFrame(rows).sort_values(["split", "record_id"]) + manifest.to_csv(manifest_path, index=False) + (paths.prepared / "behavior_vocabulary.json").write_text(json.dumps(global_vocab, indent=2)) + print(f"Prepared {len(manifest)} records in {paths.prepared}") + print(manifest.groupby("split")["n_frames"].agg(["count", "sum"])) + return manifest + + +def run_command(command: list[str], stdout_path: Path, stderr_path: Path) -> None: + print("Running:", " ".join(command)) + completed = subprocess.run(command, text=True, capture_output=True) + stdout_path.write_text(completed.stdout) + stderr_path.write_text(completed.stderr) + if completed.returncode != 0: + raise RuntimeError( + f"Command failed with exit code {completed.returncode}. See {stderr_path}\n" + f"Last stderr lines:\n{completed.stderr[-3000:]}" + ) + + +def find_exported_embedder(output_path: Path) -> tuple[Path, Path] | None: + configs = sorted(output_path.rglob("model_config.yml")) + if not configs: + configs = sorted(output_path.rglob("*.yml")) + sorted(output_path.rglob("*.yaml")) + weights = sorted(output_path.rglob("*.pt")) + sorted(output_path.rglob("*.pth")) + if len(configs) == 1 and len(weights) == 1: + return configs[0], weights[0] + if len(configs) == 0 and len(weights) == 0: + return None + raise ValueError( + f"Expected one exported config and one weight file under {output_path}; " + f"found configs={configs}, weights={weights}" + ) + + +def export_embedder(paths: Paths, run_key: str, force: bool = False) -> tuple[Path, Path]: + """Export the shared trained backbone with the embedding inference head.""" + output_path = paths.embedders / run_key + output_path.mkdir(parents=True, exist_ok=True) + existing = find_exported_embedder(output_path) + if existing is not None and not force: + print(f"[skip] {run_key}: using exported embedder {existing[0]}") + return existing + + source_config, source_weights, _ = model_paths(paths, run_key) + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "export_embedder", + str(source_config), + str(source_weights), + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"export_embedder_{run_key}_stdout.txt", + paths.logs / f"export_embedder_{run_key}_stderr.txt", + ) + exported = find_exported_embedder(output_path) + if exported is None: + raise FileNotFoundError(f"export_embedder produced no model files in {output_path}") + print(f"Exported {run_key} embedder: {exported[0]}, {exported[1]}") + return exported + + +def compute_embeddings(paths: Paths, force: bool = False) -> None: + """Run betman for each model and official data split.""" + validate_runs(paths) + manifest = prepare_calms21(paths, force=False) + paths.create_outputs() + for run_key in RUNS: + config_path, weights_path = export_embedder(paths, run_key, force=False) + for split in ("train", "test"): + data_path = paths.prepared / split / "poses" + output_path = paths.embeddings / run_key / split + output_path.mkdir(parents=True, exist_ok=True) + expected = list(output_path.rglob("features_lisbet_embedding.csv")) + expected_records = int((manifest["split"] == split).sum()) + if len(expected) == expected_records and not force: + print(f"[skip] {run_key}/{split}: found {len(expected)} embedding files") + continue + command = [ + sys.executable, + "-c", + "from lisbet.cli import main; main()", + "compute_embeddings", + str(data_path), + str(config_path), + str(weights_path), + "--data_format", + "maDLC", + "--window_size", + "200", + "--output_path", + str(output_path), + ] + run_command( + command, + paths.logs / f"embedding_{run_key}_{split}_stdout.txt", + paths.logs / f"embedding_{run_key}_{split}_stderr.txt", + ) + + +def embedding_columns(frame: pd.DataFrame) -> list[str]: + cols = [c for c in frame.columns if str(c).isdigit()] + if cols: + return sorted(cols, key=lambda c: int(str(c))) + excluded = {"frame_idx", "time", "index"} + cols = [ + c + for c in frame.columns + if c not in excluded + and not str(c).startswith("Unnamed") + and pd.api.types.is_numeric_dtype(frame[c]) + ] + if not cols: + raise ValueError(f"No embedding dimensions found. Columns: {frame.columns.tolist()}") + return cols + + +def index_embedding_files(paths: Paths, run_key: str, split: str, record_ids: list[str]) -> dict[str, Path]: + files = sorted((paths.embeddings / run_key / split).rglob("features_lisbet_embedding.csv")) + mapping = {} + for record_id in record_ids: + matches = [f for f in files if record_id in f.parts or record_id in str(f)] + if len(matches) != 1: + raise ValueError( + f"Expected exactly one embedding file for {run_key}/{split}/{record_id}; " + f"found {len(matches)}. Available examples: {files[:5]}" + ) + mapping[record_id] = matches[0] + return mapping + + +def load_split_embeddings( + paths: Paths, run_key: str, split: str, manifest: pd.DataFrame +) -> tuple[np.ndarray, np.ndarray, np.ndarray, list[str]]: + subset = manifest[manifest["split"] == split].sort_values("record_id") + record_ids = subset["record_id"].tolist() + file_map = index_embedding_files(paths, run_key, split, record_ids) + x_parts, y_parts, group_parts = [], [], [] + class_names = json.loads((paths.prepared / "behavior_vocabulary.json").read_text()) + + for row in subset.itertuples(index=False): + labels = pd.read_csv(row.labels_csv) + emb = pd.read_csv(file_map[row.record_id]) + cols = embedding_columns(emb) + if "frame_idx" in emb.columns: + merged = labels.merge(emb[["frame_idx"] + cols], on="frame_idx", how="inner", validate="one_to_one") + if len(merged) != len(labels): + raise ValueError( + f"Frame-index alignment lost rows for {run_key}/{row.record_id}: " + f"labels={len(labels)}, merged={len(merged)}" + ) + x = merged[cols].to_numpy(dtype=np.float32) + y = merged["label_id"].to_numpy(dtype=int) + else: + if len(emb) != len(labels): + raise ValueError( + f"Embedding/label length mismatch for {run_key}/{row.record_id}: " + f"embeddings={len(emb)}, labels={len(labels)}. No silent truncation is performed." + ) + x = emb[cols].to_numpy(dtype=np.float32) + y = labels["label_id"].to_numpy(dtype=int) + if not np.isfinite(x).all(): + raise ValueError(f"Non-finite embeddings in {run_key}/{row.record_id}") + x_parts.append(x) + y_parts.append(y) + group_parts.append(np.repeat(row.record_id, len(y))) + + return ( + np.concatenate(x_parts), + np.concatenate(y_parts), + np.concatenate(group_parts), + class_names, + ) + + +def balanced_sample(y: np.ndarray, max_per_class: int, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + selected = [] + for cls in np.unique(y): + idx = np.flatnonzero(y == cls) + if len(idx) > max_per_class: + idx = rng.choice(idx, size=max_per_class, replace=False) + selected.append(np.sort(idx)) + return np.sort(np.concatenate(selected)) + + +def metrics_from_cm(cm: np.ndarray) -> dict[str, np.ndarray | float]: + cm = np.asarray(cm, dtype=float) + tp = np.diag(cm) + support = cm.sum(axis=1) + predicted = cm.sum(axis=0) + recall = np.divide(tp, support, out=np.zeros_like(tp), where=support > 0) + precision = np.divide(tp, predicted, out=np.zeros_like(tp), where=predicted > 0) + f1 = np.divide(2 * precision * recall, precision + recall, out=np.zeros_like(tp), where=(precision + recall) > 0) + return { + "accuracy": float(tp.sum() / cm.sum()), + "balanced_accuracy": float(np.mean(recall)), + "macro_f1": float(np.mean(f1)), + "precision": precision, + "recall": recall, + "f1": f1, + "support": support, + } + + +def predict_in_chunks(clf: KNeighborsClassifier, x: np.ndarray, chunk_size: int) -> np.ndarray: + predictions = [] + for start in range(0, len(x), chunk_size): + predictions.append(clf.predict(x[start : start + chunk_size])) + return np.concatenate(predictions) + + +def tune_k_on_all_task_training( + x: np.ndarray, + y: np.ndarray, + groups: np.ndarray, + k_grid: list[int], + max_train_per_class: int, + max_val_per_class: int, + seed: int, +) -> tuple[int, pd.DataFrame]: + n_groups = len(np.unique(groups)) + if n_groups < 3: + raise ValueError("At least three training videos are required for grouped k selection") + splitter = GroupKFold(n_splits=min(5, n_groups)) + rows = [] + labels = np.arange(len(np.unique(y))) + for fold, (train_idx, val_idx) in enumerate(splitter.split(x, y, groups)): + train_keep = train_idx[balanced_sample(y[train_idx], max_train_per_class, seed + fold)] + val_keep = val_idx[balanced_sample(y[val_idx], max_val_per_class, seed + 100 + fold)] + scaler = StandardScaler() + x_train = scaler.fit_transform(x[train_keep]).astype(np.float32) + x_val = scaler.transform(x[val_keep]).astype(np.float32) + for k in k_grid: + clf = KNeighborsClassifier(n_neighbors=k, weights="distance", metric="euclidean", n_jobs=-1) + clf.fit(x_train, y[train_keep]) + pred = predict_in_chunks(clf, x_val, chunk_size=5000) + cm = confusion_matrix(y[val_keep], pred, labels=labels) + metric = metrics_from_cm(cm) + rows.append({"fold": fold, "k": k, "macro_f1": metric["macro_f1"]}) + results = pd.DataFrame(rows) + summary = results.groupby("k", as_index=False)["macro_f1"].agg(["mean", "std"]).reset_index() + best_k = int(summary.sort_values(["mean", "k"], ascending=[False, True]).iloc[0]["k"]) + print("Grouped training-only k selection:") + print(summary.to_string(index=False)) + print("Selected k:", best_k) + return best_k, results + + +def evaluate(paths: Paths, args: argparse.Namespace) -> None: + """Evaluate frozen representations on the untouched official task 1 test videos.""" + paths.create_outputs() + manifest = pd.read_csv(paths.prepared / "record_manifest.csv") + labels = None + + x_all, y_train, groups_train, class_names = load_split_embeddings( + paths, "all_tasks", "train", manifest + ) + labels = np.arange(len(class_names)) + best_k, tuning = tune_k_on_all_task_training( + x_all, + y_train, + groups_train, + args.k_grid, + args.max_train_per_class, + args.max_validation_per_class, + args.seed, + ) + tuning.to_csv(paths.results / "knn_k_selection_grouped_training.csv", index=False) + (paths.results / "selected_knn_k.json").write_text(json.dumps({"k": best_k}, indent=2)) + del x_all, y_train, groups_train + + model_cms: dict[str, dict[str, np.ndarray]] = {} + summary_rows = [] + per_class_rows = [] + cm_rows = [] + + for model_i, run_key in enumerate(RUNS): + print(f"Evaluating {RUNS[run_key]['label']}...") + x_train, y_train, _, names_train = load_split_embeddings(paths, run_key, "train", manifest) + if names_train != class_names: + raise ValueError("Class-name mismatch across models") + train_keep = balanced_sample(y_train, args.max_train_per_class, args.seed) + scaler = StandardScaler() + x_train_scaled = scaler.fit_transform(x_train[train_keep]).astype(np.float32) + clf = KNeighborsClassifier( + n_neighbors=best_k, weights="distance", metric="euclidean", n_jobs=-1 + ) + clf.fit(x_train_scaled, y_train[train_keep]) + del x_train, x_train_scaled, y_train + + x_test, y_test, groups_test, names_test = load_split_embeddings(paths, run_key, "test", manifest) + if names_test != class_names: + raise ValueError("Class-name mismatch across splits") + model_cms[run_key] = {} + for record_id in sorted(np.unique(groups_test)): + idx = np.flatnonzero(groups_test == record_id) + x_record = scaler.transform(x_test[idx]).astype(np.float32) + pred = predict_in_chunks(clf, x_record, args.prediction_chunk_size) + cm = confusion_matrix(y_test[idx], pred, labels=labels) + model_cms[run_key][record_id] = cm + for true_i in labels: + for pred_i in labels: + cm_rows.append( + { + "model": run_key, + "record_id": record_id, + "true_class": class_names[true_i], + "predicted_class": class_names[pred_i], + "count": int(cm[true_i, pred_i]), + } + ) + total_cm = sum(model_cms[run_key].values()) + metric = metrics_from_cm(total_cm) + summary_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "removed_task": RUNS[run_key]["removed"] or "none", + "k": best_k, + "n_train_probe": len(train_keep), + "n_test_frames": int(total_cm.sum()), + "accuracy": metric["accuracy"], + "balanced_accuracy": metric["balanced_accuracy"], + "macro_f1": metric["macro_f1"], + } + ) + for class_i, class_name in enumerate(class_names): + per_class_rows.append( + { + "model": run_key, + "label": RUNS[run_key]["label"], + "class_id": class_i, + "class_name": class_name, + "precision": metric["precision"][class_i], + "recall": metric["recall"][class_i], + "f1": metric["f1"][class_i], + "support": int(metric["support"][class_i]), + } + ) + del x_test, y_test, groups_test + + pd.DataFrame(cm_rows).to_csv(paths.results / "test_video_confusion_matrices.csv", index=False) + per_class = pd.DataFrame(per_class_rows) + per_class.to_csv(paths.results / "calms21_per_class_metrics.csv", index=False) + + rng = np.random.default_rng(args.seed) + test_videos = sorted(next(iter(model_cms.values())).keys()) + bootstrap_rows = [] + delta_rows = [] + for bootstrap_i in range(args.bootstrap_replicates): + draw = rng.choice(test_videos, size=len(test_videos), replace=True) + boot_metrics = {} + for run_key in RUNS: + cm = sum(model_cms[run_key][video] for video in draw) + boot_metrics[run_key] = metrics_from_cm(cm) + bootstrap_rows.append( + { + "bootstrap": bootstrap_i, + "model": run_key, + "macro_f1": boot_metrics[run_key]["macro_f1"], + "balanced_accuracy": boot_metrics[run_key]["balanced_accuracy"], + } + ) + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + delta_rows.append( + { + "bootstrap": bootstrap_i, + "removed_task": run["removed"], + "delta_macro_f1_all_minus_without": ( + boot_metrics["all_tasks"]["macro_f1"] - boot_metrics[run_key]["macro_f1"] + ), + "delta_per_class_f1_all_minus_without": ( + boot_metrics["all_tasks"]["f1"] - boot_metrics[run_key]["f1"] + ).tolist(), + } + ) + + bootstrap = pd.DataFrame(bootstrap_rows) + bootstrap.to_csv(paths.results / "test_video_bootstrap_metrics.csv", index=False) + delta_long = [] + for row in delta_rows: + for class_i, class_name in enumerate(class_names): + delta_long.append( + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "class_name": class_name, + "delta_f1_all_minus_without": row["delta_per_class_f1_all_minus_without"][class_i], + } + ) + pd.DataFrame(delta_long).to_csv(paths.results / "paired_bootstrap_per_class_task_effects.csv", index=False) + delta_overall = pd.DataFrame( + [ + { + "bootstrap": row["bootstrap"], + "removed_task": row["removed_task"], + "delta_macro_f1_all_minus_without": row["delta_macro_f1_all_minus_without"], + } + for row in delta_rows + ] + ) + delta_overall.to_csv(paths.results / "paired_bootstrap_macro_f1_task_effects.csv", index=False) + + summary = pd.DataFrame(summary_rows) + ci = ( + bootstrap.groupby("model")["macro_f1"] + .quantile([0.025, 0.975]) + .unstack() + .rename(columns={0.025: "macro_f1_ci_low", 0.975: "macro_f1_ci_high"}) + .reset_index() + ) + summary = summary.merge(ci, on="model", how="left") + summary.to_csv(paths.results / "calms21_model_summary.csv", index=False) + + contribution_rows = [] + all_pc = per_class[per_class["model"] == "all_tasks"].set_index("class_name") + for run_key, run in RUNS.items(): + if run["removed"] is None: + continue + ablated_pc = per_class[per_class["model"] == run_key].set_index("class_name") + for class_name in class_names: + contribution_rows.append( + { + "removed_task": run["removed"], + "class_name": class_name, + "delta_f1_all_minus_without": ( + all_pc.loc[class_name, "f1"] - ablated_pc.loc[class_name, "f1"] + ), + } + ) + pd.DataFrame(contribution_rows).to_csv(paths.results / "per_class_task_contribution.csv", index=False) + print(summary.to_string(index=False)) + + +def set_plot_style() -> None: + mpl.rcParams.update( + { + "font.family": "sans-serif", + "font.sans-serif": ["DejaVu Sans"], + "font.size": 8, + "axes.titlesize": 9, + "axes.labelsize": 8, + "xtick.labelsize": 7, + "ytick.labelsize": 7, + "legend.fontsize": 7, + "axes.linewidth": 0.7, + "pdf.fonttype": 42, + "ps.fonttype": 42, + "svg.fonttype": "none", + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.facecolor": "white", + "axes.spines.top": False, + "axes.spines.right": False, + } + ) + + +def panel_label(ax: plt.Axes, label: str, x: float = -0.14, y: float = 1.08) -> None: + ax.text(x, y, label, transform=ax.transAxes, ha="left", va="top", fontsize=12, fontweight="normal") + + +def save_figure(fig: plt.Figure, paths: Paths, stem: str) -> None: + for suffix in ("pdf", "png", "svg"): + kwargs = {"dpi": 600} if suffix == "png" else {} + fig.savefig(paths.figures / f"{stem}.{suffix}", bbox_inches="tight", **kwargs) + print("Saved figure:", paths.figures / stem) + + +def plot_training(paths: Paths) -> plt.Figure: + """Plot raw and lightly smoothed trajectories; no false replicate bands.""" + set_plot_style() + histories = {} + for run_key in RUNS: + _, _, metrics_path = model_paths(paths, run_key) + histories[run_key] = pd.read_csv(metrics_path) + + fig, axes = plt.subplots(4, 2, figsize=(8.2, 8.8), sharex=True) + for task_i, task in enumerate(TASKS): + for col_i, kind in enumerate(("score", "loss")): + ax = axes[task_i, col_i] + metric_col = f"{task}_train_{kind}" + for run_key, history in histories.items(): + if metric_col not in history: + continue + x = history["epoch"].to_numpy() + raw = history[metric_col].to_numpy() + smooth = pd.Series(raw).rolling(15, center=True, min_periods=1).mean().to_numpy() + ax.plot(x, raw, color=RUN_COLORS[run_key], alpha=0.16, linewidth=0.45) + ax.plot(x, smooth, color=RUN_COLORS[run_key], linewidth=1.25, label=RUNS[run_key]["label"]) + ax.set_title(task) + if col_i == 0: + ax.set_ylabel("Training score") + ax.set_ylim(0.4, 1.02) + else: + ax.set_ylabel("Training loss") + ax.set_ylim(bottom=0) + if task_i == len(TASKS) - 1: + ax.set_xlabel("Epoch") + panel_label(axes[0, 0], "a") + panel_label(axes[0, 1], "b") + handles = [ + mpl.lines.Line2D([0], [0], color=RUN_COLORS[k], lw=1.5, label=RUNS[k]["label"]) + for k in RUNS + ] + fig.legend(handles=handles, loc="lower center", ncol=5, frameon=False, bbox_to_anchor=(0.5, 0.005)) + fig.subplots_adjust(left=0.10, right=0.98, top=0.97, bottom=0.08, hspace=0.42, wspace=0.28) + save_figure(fig, paths, "Fig_training_curves_75ep_leave_one_task_out") + return fig + + +def annotate_heatmap(ax: plt.Axes, values: np.ndarray, fmt: str, threshold: float | None = None) -> None: + if threshold is None: + threshold = float(np.nanmedian(values)) + for i in range(values.shape[0]): + for j in range(values.shape[1]): + value = values[i, j] + ax.text(j, i, format(value, fmt), ha="center", va="center", fontsize=7, color="white" if value < threshold else "black") + + +def plot_downstream(paths: Paths) -> plt.Figure: + set_plot_style() + summary = pd.read_csv(paths.results / "calms21_model_summary.csv") + per_class = pd.read_csv(paths.results / "calms21_per_class_metrics.csv") + contribution = pd.read_csv(paths.results / "per_class_task_contribution.csv") + model_order = list(RUNS) + model_labels = [RUNS[k]["label"] for k in model_order] + class_order = per_class.sort_values("class_id")["class_name"].drop_duplicates().tolist() + + absolute = ( + per_class.pivot(index="class_name", columns="model", values="f1") + .loc[class_order, model_order] + ) + effect = ( + contribution.pivot(index="class_name", columns="removed_task", values="delta_f1_all_minus_without") + .loc[class_order, list(TASKS)] + ) + + fig = plt.figure(figsize=(11.2, 3.65)) + gs = fig.add_gridspec(1, 3, width_ratios=(1.35, 1.15, 1.0), wspace=0.48) + ax0 = fig.add_subplot(gs[0, 0]) + im0 = ax0.imshow(absolute.values, cmap="viridis", vmin=0, vmax=1, aspect="auto") + ax0.set_xticks(range(len(model_order)), model_labels, rotation=38, ha="right") + ax0.set_yticks(range(len(class_order)), class_order) + ax0.set_xlabel("Frozen LISBET encoder") + ax0.set_ylabel("CalMS21 behavior") + ax0.set_title("Per-behavior decoding") + annotate_heatmap(ax0, absolute.values, ".2f", threshold=0.55) + fig.colorbar(im0, ax=ax0, fraction=0.046, pad=0.03, label="Test F1") + panel_label(ax0, "c") + + ax1 = fig.add_subplot(gs[0, 1]) + vmax = max(0.01, float(np.nanmax(np.abs(effect.values)))) + norm = TwoSlopeNorm(vmin=-vmax, vcenter=0, vmax=vmax) + im1 = ax1.imshow(effect.values, cmap="coolwarm", norm=norm, aspect="auto") + ax1.set_xticks(range(len(TASKS)), TASKS) + ax1.set_yticks(range(len(class_order)), class_order) + ax1.set_xlabel("Removed task") + ax1.set_ylabel("CalMS21 behavior") + ax1.set_title("Task-removal effect") + for i in range(effect.shape[0]): + for j in range(effect.shape[1]): + ax1.text(j, i, f"{effect.values[i, j]:+.3f}", ha="center", va="center", fontsize=7) + fig.colorbar(im1, ax=ax1, fraction=0.046, pad=0.03, label=r"$F1_{all}-F1_{without}$") + panel_label(ax1, "d") + + ax2 = fig.add_subplot(gs[0, 2]) + summary = summary.set_index("model").loc[model_order].reset_index() + x = np.arange(len(summary)) + y = summary["macro_f1"].to_numpy() + yerr = np.vstack( + [ + y - summary["macro_f1_ci_low"].to_numpy(), + summary["macro_f1_ci_high"].to_numpy() - y, + ] + ) + ax2.errorbar(x, y, yerr=yerr, fmt="o", color="#202020", ecolor="#555555", capsize=3, linewidth=1.1) + ax2.set_xticks(x, model_labels, rotation=38, ha="right") + ax2.set_ylabel("Test macro-F1") + ax2.set_xlabel("Frozen LISBET encoder") + ax2.set_ylim(max(0, float(np.nanmin(yerr[0] * -1 + y)) - 0.05), min(1, float(np.nanmax(yerr[1] + y)) + 0.05)) + ax2.set_title("Overall behavioral decoding") + panel_label(ax2, "e") + + fig.subplots_adjust(left=0.07, right=0.98, top=0.92, bottom=0.28) + save_figure(fig, paths, "Fig_calms21_knn_task_ablation_posthoc") + return fig + + +def plot_all(paths: Paths) -> None: + paths.create_outputs() + training = plot_training(paths) + downstream = plot_downstream(paths) + plt.show() + plt.close(training) + plt.close(downstream) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("stage", choices=("validate", "prepare", "embed", "evaluate", "plot", "all")) + parser.add_argument( + "--lisbet-root", + type=Path, + default=Path.home() / "Dokumente" / "Lisbet", + help="Directory containing the LISBET repository, datasets, and ablation folder", + ) + parser.add_argument("--force", action="store_true", help="Recompute prepared data or embeddings") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--k-grid", type=int, nargs="+", default=[1, 3, 5, 11, 21]) + parser.add_argument( + "--max-train-per-class", + type=int, + default=10000, + help="Balanced cap per class for the kNN reference set; applied identically to every model", + ) + parser.add_argument("--max-validation-per-class", type=int, default=5000) + parser.add_argument("--prediction-chunk-size", type=int, default=5000) + parser.add_argument("--bootstrap-replicates", type=int, default=5000) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + paths = Paths.from_root(args.lisbet_root) + paths.create_outputs() + if args.stage in ("validate", "all"): + validate_runs(paths) + if args.stage in ("prepare", "all"): + prepare_calms21(paths, force=args.force) + if args.stage in ("embed", "all"): + compute_embeddings(paths, force=args.force) + if args.stage in ("evaluate", "all"): + evaluate(paths, args) + if args.stage in ("plot", "all"): + plot_all(paths) + + +if __name__ == "__main__": + main() diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/metadata/package_checksums.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/metadata/package_checksums.csv new file mode 100644 index 0000000..1f8c281 --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/metadata/package_checksums.csv @@ -0,0 +1,35 @@ +relative_path,size_bytes,sha256 +.gitignore,95,14a2c7f1b277c1433387f169d4923ab8983ee1bbdbe0665c1be84e64d5d4a66f +PROVENANCE.md,1476,d806329ccb76d88ed5244e528b4b9c3c721cd8bc61b93eabadb29a1c8fae42a3 +README.md,2245,e66c46273631f5a3d877815fce12d35ecbdbc9e0b4809af8e68f644ce0a390f1 +annotator_bout_metrics/figures/combined_annotator_bias_summary_shared_legend.csv,383,aea1c77f12f5b9ac36a1443caf479c85a73f697deeeb124600ab8abbc09957ee +annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.pdf,29194,f46c17792ef89ebbd26a9912a06c0cb2509c88f1f15cadc7c48bc6d6582b0246 +annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.png,809126,c51ede18fd45999ac46a991ce64e511377d91ac7c1941583bffa512311e97716 +annotator_bout_metrics/figures/combined_calms21_mars1_annotator_bias_shared_legend.svg,148462,187bf11582dd002fd39323782d80c774fcba580196437727adc47e2722f80c3b +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_annotator_file_summary.csv,239,074da5d50a2c74a47ae102c0859cc9760bfe01f9bb6dbc787d9701f1687fd708 +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_by_annotator.csv,1604,46156c5ba102be162527598b997a8ef1a3eb82492f8941e83459d526178c8be9 +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_bout_stats_long.csv,68488,fcf6d451a5bbbcd553a3b901c12a43936b08547b108d105e8d479b1f9b50f4c7 +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_by_annotator.csv,951,ccbc5cd593c2ee3b22f053ff6ef950e2202660947bb2b84704ad16675e9cc8a5 +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_label_distribution_long.csv,57274,975a9860795871454b586fd6bb8ff6fcf482e4cdb3d9ea4753ccd09e68d12752 +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_manifest.csv,23035,d7c9ce9318cbc2b18f8acaa0b04b8df0e7ed2730edb9b424717fe3dfdb9808e1 +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_agreement.csv,7383,f0f5f7ce90f9d34392c78a18acd1b1154f6342376e4b3481847a1509c774f551 +annotator_bout_metrics/results/calms21_task2_annotator_bias/calms21_task2_pairwise_interannotator_summary.csv,1851,4506a82389f1f16febca7e274cf01c57a74c28cceceab75b7f928fb339634e9a +annotator_bout_metrics/results/mars1_annotator_bias/mars1_annotator_file_summary.csv,372,02e0a7b01c08ca997d6ca23ddb1b4109104dcdb83a883b432c71802ccc49e740 +annotator_bout_metrics/results/mars1_annotator_bias/mars1_behavior_positive_fraction_by_annotator.csv,1339,bc1226a51972cf1cce559c218fd616955ac3a36e101de24d872292a984279325 +annotator_bout_metrics/results/mars1_annotator_bias/mars1_bout_stats_by_annotator.csv,1999,91205c379f3ad56baaa349b25dbe851a6ea9e1548302b88548d399908cbd0c44 +annotator_bout_metrics/results/mars1_annotator_bias/mars1_label_distribution_by_annotator.csv,1683,eab6ce948b7061827a249f06ebf6ce0d0ff0cd9bd6397158bd6e0ac8cbe9c3de +annotator_bout_metrics/results/mars1_annotator_bias/mars1_manifest.csv,18626,dab6898c5287c6cbd8181f2bd2ec03806a0a4c029b74d09394c39a638de0225c +annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_agreement_by_behavior.csv,215488,1b403e8c9ceca82939986203a619cbd2e63c7f7cc5ba7643f65eea194a2ca492 +annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_boutwise_summary.csv,3097,03afa9286b735e7fb88c54beb9abcf9c6ee4666fe2518ea8c6aec95e85a904c8 +annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_agreement.csv,81938,f902432089680171c8151f6db27036aec13f4b194371da9d2adc1f300392a26d +annotator_bout_metrics/results/mars1_annotator_bias/mars1_pairwise_interannotator_summary.csv,4764,af7b8290ac477bf3418d832f47ddcfc77920017df4424aba7824f62f568b5ce5 +annotator_bout_metrics/scripts/make_composite_calms21_task2_annotator_figure.py,4152,453fc57984df9de471b408a3b03c4e46dcb05c37523a4881f03127f33a5d62d8 +annotator_bout_metrics/scripts/plot_calms21_task2_annotator_bias.py,5011,1f552b30e94651a587eb8c52dfdf2f5810319dfbbaab5388b7cfb2c498335eb3 +annotator_bout_metrics/scripts/plot_combined_calms21_mars1_annotator_bias.py,17229,7493b434d06904613ee0afbf76f3c735edc4d0c13777247efffafe3a37938d45 +calms21_leave_one_task_out/scripts/plot_calms21_posthoc_allseeds.py,8331,74a134a68b09ffbdd63306ed76d3c7cc1b20c078ec36848f9be656b2b09438b9 +calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed0.py,41037,675dafc5f012504f0706c7a83ec7fd95941223f902e649e311820f318b6e78c9 +calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed1.py,41037,905cf2a7072e39a75458a1221a138d16bfc7ea4044cd0ca739eb044356deec73 +calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed2.py,41037,07aabea981ea584798f8128864168dbca407c38595722d72892d4c543ed44423 +calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed3.py,41037,3d8722cad7e8a980db1d08b6db5f26164846fca5b9a15ac25157753067fd08ea +calms21_leave_one_task_out/scripts/reviewer_2_2_calms21_pipeline_seed4.py,41037,d9e6fe18bdab4d84fd26e682e573dafa923765dd7e91529ab6134220ce122f31 +metadata/source_checksums.csv,867,28c15eae3d0ddbd499a6c7130e497af5f4c5fb46fbecfa2c705b06fd0b2d473d diff --git a/paper/nature_neuroscience_revision/auxiliary_task_assessment/metadata/source_checksums.csv b/paper/nature_neuroscience_revision/auxiliary_task_assessment/metadata/source_checksums.csv new file mode 100644 index 0000000..91fddfa --- /dev/null +++ b/paper/nature_neuroscience_revision/auxiliary_task_assessment/metadata/source_checksums.csv @@ -0,0 +1,8 @@ +source_item,use,sha256 +reviewer_2_2_calms21_pipeline_seed0.py,exact_copy,675dafc5f012504f0706c7a83ec7fd95941223f902e649e311820f318b6e78c9 +reviewer_2_2_calms21_pipeline_seed1.py,exact_copy,905cf2a7072e39a75458a1221a138d16bfc7ea4044cd0ca739eb044356deec73 +reviewer_2_2_calms21_pipeline_seed2.py,exact_copy,07aabea981ea584798f8128864168dbca407c38595722d72892d4c543ed44423 +reviewer_2_2_calms21_pipeline_seed3.py,exact_copy,3d8722cad7e8a980db1d08b6db5f26164846fca5b9a15ac25157753067fd08ea +reviewer_2_2_calms21_pipeline_seed4.py,exact_copy,d9e6fe18bdab4d84fd26e682e573dafa923765dd7e91529ab6134220ce122f31 +plot_calms21_posthoc_allseeds.py,path_only_portable_adaptation,d7bc0fd58ed6e3d075c8c8111885bc172113213e3c4bf4e39675dc1074a41542 +annotator-bias.ipynb final executed cell,portable_script_extraction,72a1de49db767401585581b67990a3765e6b0188171c38f261c730f03373fb33