{ "schemaVersion": "kiro.layout-ab-results.v1", "cohort": { "documents": 31, "pages": 142, "rawLayoutGroundTruthDocuments": 5 }, "recommendation": "현재 Heron 유지", "winner": "heron", "profile": { "id": "kiro-dpa-v2-connected-20261103-v2", "label": "DPA v2 · 고정 평가 프로필", "areaWeight": 1, "missingPolicy": "GT-positive omission is zero; absent/unsupported GT is N/A and excluded, with explicit status. No zero/100 imputation.", "scope": "Per-input DPA averages available area scores only with at least three measured areas. Overall retains official native aggregation, averages declared native/supplemental protocol cohorts equally within dataset/metric, then datasets within each area, then measured areas. Per-page diagnostics never enter the overall score.", "areas": [ { "id": "flow", "label": "읽기 흐름", "metrics": [ "reading_order" ] }, { "id": "content", "label": "문자 내용", "metrics": [ "text" ] }, { "id": "structure", "label": "문서 구조·목차", "metrics": [ "heading", "layout" ] }, { "id": "media", "label": "그림·표·수식", "metrics": [ "table", "figure", "formula" ] }, { "id": "typography", "label": "글자 표현", "metrics": [ "styling" ] }, { "id": "notes", "label": "주석·참조", "metrics": [ "notes" ] }, { "id": "running", "label": "반복 요소·원본 페이지", "metrics": [ "running_matter", "source_page" ] } ], "notes": [ "서식 종합은 제목을 포함하므로 DPA의 글자 표현에는 공식 글자 스타일 하위 점수를 사용하고 제목은 구조 영역에서 계산한다.", "병합·연결·표 구조만 지표는 진단용으로 보존하고 이번 대표 프로필에는 중복 합산하지 않는다.", "샘플마다 측정 가능한 영역이 다르다. DPA 옆의 영역 수와 메트릭을 함께 비교한다.", "표 구조 0점은 셀 구조 복원에 대한 점수다. EPUB에 표 이미지가 보존되었는지와 독자에게 읽히는지는 별도 증거로 확인한다.", "머리말·꼬리말·쪽번호의 본문 잔류는 공개 GT 라벨을 ParseBench 공식 leak 축에 연결했다. 원본 쪽 탐색은 명시적인 인쇄 번호가 있는 쪽의 관계 F1이다.", "주석 표식↔본문↔복귀 관계는 이 묶음에 연결 정답이 없다. 제거·원본쪽 영역과 구분한다.", "단일·제한 과제 표본은 DPA를 발행하지 않는다. 샘플 DPA는 최소 3개 영역이 측정된 경우만 제공하며, 개별 메트릭은 항상 보존한다.", "레이아웃/표 GT에 있는 정보를 연결한 보충 평가를 원래 벤치마크 결과와 구분한다. 표 전용 GT의 문자 점수 범위는 표 셀 문자다.", "제목 인식과 계층은 구분한다. is_title은 굵은 제목도 허용하는 공식 규칙이며 수준/부모 관계를 채점하지 않는다. 계층 정답을 임의 생성하지 않는다." ], "normalization": "Official raw metrics are preserved. Only quality inputs to DPA are bounded to 0..100, including negative official TEDS values.", "minimumSampleAreas": 3 }, "models": [ { "key": "heron", "name": "Heron · 현재", "model": { "key": "heron", "id": "docling-project/docling-layout-heron", "revision": "8f39ad3c0b4c58e9c2d2c84a38465abf757272d8", "localPath": "/Volumes/GitHubData/kiro-python/experiments/ebook-structure-20261103/huggingface/hub/models--docling-project--docling-layout-heron/snapshots/8f39ad3c0b4c58e9c2d2c84a38465abf757272d8" }, "rawLayout": { "avg_mAP@[.50:.95]": 0.6811955257997124, "min_mAP@[.50:.95]": 0.5741470844791683, "max_mAP@[.50:.95]": 0.75, "avg_AP50": 0.7186622765090599, "min_AP50": 0.6746468711358418, "max_AP50": 0.7519412878787879, "avg_AP75": 0.6762364549425145, "min_AP75": 0.5672220731974404, "max_AP75": 0.75, "avg_f1_Page-header": 0, "min_f1_Page-header": 0, "max_f1_Page-header": 0, "avg_precision_Page-header": 0, "min_precision_Page-header": 0, "max_precision_Page-header": 0, "avg_recall_Page-header": 0, "min_recall_Page-header": 0, "max_recall_Page-header": 0, "avg_f1_Picture": 0.4666666666666667, "min_f1_Picture": 0, "max_f1_Picture": 1, "avg_precision_Picture": 0.40073260073260075, "min_precision_Picture": 0, "max_precision_Picture": 1, "avg_recall_Picture": 0.6666666666666667, "min_recall_Picture": 0, "max_recall_Picture": 1, "avg_f1_Section": 0.5023361823361824, "min_f1_Section": 0.19999999999999998, "max_f1_Section": 1, "avg_precision_Section": 0.3974747474747475, "min_precision_Section": 0.1111111111111111, "max_precision_Section": 1, "avg_recall_Section": 0.96, "min_recall_Section": 0.8, "max_recall_Section": 1, "avg_f1_Text": 0.5170780205388299, "min_f1_Text": 0.21621621621621623, "max_f1_Text": 1, "avg_precision_Text": 0.4120596946912737, "min_precision_Text": 0.12631578947368421, "max_precision_Text": 1, "avg_recall_Text": 0.8738888888888889, "min_recall_Text": 0.75, "max_recall_Text": 1, "avg_mean_f1": 0.4640658616212048, "min_mean_f1": 0.3095785440613027, "max_mean_f1": 0.75, "avg_num_predictions": 83, "min_num_predictions": 26, "max_num_predictions": 124, "avg_num_ground_truth": 24.8, "min_num_ground_truth": 24, "max_num_ground_truth": 25, "avg_layout_localization_pass_rate": 0.9356666666666668, "min_layout_localization_pass_rate": 0.8, "max_layout_localization_pass_rate": 1, "avg_layout_classification_pass_rate": 0.8073333333333335, "min_layout_classification_pass_rate": 0.6, "max_layout_classification_pass_rate": 1, "avg_layout_attribution_pass_rate": 0, "min_layout_attribution_pass_rate": 0, "max_layout_attribution_pass_rate": 0, "avg_layout_rule_pass_rate": 0.6093333214807439, "min_layout_rule_pass_rate": 0.4794520547945205, "max_layout_rule_pass_rate": 0.684931506849315, "avg_rule_pass_rate": 0.6093333214807439, "min_rule_pass_rate": 0.4794520547945205, "max_rule_pass_rate": 0.684931506849315, "avg_layout_element_rule_pass_rate": 0.08966666666666667, "min_layout_element_rule_pass_rate": 0, "max_layout_element_rule_pass_rate": 0.20833333333333334, "avg_lap": 1, "min_lap": 1, "max_lap": 1, "avg_lar": 0, "min_lar": 0, "max_lar": 0, "avg_af1": 0, "min_af1": 0, "max_af1": 0, "avg_unmatched_gt_elements": 1.6, "min_unmatched_gt_elements": 0, "max_unmatched_gt_elements": 5, "avg_unmatched_pred_elements": 31, "min_unmatched_pred_elements": 0, "max_unmatched_pred_elements": 82, "avg_f1_Page-footer": 0.40740740740740744, "min_f1_Page-footer": 0, "max_f1_Page-footer": 1, "avg_precision_Page-footer": 0.375, "min_precision_Page-footer": 0, "max_precision_Page-footer": 1, "avg_recall_Page-footer": 0.6666666666666666, "min_recall_Page-footer": 0, "max_recall_Page-footer": 1, "avg_f1_Table": 1, "min_f1_Table": 1, "max_f1_Table": 1, "avg_precision_Table": 1, "min_precision_Table": 1, "max_precision_Table": 1, "avg_recall_Table": 1, "min_recall_Table": 1, "max_recall_Table": 1, "total_layout_localization_pass_rate_passed": 116, "total_layout_localization_pass_rate_evaluated": 124, "micro_layout_localization_pass_rate": 0.9354838709677419, "total_layout_classification_pass_rate_passed": 100, "total_layout_classification_pass_rate_evaluated": 124, "micro_layout_classification_pass_rate": 0.8064516129032258, "total_layout_attribution_pass_rate_passed": 0, "total_layout_attribution_pass_rate_evaluated": 107, "micro_layout_attribution_pass_rate": 0, "total_layout_rule_pass_rate_passed": 216, "total_layout_rule_pass_rate_evaluated": 355, "micro_layout_rule_pass_rate": 0.6084507042253521, "total_rule_pass_rate_passed": 216, "total_rule_pass_rate_evaluated": 355, "micro_rule_pass_rate": 0.6084507042253521, "total_layout_element_rule_pass_rate_passed": 11, "total_layout_element_rule_pass_rate_evaluated": 124, "micro_layout_element_rule_pass_rate": 0.08870967741935484, "total_unmatched_gt_elements": 8, "total_unmatched_pred_elements": 155 }, "runtime": { "startupSeconds": 3.72526358399773, "warmMedianSeconds": 0.27773204195545986, "warmMeanSeconds": 0.300242024531228, "peakRssBytes": 1696235520, "processorSize": "SizeDict(height=640, width=640, longest_edge=None, shortest_edge=None, max_height=None, max_width=None, min_pixels=None, max_pixels=None)" }, "overall": { "score": 59.30293445845636, "measuredAreas": 6, "totalAreas": 7, "areas": [ { "id": "flow", "label": "읽기 흐름", "score": 73.57406045048964, "datasets": [ { "dataset": "marathon", "score": 69.36274509803921 }, { "dataset": "mdp", "score": 88.57142857142857 }, { "dataset": "parsebench", "score": 62.788007682001115 } ] }, { "id": "content", "label": "문자 내용", "score": 86.38417368843626, "datasets": [ { "dataset": "marathon", "score": 86.00538901987201 }, { "dataset": "mdp", "score": 91.91674734033208 }, { "dataset": "parsebench", "score": 81.23038470510468 } ] }, { "id": "structure", "label": "문서 구조·목차", "score": 44.92035790579114, "datasets": [ { "dataset": "marathon", "score": 22.612264193563902 }, { "dataset": "mdp", "score": 50 }, { "dataset": "parsebench", "score": 62.148809523809526 } ] }, { "id": "media", "label": "그림·표·수식", "score": 47.71675743774619, "datasets": [ { "dataset": "marathon", "score": 49.52606648595966 }, { "dataset": "mdp", "score": 67.91806656028115 }, { "dataset": "parsebench", "score": 25.706139266997756 } ] }, { "id": "typography", "label": "글자 표현", "score": 46.25506337732217, "datasets": [ { "dataset": "parsebench", "score": 46.25506337732217 } ] }, { "id": "notes", "label": "주석·참조", "score": null, "datasets": [] }, { "id": "running", "label": "반복 요소·원본 페이지", "score": 56.96719389095276, "datasets": [ { "dataset": "marathon", "score": 96.79443881571541 }, { "dataset": "mdp", "score": 43.75 }, { "dataset": "parsebench", "score": 30.357142857142858 } ] } ], "fullSevenAreaBounds": [ 50.831086678676876, 65.11680096439116 ], "profileId": "kiro-dpa-v2-connected-20261103-v2", "aggregation": "공식 원래 집계 보존 → 데이터셋·메트릭별 원래/보충 프로토콜 균등 평균 → 영역별 데이터셋 균등 평균 → 측정 영역 균등 평균" }, "dpaDelta": 0, "scoreSha256": "11c8af259b0fc64bafdaeab0bb0fc5f696f1ee24b2ca3f8773930c2de274923d", "sampleChanges": [ { "id": "marathon-mp3lan_0638", "label": "MA86E/L104 기술 설명서 · 연속 110쪽", "previous": 64.86018072263003, "current": 64.86018072263003, "delta": 0 }, { "id": "parsebench-575ef6c72ec5", "label": "2007.03579v2_p5.pdf", "previous": 78.98674351585015, "current": 78.98674351585015, "delta": 0 }, { "id": "parsebench-44474d2f099f", "label": "20190601_ECR_UKPB_081_p1.pdf", "previous": 62.17073418751271, "current": 62.17073418751271, "delta": 0 }, { "id": "parsebench-ce9d90e069d1", "label": "2050247507.jpg", "previous": 49.50824372759857, "current": 49.50824372759857, "delta": 0 }, { "id": "parsebench-6e927d5e9b85", "label": "delltechnologies-fy24-esg-report_p6.pdf", "previous": 61.93705714645088, "current": 61.93705714645088, "delta": 0 }, { "id": "parsebench-b895f53f5325", "label": "maersk-annual-report-2024_p8.pdf", "previous": 56.03091005104735, "current": 56.03091005104735, "delta": 0 }, { "id": "parsebench-fc437565b684", "label": "text_dense__canara.pdf", "previous": 67.2227952784727, "current": 67.2227952784727, "delta": 0 }, { "id": "parsebench-a6ac79e04e6c", "label": "text_dense__de.pdf", "previous": 33.082492186688, "current": 33.082492186688, "delta": 0 }, { "id": "parsebench-7ebefff650eb", "label": "text_misc__edit2.pdf", "previous": 63.98186941057306, "current": 63.98186941057306, "delta": 0 }, { "id": "parsebench-5dd32f0c5770", "label": "text_misc__marks.pdf", "previous": 27.040816326530614, "current": 27.040816326530614, "delta": 0 }, { "id": "parsebench-5847401fad94", "label": "text_multicolumns__10k2col.pdf", "previous": 27.644699272358846, "current": 27.644699272358846, "delta": 0 }, { "id": "parsebench-08cc5c475326", "label": "text_multicolumns__14999.pdf", "previous": 87.34619017427471, "current": 87.34619017427471, "delta": 0 }, { "id": "parsebench-eb2521cc7842", "label": "text_multicolumns__paperchem.pdf", "previous": 50.0727345047103, "current": 50.0727345047103, "delta": 0 }, { "id": "parsebench-45cdc3d8e1a0", "label": "text_ocr__012-25.pdf", "previous": 74.86848282130498, "current": 74.86848282130498, "delta": 0 }, { "id": "parsebench-18bee151fa52", "label": "text_ocr__069.pdf", "previous": 34.68936352687186, "current": 34.68936352687186, "delta": 0 }, { "id": "parsebench-16807829a472", "label": "text_simple__marknotTable.pdf", "previous": 87.9080459770115, "current": 87.9080459770115, "delta": 0 }, { "id": "mdp-612e15095482", "label": "ko_academicpaper_39.jpg", "previous": 79.41197036817664, "current": 79.41197036817664, "delta": 0 }, { "id": "mdp-08b85b25b6df", "label": "ko_academicpaper_40.jpg", "previous": 100, "current": 100, "delta": 0 }, { "id": "mdp-54fb82b1726b", "label": "ko_book_1.jpg", "previous": 87.4378928916828, "current": 87.4378928916828, "delta": 0 }, { "id": "mdp-74ccbbcee6c4", "label": "ko_book_2.jpg", "previous": 42.04428483139357, "current": 42.04428483139357, "delta": 0 }, { "id": "mdp-9059e2e92a9a", "label": "ko_exampaper_24.jpg", "previous": 86.59003831417624, "current": 86.59003831417624, "delta": 0 }, { "id": "mdp-6213848de2f2", "label": "ko_financialreport_12.jpg", "previous": 88.37153155465084, "current": 88.37153155465084, "delta": 0 }, { "id": "mdp-bb635290130a", "label": "ko_magazine_30.jpg", "previous": 75.45372256166554, "current": 75.45372256166554, "delta": 0 }, { "id": "mdp-b306e62fcf08", "label": "ko_newspaper_43.jpg", "previous": 34.4767252929982, "current": 34.4767252929982, "delta": 0 }, { "id": "mdp-53b6614478ad", "label": "ko_slide_10.jpg", "previous": 65.993265993266, "current": 65.993265993266, "delta": 0 }, { "id": "mdp-80d11cefe9f9", "label": "ko_textbook_17.jpg", "previous": 74.68051118210863, "current": 74.68051118210863, "delta": 0 } ] }, { "key": "pp-doclayout-v3", "name": "PP-DocLayoutV3", "model": { "key": "pp-doclayout-v3", "id": "PaddlePaddle/PP-DocLayoutV3_safetensors", "revision": "e4f489dc536556fc1be02b973ba06756e0d4a2ac", "localPath": "/Volumes/GitHubData/kiro-python/experiments/ebook-structure-20261103/huggingface/hub/models--PaddlePaddle--PP-DocLayoutV3_safetensors/snapshots/e4f489dc536556fc1be02b973ba06756e0d4a2ac" }, "rawLayout": { "avg_mAP@[.50:.95]": 0.5371750211435526, "min_mAP@[.50:.95]": 0.21112208624708625, "max_mAP@[.50:.95]": 0.719863011988012, "avg_AP50": 0.6586331988585578, "min_AP50": 0.22846736596736597, "max_AP50": 0.9279895104895104, "avg_AP75": 0.6325713591963592, "min_AP75": 0.22846736596736597, "max_AP75": 0.9740530303030304, "avg_f1_Page-header": 0.3333333333333333, "min_f1_Page-header": 0, "max_f1_Page-header": 0.6666666666666666, "avg_precision_Page-header": 0.25, "min_precision_Page-header": 0, "max_precision_Page-header": 0.5, "avg_recall_Page-header": 0.5, "min_recall_Page-header": 0, "max_recall_Page-header": 1, "avg_f1_Picture": 0.4809523809523809, "min_f1_Picture": 0, "max_f1_Picture": 1, "avg_precision_Picture": 0.55, "min_precision_Picture": 0, "max_precision_Picture": 1, "avg_recall_Picture": 0.48, "min_recall_Picture": 0, "max_recall_Picture": 1, "avg_f1_Section": 0.4444444444444445, "min_f1_Section": 0, "max_f1_Section": 0.8571428571428571, "avg_precision_Section": 0.475, "min_precision_Section": 0, "max_precision_Section": 1, "avg_recall_Section": 0.5900000000000001, "min_recall_Section": 0, "max_recall_Section": 1, "avg_f1_Text": 0.7123232323232324, "min_f1_Text": 0.5555555555555556, "max_f1_Text": 1, "avg_precision_Text": 0.7256302521008403, "min_precision_Text": 0.47619047619047616, "max_precision_Text": 1, "avg_recall_Text": 0.7180555555555556, "min_recall_Text": 0.6111111111111112, "max_recall_Text": 1, "avg_mean_f1": 0.5043506493506493, "min_mean_f1": 0.16666666666666669, "max_mean_f1": 0.7142857142857143, "avg_num_predictions": 26, "min_num_predictions": 19, "max_num_predictions": 31, "avg_num_ground_truth": 24.8, "min_num_ground_truth": 24, "max_num_ground_truth": 25, "avg_layout_localization_pass_rate": 0.807, "min_layout_localization_pass_rate": 0.68, "max_layout_localization_pass_rate": 1, "avg_layout_classification_pass_rate": 0.6283333333333333, "min_layout_classification_pass_rate": 0.44, "max_layout_classification_pass_rate": 1, "avg_layout_attribution_pass_rate": 0, "min_layout_attribution_pass_rate": 0, "max_layout_attribution_pass_rate": 0, "avg_layout_rule_pass_rate": 0.5006190014786105, "min_layout_rule_pass_rate": 0.42028985507246375, "max_layout_rule_pass_rate": 0.684931506849315, "avg_rule_pass_rate": 0.5006190014786105, "min_rule_pass_rate": 0.42028985507246375, "max_rule_pass_rate": 0.684931506849315, "avg_layout_element_rule_pass_rate": 0.04866666666666667, "min_layout_element_rule_pass_rate": 0, "max_layout_element_rule_pass_rate": 0.08333333333333333, "avg_lap": 1, "min_lap": 1, "max_lap": 1, "avg_lar": 0, "min_lar": 0, "max_lar": 0, "avg_af1": 0, "min_af1": 0, "max_af1": 0, "avg_unmatched_gt_elements": 4.8, "min_unmatched_gt_elements": 0, "max_unmatched_gt_elements": 8, "avg_unmatched_pred_elements": 4.2, "min_unmatched_pred_elements": 0, "max_unmatched_pred_elements": 9, "avg_f1_Page-footer": 0.5555555555555555, "min_f1_Page-footer": 0, "max_f1_Page-footer": 1, "avg_precision_Page-footer": 0.5, "min_precision_Page-footer": 0, "max_precision_Page-footer": 1, "avg_recall_Page-footer": 0.6666666666666666, "min_recall_Page-footer": 0, "max_recall_Page-footer": 1, "avg_f1_Table": 0, "min_f1_Table": 0, "max_f1_Table": 0, "avg_precision_Table": 0, "min_precision_Table": 0, "max_precision_Table": 0, "avg_recall_Table": 0, "min_recall_Table": 0, "max_recall_Table": 0, "total_layout_localization_pass_rate_passed": 100, "total_layout_localization_pass_rate_evaluated": 124, "micro_layout_localization_pass_rate": 0.8064516129032258, "total_layout_classification_pass_rate_passed": 78, "total_layout_classification_pass_rate_evaluated": 124, "micro_layout_classification_pass_rate": 0.6290322580645161, "total_layout_attribution_pass_rate_passed": 0, "total_layout_attribution_pass_rate_evaluated": 107, "micro_layout_attribution_pass_rate": 0, "total_layout_rule_pass_rate_passed": 178, "total_layout_rule_pass_rate_evaluated": 355, "micro_layout_rule_pass_rate": 0.5014084507042254, "total_rule_pass_rate_passed": 178, "total_rule_pass_rate_evaluated": 355, "micro_rule_pass_rate": 0.5014084507042254, "total_layout_element_rule_pass_rate_passed": 6, "total_layout_element_rule_pass_rate_evaluated": 124, "micro_layout_element_rule_pass_rate": 0.04838709677419355, "total_unmatched_gt_elements": 24, "total_unmatched_pred_elements": 21 }, "runtime": { "startupSeconds": 3.580443707993254, "warmMedianSeconds": 0.5019186669960618, "warmMeanSeconds": 0.51700757565852, "peakRssBytes": 2076311552, "processorSize": "SizeDict(height=800, width=800, longest_edge=None, shortest_edge=None, max_height=None, max_width=None, min_pixels=None, max_pixels=None)" }, "overall": { "score": 57.75140542865944, "measuredAreas": 6, "totalAreas": 7, "areas": [ { "id": "flow", "label": "읽기 흐름", "score": 72.53053123996597, "datasets": [ { "dataset": "marathon", "score": 64.05228758169935 }, { "dataset": "mdp", "score": 89.42857142857143 }, { "dataset": "parsebench", "score": 64.11073470962712 } ] }, { "id": "content", "label": "문자 내용", "score": 85.76039214075804, "datasets": [ { "dataset": "marathon", "score": 85.83387301815739 }, { "dataset": "mdp", "score": 90.5472974550525 }, { "dataset": "parsebench", "score": 80.90000594906425 } ] }, { "id": "structure", "label": "문서 구조·목차", "score": 38.31345674332774, "datasets": [ { "dataset": "marathon", "score": 22.729523669136665 }, { "dataset": "mdp", "score": 44.44444444444444 }, { "dataset": "parsebench", "score": 47.76640211640211 } ] }, { "id": "media", "label": "그림·표·수식", "score": 47.73731897100278, "datasets": [ { "dataset": "marathon", "score": 57.39235147957296 }, { "dataset": "mdp", "score": 67.91806656028115 }, { "dataset": "parsebench", "score": 17.901538873154223 } ] }, { "id": "typography", "label": "글자 표현", "score": 45.199539585949324, "datasets": [ { "dataset": "parsebench", "score": 45.199539585949324 } ] }, { "id": "notes", "label": "주석·참조", "score": null, "datasets": [] }, { "id": "running", "label": "반복 요소·원본 페이지", "score": 56.96719389095276, "datasets": [ { "dataset": "marathon", "score": 96.79443881571541 }, { "dataset": "mdp", "score": 43.75 }, { "dataset": "parsebench", "score": 30.357142857142858 } ] } ], "fullSevenAreaBounds": [ 49.50120465313666, 63.786918938850945 ], "profileId": "kiro-dpa-v2-connected-20261103-v2", "aggregation": "공식 원래 집계 보존 → 데이터셋·메트릭별 원래/보충 프로토콜 균등 평균 → 영역별 데이터셋 균등 평균 → 측정 영역 균등 평균" }, "dpaDelta": -1.5515290297969244, "scoreSha256": "b819aa437178e59e9c211471fe57c3a419f970d41e94f234e5b0d132c99aacaf", "sampleChanges": [ { "id": "mdp-54fb82b1726b", "label": "ko_book_1.jpg", "previous": 87.4378928916828, "current": 61.10376911224369, "delta": -26.334123779439103 }, { "id": "parsebench-44474d2f099f", "label": "20190601_ECR_UKPB_081_p1.pdf", "previous": 62.17073418751271, "current": 43.13836586105774, "delta": -19.032368326454964 }, { "id": "parsebench-6e927d5e9b85", "label": "delltechnologies-fy24-esg-report_p6.pdf", "previous": 61.93705714645088, "current": 53.269864043178245, "delta": -8.667193103272638 }, { "id": "parsebench-08cc5c475326", "label": "text_multicolumns__14999.pdf", "previous": 87.34619017427471, "current": 78.70127812429936, "delta": -8.644912049975346 }, { "id": "parsebench-575ef6c72ec5", "label": "2007.03579v2_p5.pdf", "previous": 78.98674351585015, "current": 71.14581639877285, "delta": -7.8409271170773 }, { "id": "mdp-6213848de2f2", "label": "ko_financialreport_12.jpg", "previous": 88.37153155465084, "current": 84.37153155465084, "delta": -4 }, { "id": "parsebench-ce9d90e069d1", "label": "2050247507.jpg", "previous": 49.50824372759857, "current": 48.397132616487454, "delta": -1.1111111111111143 }, { "id": "parsebench-5dd32f0c5770", "label": "text_misc__marks.pdf", "previous": 27.040816326530614, "current": 26.13378684807256, "delta": -0.9070294784580533 }, { "id": "parsebench-eb2521cc7842", "label": "text_multicolumns__paperchem.pdf", "previous": 50.0727345047103, "current": 49.32958876156455, "delta": -0.743145743145746 }, { "id": "parsebench-18bee151fa52", "label": "text_ocr__069.pdf", "previous": 34.68936352687186, "current": 34.58819611998156, "delta": -0.10116740689029768 }, { "id": "parsebench-5847401fad94", "label": "text_multicolumns__10k2col.pdf", "previous": 27.644699272358846, "current": 27.644699272358846, "delta": 0 }, { "id": "mdp-612e15095482", "label": "ko_academicpaper_39.jpg", "previous": 79.41197036817664, "current": 79.41197036817664, "delta": 0 }, { "id": "mdp-08b85b25b6df", "label": "ko_academicpaper_40.jpg", "previous": 100, "current": 100, "delta": 0 }, { "id": "mdp-74ccbbcee6c4", "label": "ko_book_2.jpg", "previous": 42.04428483139357, "current": 42.04428483139357, "delta": 0 }, { "id": "mdp-9059e2e92a9a", "label": "ko_exampaper_24.jpg", "previous": 86.59003831417624, "current": 86.59003831417624, "delta": 0 }, { "id": "mdp-80d11cefe9f9", "label": "ko_textbook_17.jpg", "previous": 74.68051118210863, "current": 74.68051118210863, "delta": 0 }, { "id": "parsebench-a6ac79e04e6c", "label": "text_dense__de.pdf", "previous": 33.082492186688, "current": 33.27296837716419, "delta": 0.1904761904761898 }, { "id": "parsebench-7ebefff650eb", "label": "text_misc__edit2.pdf", "previous": 63.98186941057306, "current": 64.27577097006449, "delta": 0.2939015594914238 }, { "id": "marathon-mp3lan_0638", "label": "MA86E/L104 기술 설명서 · 연속 110쪽", "previous": 64.86018072263003, "current": 65.36049491285635, "delta": 0.5003141902263195 }, { "id": "parsebench-fc437565b684", "label": "text_dense__canara.pdf", "previous": 67.2227952784727, "current": 67.90282880417371, "delta": 0.6800335257010062 }, { "id": "parsebench-45cdc3d8e1a0", "label": "text_ocr__012-25.pdf", "previous": 74.86848282130498, "current": 75.79660977880148, "delta": 0.9281269574965023 }, { "id": "mdp-b306e62fcf08", "label": "ko_newspaper_43.jpg", "previous": 34.4767252929982, "current": 36.258916522074415, "delta": 1.7821912290762114 }, { "id": "parsebench-b895f53f5325", "label": "maersk-annual-report-2024_p8.pdf", "previous": 56.03091005104735, "current": 58.38910403098046, "delta": 2.3581939799331124 }, { "id": "parsebench-16807829a472", "label": "text_simple__marknotTable.pdf", "previous": 87.9080459770115, "current": 92.07471264367817, "delta": 4.166666666666671 }, { "id": "mdp-bb635290130a", "label": "ko_magazine_30.jpg", "previous": 75.45372256166554, "current": 79.89816700610999, "delta": 4.444444444444443 }, { "id": "mdp-53b6614478ad", "label": "ko_slide_10.jpg", "previous": 65.993265993266, "current": 79.2063492063492, "delta": 13.213083213083209 } ] }, { "key": "heron-101", "name": "Heron-101", "model": { "key": "heron-101", "id": "docling-project/docling-layout-heron-101", "revision": "2e4993cf6bb211112084a2f80938f26138008917", "localPath": "/Volumes/GitHubData/kiro-python/experiments/ebook-structure-20261103/huggingface/hub/models--docling-project--docling-layout-heron-101/snapshots/2e4993cf6bb211112084a2f80938f26138008917" }, "rawLayout": { "avg_mAP@[.50:.95]": 0.6407609086306566, "min_mAP@[.50:.95]": 0.5337079071742148, "max_mAP@[.50:.95]": 0.6978435672514619, "avg_AP50": 0.7228255885249725, "min_AP50": 0.6342623135424019, "max_AP50": 0.7965185185185185, "avg_AP75": 0.6451859704151289, "min_AP75": 0.4787146992792154, "max_AP75": 0.75, "avg_f1_Page-header": 0, "min_f1_Page-header": 0, "max_f1_Page-header": 0, "avg_precision_Page-header": 0, "min_precision_Page-header": 0, "max_precision_Page-header": 0, "avg_recall_Page-header": 0, "min_recall_Page-header": 0, "max_recall_Page-header": 0, "avg_f1_Picture": 0.3562237762237762, "min_f1_Picture": 0.06060606060606061, "max_f1_Picture": 0.6666666666666666, "avg_precision_Picture": 0.2683563748079877, "min_precision_Picture": 0.03225806451612903, "max_precision_Picture": 0.5, "avg_recall_Picture": 0.6333333333333333, "min_recall_Picture": 0.16666666666666666, "max_recall_Picture": 1, "avg_f1_Section": 0.5430303030303031, "min_f1_Section": 0.18181818181818182, "max_f1_Section": 1, "avg_precision_Section": 0.4454385964912281, "min_precision_Section": 0.1, "max_precision_Section": 1, "avg_recall_Section": 0.96, "min_recall_Section": 0.8, "max_recall_Section": 1, "avg_f1_Text": 0.5035671505824783, "min_f1_Text": 0.1889763779527559, "max_f1_Text": 0.9743589743589743, "avg_precision_Text": 0.40269346537888734, "min_precision_Text": 0.10810810810810811, "max_precision_Text": 0.95, "avg_recall_Text": 0.8388888888888889, "min_recall_Text": 0.6666666666666666, "max_recall_Text": 1, "avg_mean_f1": 0.4194872249951553, "min_mean_f1": 0.28570563294972745, "max_mean_f1": 0.6602564102564102, "avg_num_predictions": 85.2, "min_num_predictions": 29, "max_num_predictions": 135, "avg_num_ground_truth": 24.8, "min_num_ground_truth": 24, "max_num_ground_truth": 25, "avg_layout_localization_pass_rate": 0.9120000000000001, "min_layout_localization_pass_rate": 0.8, "max_layout_localization_pass_rate": 1, "avg_layout_classification_pass_rate": 0.8073333333333335, "min_layout_classification_pass_rate": 0.6, "max_layout_classification_pass_rate": 1, "avg_layout_attribution_pass_rate": 0, "min_layout_attribution_pass_rate": 0, "max_layout_attribution_pass_rate": 0, "avg_layout_rule_pass_rate": 0.6008830179062994, "min_layout_rule_pass_rate": 0.5068493150684932, "max_layout_rule_pass_rate": 0.6865671641791045, "avg_rule_pass_rate": 0.6008830179062994, "min_rule_pass_rate": 0.5068493150684932, "max_rule_pass_rate": 0.6865671641791045, "avg_layout_element_rule_pass_rate": 0.08966666666666667, "min_layout_element_rule_pass_rate": 0.04, "max_layout_element_rule_pass_rate": 0.20833333333333334, "avg_lap": 1, "min_lap": 1, "max_lap": 1, "avg_lar": 0, "min_lar": 0, "max_lar": 0, "avg_af1": 0, "min_af1": 0, "max_af1": 0, "avg_unmatched_gt_elements": 2.2, "min_unmatched_gt_elements": 0, "max_unmatched_gt_elements": 5, "avg_unmatched_pred_elements": 30.8, "min_unmatched_pred_elements": 3, "max_unmatched_pred_elements": 70, "avg_f1_Page-footer": 0.26666666666666666, "min_f1_Page-footer": 0, "max_f1_Page-footer": 0.4, "avg_precision_Page-footer": 0.16666666666666666, "min_precision_Page-footer": 0, "max_precision_Page-footer": 0.25, "avg_recall_Page-footer": 0.6666666666666666, "min_recall_Page-footer": 0, "max_recall_Page-footer": 1, "avg_f1_Table": 1, "min_f1_Table": 1, "max_f1_Table": 1, "avg_precision_Table": 1, "min_precision_Table": 1, "max_precision_Table": 1, "avg_recall_Table": 1, "min_recall_Table": 1, "max_recall_Table": 1, "total_layout_localization_pass_rate_passed": 113, "total_layout_localization_pass_rate_evaluated": 124, "micro_layout_localization_pass_rate": 0.9112903225806451, "total_layout_classification_pass_rate_passed": 100, "total_layout_classification_pass_rate_evaluated": 124, "micro_layout_classification_pass_rate": 0.8064516129032258, "total_layout_attribution_pass_rate_passed": 0, "total_layout_attribution_pass_rate_evaluated": 107, "micro_layout_attribution_pass_rate": 0, "total_layout_rule_pass_rate_passed": 213, "total_layout_rule_pass_rate_evaluated": 355, "micro_layout_rule_pass_rate": 0.6, "total_rule_pass_rate_passed": 213, "total_rule_pass_rate_evaluated": 355, "micro_rule_pass_rate": 0.6, "total_layout_element_rule_pass_rate_passed": 11, "total_layout_element_rule_pass_rate_evaluated": 124, "micro_layout_element_rule_pass_rate": 0.08870967741935484, "total_unmatched_gt_elements": 11, "total_unmatched_pred_elements": 154 }, "runtime": { "startupSeconds": 3.6690884580020793, "warmMedianSeconds": 0.37712754198582843, "warmMeanSeconds": 0.39198054963968865, "peakRssBytes": 1919303680, "processorSize": "SizeDict(height=640, width=640, longest_edge=None, shortest_edge=None, max_height=None, max_width=None, min_pixels=None, max_pixels=None)" }, "overall": { "score": 56.995458010174644, "measuredAreas": 6, "totalAreas": 7, "areas": [ { "id": "flow", "label": "읽기 흐름", "score": 71.89830987381026, "datasets": [ { "dataset": "marathon", "score": 69.0359477124183 }, { "dataset": "mdp", "score": 88.57142857142857 }, { "dataset": "parsebench", "score": 58.08755333758393 } ] }, { "id": "content", "label": "문자 내용", "score": 84.55890744652905, "datasets": [ { "dataset": "marathon", "score": 84.96236997267629 }, { "dataset": "mdp", "score": 89.87706295703904 }, { "dataset": "parsebench", "score": 78.8372894098718 } ] }, { "id": "structure", "label": "문서 구조·목차", "score": 40.040415282500774, "datasets": [ { "dataset": "marathon", "score": 21.799751138507606 }, { "dataset": "mdp", "score": 38.888888888888886 }, { "dataset": "parsebench", "score": 59.43260582010583 } ] }, { "id": "media", "label": "그림·표·수식", "score": 45.3354970418893, "datasets": [ { "dataset": "marathon", "score": 45.96503922355386 }, { "dataset": "mdp", "score": 67.91806656028115 }, { "dataset": "parsebench", "score": 22.123385341832897 } ] }, { "id": "typography", "label": "글자 표현", "score": 43.7676626206038, "datasets": [ { "dataset": "parsebench", "score": 43.7676626206038 } ] }, { "id": "notes", "label": "주석·참조", "score": null, "datasets": [] }, { "id": "running", "label": "반복 요소·원본 페이지", "score": 56.37195579571466, "datasets": [ { "dataset": "marathon", "score": 96.79443881571541 }, { "dataset": "mdp", "score": 43.75 }, { "dataset": "parsebench", "score": 28.571428571428573 } ] } ], "fullSevenAreaBounds": [ 48.85324972300684, 63.138964008721125 ], "profileId": "kiro-dpa-v2-connected-20261103-v2", "aggregation": "공식 원래 집계 보존 → 데이터셋·메트릭별 원래/보충 프로토콜 균등 평균 → 영역별 데이터셋 균등 평균 → 측정 영역 균등 평균" }, "dpaDelta": -2.3074764482817187, "scoreSha256": "23c355227f109dd91418649f10519e9eec50d336a4ad0b7c5a1ca9fa52ff2503", "sampleChanges": [ { "id": "mdp-54fb82b1726b", "label": "ko_book_1.jpg", "previous": 87.4378928916828, "current": 61.10376911224369, "delta": -26.334123779439103 }, { "id": "parsebench-44474d2f099f", "label": "20190601_ECR_UKPB_081_p1.pdf", "previous": 62.17073418751271, "current": 44.47849911282747, "delta": -17.692235074685236 }, { "id": "parsebench-6e927d5e9b85", "label": "delltechnologies-fy24-esg-report_p6.pdf", "previous": 61.93705714645088, "current": 44.88767081195175, "delta": -17.049386334499133 }, { "id": "parsebench-7ebefff650eb", "label": "text_misc__edit2.pdf", "previous": 63.98186941057306, "current": 55.88663131533497, "delta": -8.095238095238095 }, { "id": "mdp-53b6614478ad", "label": "ko_slide_10.jpg", "previous": 65.993265993266, "current": 60.15503875968992, "delta": -5.83822723357607 }, { "id": "parsebench-08cc5c475326", "label": "text_multicolumns__14999.pdf", "previous": 87.34619017427471, "current": 82.6872088602301, "delta": -4.65898131404461 }, { "id": "parsebench-fc437565b684", "label": "text_dense__canara.pdf", "previous": 67.2227952784727, "current": 63.026656902380694, "delta": -4.196138376092009 }, { "id": "parsebench-b895f53f5325", "label": "maersk-annual-report-2024_p8.pdf", "previous": 56.03091005104735, "current": 54.60951712336281, "delta": -1.4213929276845434 }, { "id": "marathon-mp3lan_0638", "label": "MA86E/L104 기술 설명서 · 연속 110쪽", "previous": 64.86018072263003, "current": 63.71150937257429, "delta": -1.1486713500557428 }, { "id": "parsebench-5dd32f0c5770", "label": "text_misc__marks.pdf", "previous": 27.040816326530614, "current": 26.13378684807256, "delta": -0.9070294784580533 }, { "id": "parsebench-eb2521cc7842", "label": "text_multicolumns__paperchem.pdf", "previous": 50.0727345047103, "current": 49.32958876156455, "delta": -0.743145743145746 }, { "id": "parsebench-18bee151fa52", "label": "text_ocr__069.pdf", "previous": 34.68936352687186, "current": 34.474742494054404, "delta": -0.2146210328174547 }, { "id": "parsebench-575ef6c72ec5", "label": "2007.03579v2_p5.pdf", "previous": 78.98674351585015, "current": 78.98674351585015, "delta": 0 }, { "id": "parsebench-ce9d90e069d1", "label": "2050247507.jpg", "previous": 49.50824372759857, "current": 49.50824372759857, "delta": 0 }, { "id": "parsebench-5847401fad94", "label": "text_multicolumns__10k2col.pdf", "previous": 27.644699272358846, "current": 27.644699272358846, "delta": 0 }, { "id": "parsebench-45cdc3d8e1a0", "label": "text_ocr__012-25.pdf", "previous": 74.86848282130498, "current": 74.86848282130498, "delta": 0 }, { "id": "mdp-612e15095482", "label": "ko_academicpaper_39.jpg", "previous": 79.41197036817664, "current": 79.41197036817664, "delta": 0 }, { "id": "mdp-08b85b25b6df", "label": "ko_academicpaper_40.jpg", "previous": 100, "current": 100, "delta": 0 }, { "id": "mdp-74ccbbcee6c4", "label": "ko_book_2.jpg", "previous": 42.04428483139357, "current": 42.04428483139357, "delta": 0 }, { "id": "mdp-9059e2e92a9a", "label": "ko_exampaper_24.jpg", "previous": 86.59003831417624, "current": 86.59003831417624, "delta": 0 }, { "id": "mdp-6213848de2f2", "label": "ko_financialreport_12.jpg", "previous": 88.37153155465084, "current": 88.37153155465084, "delta": 0 }, { "id": "mdp-bb635290130a", "label": "ko_magazine_30.jpg", "previous": 75.45372256166554, "current": 75.45372256166554, "delta": 0 }, { "id": "mdp-b306e62fcf08", "label": "ko_newspaper_43.jpg", "previous": 34.4767252929982, "current": 34.4767252929982, "delta": 0 }, { "id": "mdp-80d11cefe9f9", "label": "ko_textbook_17.jpg", "previous": 74.68051118210863, "current": 74.68051118210863, "delta": 0 }, { "id": "parsebench-a6ac79e04e6c", "label": "text_dense__de.pdf", "previous": 33.082492186688, "current": 33.124158853354665, "delta": 0.0416666666666643 }, { "id": "parsebench-16807829a472", "label": "text_simple__marknotTable.pdf", "previous": 87.9080459770115, "current": 89.83333333333333, "delta": 1.9252873563218316 } ] }, { "key": "egret-large", "name": "Egret-Large", "model": { "key": "egret-large", "id": "docling-project/docling-layout-egret-large", "revision": "fff417c78abd6bab338c87706c95a8d79dc68f1e", "localPath": "/Volumes/GitHubData/kiro-python/experiments/ebook-structure-20261103/huggingface/hub/models--docling-project--docling-layout-egret-large/snapshots/fff417c78abd6bab338c87706c95a8d79dc68f1e" }, "rawLayout": { "avg_mAP@[.50:.95]": 0.6593124101962563, "min_mAP@[.50:.95]": 0.539608356212136, "max_mAP@[.50:.95]": 0.7079156091119521, "avg_AP50": 0.7277430332914263, "min_AP50": 0.582359509580995, "max_AP50": 0.8333333333333333, "avg_AP75": 0.678220657834175, "min_AP75": 0.5725510917321261, "max_AP75": 0.75, "avg_f1_Page-header": 0.25, "min_f1_Page-header": 0, "max_f1_Page-header": 0.5, "avg_precision_Page-header": 0.16666666666666666, "min_precision_Page-header": 0, "max_precision_Page-header": 0.3333333333333333, "avg_recall_Page-header": 0.5, "min_recall_Page-header": 0, "max_recall_Page-header": 1, "avg_f1_Picture": 0.5291025641025641, "min_f1_Picture": 0.15384615384615385, "max_f1_Picture": 0.8, "avg_precision_Picture": 0.40757575757575754, "min_precision_Picture": 0.08333333333333333, "max_precision_Picture": 0.6666666666666666, "avg_recall_Picture": 0.9, "min_recall_Picture": 0.5, "max_recall_Picture": 1, "avg_f1_Section": 0.49917560700169394, "min_f1_Section": 0.19999999999999998, "max_f1_Section": 0.8571428571428571, "avg_precision_Section": 0.3753968253968254, "min_precision_Section": 0.1111111111111111, "max_precision_Section": 0.75, "avg_recall_Section": 0.96, "min_recall_Section": 0.8, "max_recall_Section": 1, "avg_f1_Text": 0.47453473444988725, "min_f1_Text": 0.30952380952380953, "max_f1_Text": 0.7169811320754718, "avg_precision_Text": 0.3330969006982933, "min_precision_Text": 0.19117647058823528, "max_precision_Text": 0.5588235294117647, "avg_recall_Text": 0.8788888888888889, "min_recall_Text": 0.7777777777777778, "max_recall_Text": 1, "avg_mean_f1": 0.46662487933064967, "min_mean_f1": 0.3516233766233766, "max_mean_f1": 0.6851976639712489, "avg_num_predictions": 80.2, "min_num_predictions": 45, "max_num_predictions": 118, "avg_num_ground_truth": 24.8, "min_num_ground_truth": 24, "max_num_ground_truth": 25, "avg_layout_localization_pass_rate": 0.96, "min_layout_localization_pass_rate": 0.92, "max_layout_localization_pass_rate": 1, "avg_layout_classification_pass_rate": 0.8386666666666667, "min_layout_classification_pass_rate": 0.68, "max_layout_classification_pass_rate": 1, "avg_layout_attribution_pass_rate": 0, "min_layout_attribution_pass_rate": 0, "max_layout_attribution_pass_rate": 0, "avg_layout_rule_pass_rate": 0.6282660550730563, "min_layout_rule_pass_rate": 0.5753424657534246, "max_layout_rule_pass_rate": 0.684931506849315, "avg_rule_pass_rate": 0.6282660550730563, "min_rule_pass_rate": 0.5753424657534246, "max_rule_pass_rate": 0.684931506849315, "avg_layout_element_rule_pass_rate": 0.10566666666666666, "min_layout_element_rule_pass_rate": 0.08, "max_layout_element_rule_pass_rate": 0.20833333333333334, "avg_lap": 1, "min_lap": 1, "max_lap": 1, "avg_lar": 0, "min_lar": 0, "max_lar": 0, "avg_af1": 0, "min_af1": 0, "max_af1": 0, "avg_unmatched_gt_elements": 1, "min_unmatched_gt_elements": 0, "max_unmatched_gt_elements": 2, "avg_unmatched_pred_elements": 23.6, "min_unmatched_pred_elements": 3, "max_unmatched_pred_elements": 62, "avg_f1_Page-footer": 0.253968253968254, "min_f1_Page-footer": 0.09523809523809525, "max_f1_Page-footer": 0.33333333333333337, "avg_precision_Page-footer": 0.15087719298245614, "min_precision_Page-footer": 0.05263157894736842, "max_precision_Page-footer": 0.2, "avg_recall_Page-footer": 0.8333333333333334, "min_recall_Page-footer": 0.5, "max_recall_Page-footer": 1, "avg_f1_Table": 1, "min_f1_Table": 1, "max_f1_Table": 1, "avg_precision_Table": 1, "min_precision_Table": 1, "max_precision_Table": 1, "avg_recall_Table": 1, "min_recall_Table": 1, "max_recall_Table": 1, "total_layout_localization_pass_rate_passed": 119, "total_layout_localization_pass_rate_evaluated": 124, "micro_layout_localization_pass_rate": 0.9596774193548387, "total_layout_classification_pass_rate_passed": 104, "total_layout_classification_pass_rate_evaluated": 124, "micro_layout_classification_pass_rate": 0.8387096774193549, "total_layout_attribution_pass_rate_passed": 0, "total_layout_attribution_pass_rate_evaluated": 107, "micro_layout_attribution_pass_rate": 0, "total_layout_rule_pass_rate_passed": 223, "total_layout_rule_pass_rate_evaluated": 355, "micro_layout_rule_pass_rate": 0.6281690140845071, "total_rule_pass_rate_passed": 223, "total_rule_pass_rate_evaluated": 355, "micro_rule_pass_rate": 0.6281690140845071, "total_layout_element_rule_pass_rate_passed": 13, "total_layout_element_rule_pass_rate_evaluated": 124, "micro_layout_element_rule_pass_rate": 0.10483870967741936, "total_unmatched_gt_elements": 5, "total_unmatched_pred_elements": 118 }, "runtime": { "startupSeconds": 3.387743916013278, "warmMedianSeconds": 0.3142212080419995, "warmMeanSeconds": 0.3276640544294591, "peakRssBytes": 1539686400, "processorSize": "SizeDict(height=640, width=640, longest_edge=None, shortest_edge=None, max_height=None, max_width=None, min_pixels=None, max_pixels=None)" }, "overall": { "score": 58.321255794215816, "measuredAreas": 6, "totalAreas": 7, "areas": [ { "id": "flow", "label": "읽기 흐름", "score": 71.8630928505819, "datasets": [ { "dataset": "marathon", "score": 67.15686274509804 }, { "dataset": "mdp", "score": 88.57142857142857 }, { "dataset": "parsebench", "score": 59.86098723521913 } ] }, { "id": "content", "label": "문자 내용", "score": 85.24467333690527, "datasets": [ { "dataset": "marathon", "score": 84.72496373278794 }, { "dataset": "mdp", "score": 90.40849534893655 }, { "dataset": "parsebench", "score": 80.60056092899129 } ] }, { "id": "structure", "label": "문서 구조·목차", "score": 42.45257794781865, "datasets": [ { "dataset": "marathon", "score": 21.60429469001679 }, { "dataset": "mdp", "score": 44.44444444444444 }, { "dataset": "parsebench", "score": 61.30899470899471 } ] }, { "id": "media", "label": "그림·표·수식", "score": 48.386068113487184, "datasets": [ { "dataset": "marathon", "score": 48.32277212900677 }, { "dataset": "mdp", "score": 67.91806656028115 }, { "dataset": "parsebench", "score": 28.917365651173643 } ] }, { "id": "typography", "label": "글자 표현", "score": 46.25506337732217, "datasets": [ { "dataset": "parsebench", "score": 46.25506337732217 } ] }, { "id": "notes", "label": "주석·참조", "score": null, "datasets": [] }, { "id": "running", "label": "반복 요소·원본 페이지", "score": 55.7260591391797, "datasets": [ { "dataset": "marathon", "score": 96.64246313182483 }, { "dataset": "mdp", "score": 43.75 }, { "dataset": "parsebench", "score": 26.785714285714285 } ] } ], "fullSevenAreaBounds": [ 49.98964782361355, 64.27536210932784 ], "profileId": "kiro-dpa-v2-connected-20261103-v2", "aggregation": "공식 원래 집계 보존 → 데이터셋·메트릭별 원래/보충 프로토콜 균등 평균 → 영역별 데이터셋 균등 평균 → 측정 영역 균등 평균" }, "dpaDelta": -0.9816786642405475, "scoreSha256": "3c885ab4ebe66f9174a75ee676de2e2a65cbb5cc31651771681964fb9f34732f", "sampleChanges": [ { "id": "mdp-54fb82b1726b", "label": "ko_book_1.jpg", "previous": 87.4378928916828, "current": 61.10376911224369, "delta": -26.334123779439103 }, { "id": "parsebench-b895f53f5325", "label": "maersk-annual-report-2024_p8.pdf", "previous": 56.03091005104735, "current": 44.46371577129568, "delta": -11.567194279751668 }, { "id": "parsebench-6e927d5e9b85", "label": "delltechnologies-fy24-esg-report_p6.pdf", "previous": 61.93705714645088, "current": 54.047839766279004, "delta": -7.889217380171878 }, { "id": "parsebench-7ebefff650eb", "label": "text_misc__edit2.pdf", "previous": 63.98186941057306, "current": 59.2497265534302, "delta": -4.732142857142861 }, { "id": "parsebench-08cc5c475326", "label": "text_multicolumns__14999.pdf", "previous": 87.34619017427471, "current": 82.67952350760804, "delta": -4.666666666666671 }, { "id": "marathon-mp3lan_0638", "label": "MA86E/L104 기술 설명서 · 연속 110쪽", "previous": 64.86018072263003, "current": 63.690271285746874, "delta": -1.1699094368831595 }, { "id": "parsebench-44474d2f099f", "label": "20190601_ECR_UKPB_081_p1.pdf", "previous": 62.17073418751271, "current": 61.21050873064296, "delta": -0.9602254568697504 }, { "id": "parsebench-18bee151fa52", "label": "text_ocr__069.pdf", "previous": 34.68936352687186, "current": 34.305744825056735, "delta": -0.3836187018151236 }, { "id": "parsebench-575ef6c72ec5", "label": "2007.03579v2_p5.pdf", "previous": 78.98674351585015, "current": 78.98674351585015, "delta": 0 }, { "id": "parsebench-ce9d90e069d1", "label": "2050247507.jpg", "previous": 49.50824372759857, "current": 49.50824372759857, "delta": 0 }, { "id": "parsebench-5847401fad94", "label": "text_multicolumns__10k2col.pdf", "previous": 27.644699272358846, "current": 27.644699272358846, "delta": 0 }, { "id": "parsebench-eb2521cc7842", "label": "text_multicolumns__paperchem.pdf", "previous": 50.0727345047103, "current": 50.0727345047103, "delta": 0 }, { "id": "mdp-612e15095482", "label": "ko_academicpaper_39.jpg", "previous": 79.41197036817664, "current": 79.41197036817664, "delta": 0 }, { "id": "mdp-08b85b25b6df", "label": "ko_academicpaper_40.jpg", "previous": 100, "current": 100, "delta": 0 }, { "id": "mdp-74ccbbcee6c4", "label": "ko_book_2.jpg", "previous": 42.04428483139357, "current": 42.04428483139357, "delta": 0 }, { "id": "mdp-9059e2e92a9a", "label": "ko_exampaper_24.jpg", "previous": 86.59003831417624, "current": 86.59003831417624, "delta": 0 }, { "id": "mdp-6213848de2f2", "label": "ko_financialreport_12.jpg", "previous": 88.37153155465084, "current": 88.37153155465084, "delta": 0 }, { "id": "mdp-bb635290130a", "label": "ko_magazine_30.jpg", "previous": 75.45372256166554, "current": 75.45372256166554, "delta": 0 }, { "id": "mdp-b306e62fcf08", "label": "ko_newspaper_43.jpg", "previous": 34.4767252929982, "current": 34.4767252929982, "delta": 0 }, { "id": "mdp-80d11cefe9f9", "label": "ko_textbook_17.jpg", "previous": 74.68051118210863, "current": 74.68051118210863, "delta": 0 }, { "id": "parsebench-5dd32f0c5770", "label": "text_misc__marks.pdf", "previous": 27.040816326530614, "current": 27.063492063492063, "delta": 0.022675736961449644 }, { "id": "parsebench-45cdc3d8e1a0", "label": "text_ocr__012-25.pdf", "previous": 74.86848282130498, "current": 74.9109222246721, "delta": 0.04243940336712626 }, { "id": "parsebench-a6ac79e04e6c", "label": "text_dense__de.pdf", "previous": 33.082492186688, "current": 33.27296837716419, "delta": 0.1904761904761898 }, { "id": "parsebench-fc437565b684", "label": "text_dense__canara.pdf", "previous": 67.2227952784727, "current": 68.97414518523134, "delta": 1.7513499067586338 }, { "id": "parsebench-16807829a472", "label": "text_simple__marknotTable.pdf", "previous": 87.9080459770115, "current": 92.07471264367817, "delta": 4.166666666666671 }, { "id": "mdp-53b6614478ad", "label": "ko_slide_10.jpg", "previous": 65.993265993266, "current": 79.12457912457913, "delta": 13.131313131313135 } ] } ], "resources": [ { "model": "heron", "stage": "convert", "scope": "latest invocation; may reuse completed conversion/evaluation steps", "exitCode": 0, "stopped": null, "peakGroupRssMiB": 3.734375, "minimumFreePercent": 45, "maxSwapGrowthMiB": 0 }, { "model": "pp-doclayout-v3", "stage": "convert", "scope": "latest invocation; may reuse completed conversion/evaluation steps", "exitCode": 0, "stopped": null, "peakGroupRssMiB": 1651.421875, "minimumFreePercent": 44, "maxSwapGrowthMiB": 0 }, { "model": "pp-doclayout-v3", "stage": "evaluate", "scope": "latest invocation; may reuse completed conversion/evaluation steps", "exitCode": 0, "stopped": null, "peakGroupRssMiB": 274.671875, "minimumFreePercent": 37, "maxSwapGrowthMiB": 0 }, { "model": "heron-101", "stage": "convert", "scope": "latest invocation; may reuse completed conversion/evaluation steps", "exitCode": 0, "stopped": null, "peakGroupRssMiB": 2266.15625, "minimumFreePercent": 39, "maxSwapGrowthMiB": 0 }, { "model": "heron-101", "stage": "evaluate", "scope": "latest invocation; may reuse completed conversion/evaluation steps", "exitCode": 0, "stopped": null, "peakGroupRssMiB": 1119.609375, "minimumFreePercent": 33, "maxSwapGrowthMiB": 198 }, { "model": "egret-large", "stage": "convert", "scope": "latest invocation; may reuse completed conversion/evaluation steps", "exitCode": 0, "stopped": null, "peakGroupRssMiB": 2085.6875, "minimumFreePercent": 36, "maxSwapGrowthMiB": 0 }, { "model": "egret-large", "stage": "evaluate", "scope": "latest invocation; may reuse completed conversion/evaluation steps", "exitCode": 0, "stopped": null, "peakGroupRssMiB": 1088.890625, "minimumFreePercent": 32, "maxSwapGrowthMiB": 173.3100000000013 } ], "guardProbe": { "passed": true, "guardedProcessGroupTerminated": true, "separateSentinelAlive": true, "memoryPressureNotInduced": true }, "controlReplay": { "reference": "/Volumes/ORICO/kiro-artifacts/ebook-dpa-css-fix-20261103/candidate", "rows": [ { "job": "marathon-mp3lan_0638", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-575ef6c72ec5", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-44474d2f099f", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-ce9d90e069d1", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-6e927d5e9b85", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-b895f53f5325", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-0b84261b4fc6", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-1265109a6ee7", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-2bb179df3d04", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-04c11de4a0e6", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-7ab2d83d1cd6", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-fc437565b684", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-a6ac79e04e6c", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-7ebefff650eb", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-5dd32f0c5770", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-5847401fad94", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-08cc5c475326", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-eb2521cc7842", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-45cdc3d8e1a0", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-18bee151fa52", "parser": true, "epub": true, "layout_pages": true }, { "job": "parsebench-16807829a472", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-612e15095482", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-08b85b25b6df", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-54fb82b1726b", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-74ccbbcee6c4", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-9059e2e92a9a", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-6213848de2f2", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-bb635290130a", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-b306e62fcf08", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-53b6614478ad", "parser": true, "epub": true, "layout_pages": true }, { "job": "mdp-80d11cefe9f9", "parser": true, "epub": true, "layout_pages": true } ] }, "limits": [ "5 layout GT documents; timing covers 142 pages; DPA covers 31 documents/142 pages", "Existing hybrid route preserved: native 18, OCR 25, provider semantic blocks retained on DP 99 pages", "Raw detection timing excludes OCR, table recognition, formula, EPUB assembly and evaluation", "Model native resolutions differ; threshold 0.3, CPU FP32, batch 1, 2 threads", "No note round-trip relation ground truth; six of seven DPA areas measured" ], "productionChanges": false }