diff --git a/mineru b/mineru new file mode 100644 index 0000000..a3b0c8f --- /dev/null +++ b/mineru @@ -0,0 +1,1364 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "95be98c2-a20c-48ed-983a-ca5991afba1b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_id: 4f92bfe5-a643-49c8-a0c8-40d1421f0b97\n", + "状态: completed\n", + "\n", + "解析成功，长度：6327 字符\n", + "In Re: 38047970\n", + "\n", + "Date: MAY 22, 2025\n", + "\n", + "Appeal of Service Center Operations (SCOPS) Decision\n", + "\n", + "Form 1-140, Immigrant Petition for Alien Workers (National Interest Waiver)\n", + "\n", + "The Petitioner, a nanotechnology scientist and researcher in the field of renewable energy, seeks employment-based second preference\n" + ] + } + ], + "source": [ + "import requests, json, time\n", + "\n", + "MINERU_API = \"http://localhost:8000\"\n", + "\n", + "def parse_pdf_with_mineru(pdf_path):\n", + " with open(pdf_path, \"rb\") as f:\n", + " files = {\"files\": (pdf_path.split(\"/\")[-1], f, \"application/pdf\")}\n", + " resp = requests.post(\n", + " f\"{MINERU_API}/tasks\",\n", + " files=files,\n", + " data={\"parse_method\": \"txt\", \"backend\": \"pipeline\"},\n", + " timeout=30\n", + " )\n", + " \n", + " task_id = resp.json()[\"task_id\"]\n", + " print(f\"task_id: {task_id}\")\n", + " \n", + " for _ in range(30):\n", + " time.sleep(2)\n", + " status = requests.get(f\"{MINERU_API}/tasks/{task_id}\").json()\n", + " print(f\"状态: {status['status']}\")\n", + " if status[\"status\"] == \"completed\":\n", + " result = requests.get(f\"{MINERU_API}/tasks/{task_id}/result\").json()\n", + " filename = list(result[\"results\"].keys())[0]\n", + " return result[\"results\"][filename][\"md_content\"]\n", + " elif status[\"status\"] == \"failed\":\n", + " raise Exception(f\"失败: {status['error']}\")\n", + " \n", + " raise Exception(\"超时\")\n", + "\n", + "# 测试\n", + "md = parse_pdf_with_mineru(\"/home/maxta/test_B5.pdf\")\n", + "print(f\"\\n解析成功，长度：{len(md)} 字符\")\n", + "print(md[:300])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cf534c5a-0271-41a4-9236-e8367fe61c5c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"backend\": \"pipeline\",\n", + " \"version\": \"3.2.2\",\n", + " \"results\": {\n", + " \"test_B5\": {\n", + " \"md_content\": \"In Re: 38047970\\n\\nDate: MAY 22, 2025\\n\\nAppeal of Service Center Operations (SCOPS) Decision\\n\\nForm 1-140, Immigrant Petition for Alien Workers (National Interest Waiver)\\n\\nThe Petitioner, a nanotechnology scientist and researcher in the field of renewable energy, seeks employment-based second preference (EB-2) immigrant classification as either a member of the professions holding an advanced degree, as well as a national interest waiver of the job offer requirement attached to this classification. See Immigration and Nationality Act (the Act) section 203(b )(2), 8 U.S .C. § l l 53(b )(2).\\n\\nSCOPS denied the petition, concluding that although the Petitioner had established his qualification for EB-2 classification as a member of the professions holding an advanced degree, he had not established that a waiver of the required job offer, and thus of the attached labor certification, would be in the national interest. The matter is now before us on appeal pursuant to 8 C.F.R. § 103.3.\\n\\nThe Petitioner bears the burden of proof to demonstrate eligibility by a preponderance of the evidence. Matter ofChawathe, 25 l&N Dec. 369, 375-76 (AAO 2010). We review the questions in this matter de novo. Matter of Christa's, Inc., 26 I&N Dec. 537,537 n.2 (AAO 2015). Upon de novo review, we will withdraw the decision and remand the matter for entry of a new decision consistent with the following analysis.\\n\\nTo qualify for the underlying EB-2 visa classification, a petitioner must establish they are an advanced degree professional or an individual of exceptional ability in the sciences, arts, or business. Section 203(b)(2)(A) of the Act.\\n\\nIf a petitioner establishes eligibility for the underlying EB-2 classification, they must then demonstrate that they merit a discretionary waiver of the job offer requirement \\\"in the national interest.\\\" Section 203(b )(2)(B)(i) ofthe Act. Matter ofDhanasar, 26 I&N Dec. 884, 889 (AAO 2016), provides the framework for adjudicating national interest waiver petitions. Dhanasar states that U.S. Citizenship and Immigration Services (USCIS) may, as matter of discretion, 1 grant a national interest waiver if the petitioner demonstrates that:\\n\\n• The proposed endeavor has both substantial merit and national importance;\\n\\n• The individual is well-positioned to advance their proposed endeavor; and\\n\\n• On balance, waiving the job offer requirement would benefit the United States.\\n\\n## Id.\\n\\nIn a document titled \\\"Proposed Endeavor & Future Plans in the Field\\\", the Petitioner describes his proposed endeavor as the advancement of \\\"research into nano piezolelectric and renewable energy production using polymeric materials at both nano and macro scales in order to develop products that generate renewable electric energy for applications in industrial sensors, medical textiles and instruments, and rechargeable batteries.\\\" In an updated statement in the record, the Petitioner identifies several research projects that he intends to pursue, including the development ofrenewable energy harvesting devices, the integration of energy harvesting into medical textiles; a project that would enable the generation and storage of electrical energy from body movements; and the development of highly sensitive sensors for monitoring critical infrastructure, using the unique properties of piezoelectric materials to enhance safety and operational efficiency in industries such as oil and petrochemicals. The Petitioner further explains that he will continue \\\"research focused on developing wearable textiles with inherently flexible nanocomposites\\\", mak[ing] them \\\"suitable for dynamic and comfortable medical wearables.\\\"\\n\\nSCOPS denied the petition, concluding that although the Petitioner established that his proposed endeavor was of substantial merit, he had not demonstrated that it was of national importance and therefore had not satisfied the first prong of the Dhanasar analytic framework. SCOPS determined that Petitioner had not shown that he is eligible for or otherwise merits a national interest waiver as a matter of discretion. 2\\n\\nThe Petitioner asserts on appeal that he has established that he meets the first prong of the Dhanasar analytical framework. Upon de novo review, we agree. We will withdraw SCOPS' determination to the contrary.\\n\\nThe first prong, substantial merit and national importance, focuses on the specific endeavor that the individual proposes to undertake. Matter ofDhanasar, 26 I&N Dec. at 889. The endeavor's merit may be demonstrated in a range of areas such as business, entrepreneurialism, science, technology, culture, health, or education. Id. In determining whether the proposed endeavor has national importance, we consider its potential prospective impact. Id.\\n\\nSCOPS concluded that the Petitioner had established the substantial merit of the proposed endeavor. We agree. To satisfy the national importance requirement, the Petitioner must demonstrate the \\\"potential prospective impact\\\" of his work. Here the record includes the Petitioner's statements indicating his intent to continue his research in the field of renewable energy with a focus on \\\"improving energy capture from piezoelectric polymer by the effect of natural stretching movements on it by using material technologies\\\" and to disseminate this research through publication in peerreviewed journals and conference presentations. This is sufficient to show, by a preponderance of the evidence, that the benefit of his proposed research has broader implications for the field, as the results will be disseminated to others in the field through peer-reviewed journals and conferences.\\n\\nThe Petitioner has demonstrated his eligibility for EB-2 classification as an advanced-degree professional. Further, the record demonstrates that the Petitioner has established that the proposed endeavor satisfies the first prong of the Dhanasar analytical framework. He therefore has overcome the sole ground for SCOPS' dismissal. We will remand the matter to SCOPS for consideration of whether the record establishes the Petitioner's eligibility under Dhanasar 's second and third prongs and whether he otherwise merits a national interest waiver as a matter of discretion.\\n\\nORDER: The decision is withdrawn. The matter is remanded for the entry of a new decision consistent with the foregoing analysis.\"\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "# 查看完整的返回结构\n", + "result = requests.get(f\"{MINERU_API}/tasks/fd713df8-2fbf-4149-80dd-24390ba35264/result\").json()\n", + "print(json.dumps(result, indent=2, ensure_ascii=False))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "6bd181a4-262f-4537-b290-00439607e126", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "251230->pdfplumber) (1.17.1)\n", + "Requirement already satisfied: pycparser in /opt/miniconda3/lib/python3.13/site-packages (from cffi>=1.14->cryptography>=36.0.0->pdfminer.six==20251230->pdfplumber) (3.0)\n", + "\n", + "\n" + ] + } + ], + "source": [ + "import subprocess\n", + "result = subprocess.run(['pip', 'install', 'pdfplumber', '--break-system-packages'], \n", + " capture_output=True, text=True)\n", + "print(result.stdout[-200:])\n", + "print(result.stderr[-200:])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "231fcb1e-95ca-482b-b31f-ff9ed2cb03a3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "共 3 页\n", + "task_id: fbb3be32-91a8-4879-af28-0c14206b0366\n", + "状态: completed\n", + " Page 1/3 完成，2254 字符\n", + "task_id: 29a086ed-bae4-4078-997f-9c76308347c1\n", + "状态: completed\n", + " Page 2/3 完成，3364 字符\n", + "task_id: f9607a1c-6140-488c-a852-787f594f5fab\n", + "状态: completed\n", + " Page 3/3 完成，705 字符\n", + "\n", + "共 3 个 chunk\n", + " Page 1: 2254 字符\n", + " Page 2: 3364 字符\n", + " Page 3: 705 字符\n" + ] + } + ], + "source": [ + "import pdfplumber, tempfile, os\n", + "\n", + "def parse_pdf_by_page(pdf_path, case_id):\n", + " \"\"\"按页切分 PDF，每页单独传给 MinerU，返回带页码索引的 chunk 列表\"\"\"\n", + " chunks = []\n", + " \n", + " with pdfplumber.open(pdf_path) as pdf:\n", + " total_pages = len(pdf.pages)\n", + " print(f\"共 {total_pages} 页\")\n", + " \n", + " for page_num, page in enumerate(pdf.pages, 1):\n", + " # 提取单页文本\n", + " text = page.extract_text()\n", + " if not text or len(text.strip()) < 50:\n", + " continue\n", + " \n", + " # 存成临时 PDF 传给 MinerU\n", + " import pypdfium2 as pdfium\n", + " src = pdfium.PdfDocument(pdf_path)\n", + " new_pdf = pdfium.PdfDocument.new()\n", + " new_pdf.import_pages(src, [page_num - 1])\n", + " \n", + " with tempfile.NamedTemporaryFile(suffix=\".pdf\", delete=False) as tmp:\n", + " tmp_path = tmp.name\n", + " new_pdf.save(tmp_path)\n", + " \n", + " # 调用 MinerU\n", + " try:\n", + " md = parse_pdf_with_mineru(tmp_path)\n", + " chunks.append({\n", + " \"case_id\": case_id,\n", + " \"filename\": os.path.basename(pdf_path),\n", + " \"page_num\": page_num,\n", + " \"total_pages\": total_pages,\n", + " \"text\": md,\n", + " \"char_count\": len(md)\n", + " })\n", + " print(f\" Page {page_num}/{total_pages} 完成，{len(md)} 字符\")\n", + " except Exception as e:\n", + " print(f\" Page {page_num} 失败: {e}\")\n", + " finally:\n", + " os.unlink(tmp_path)\n", + " \n", + " return chunks\n", + "\n", + "# 测试\n", + "chunks = parse_pdf_by_page(\"/home/maxta/test_B5.pdf\", \"38047970\")\n", + "print(f\"\\n共 {len(chunks)} 个 chunk\")\n", + "for c in chunks:\n", + " print(f\" Page {c['page_num']}: {c['char_count']} 字符\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "edc00b4c-bbd9-4f27-8bdb-1cf2c0a9eb7c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== Page 1 ===\n", + "In Re: 38047970\n", + "\n", + "Date: MAY 22, 2025\n", + "\n", + "Appeal of Service Center Operations (SCOPS) Decision\n", + "\n", + "Form 1-140, Immigrant Petition for Alien Workers (National Interest Waiver)\n", + "\n", + "The Petitioner, a nanotechnology scientist and researcher in the field of renewable energy, seeks employment-based second preference\n", + "...\n", + "\n", + "=== Page 2 ===\n", + "• The individual is well-positioned to advance their proposed endeavor; and\n", + "\n", + "• On balance, waiving the job offer requirement would benefit the United States.\n", + "\n", + "## Id.\n", + "\n", + "In a document titled \"Proposed Endeavor & Future Plans in the Field\", the Petitioner describes his proposed endeavor as the advanceme\n", + "...\n", + "\n", + "=== Page 3 ===\n", + "The Petitioner has demonstrated his eligibility for EB-2 classification as an advanced-degree professional. Further, the record demonstrates that the Petitioner has established that the proposed endeavor satisfies the first prong of the Dhanasar analytical framework. He therefore has overcome the so\n", + "...\n" + ] + } + ], + "source": [ + "for c in chunks:\n", + " print(f\"\\n=== Page {c['page_num']} ===\")\n", + " print(c['text'][:300])\n", + " print(\"...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "913167d4-81c2-4e73-a146-f1cc925839ef", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "nt already satisfied: charset_normalizer<4,>=2 in /opt/miniconda3/lib/python3.13/site-packages (from requests->kubernetes>=28.1.0->chromadb) (3.4.4)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in ./.local/lib/python3.13/site-packages (from requests-oauthlib->kubernetes>=28.1.0->chromadb) (3.3.1)\n", + "\n", + "\n" + ] + } + ], + "source": [ + "import subprocess\n", + "result = subprocess.run(\n", + " ['pip', 'install', 'chromadb', 'neo4j', '--break-system-packages'],\n", + " capture_output=True, text=True\n", + ")\n", + "print(result.stdout[-300:])\n", + "print(result.stderr[-100:])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9efc60d8-6126-4db8-8171-c1ff2c0f7730", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ChromaDB 正常\n" + ] + } + ], + "source": [ + "import chromadb\n", + "\n", + "client = chromadb.Client()\n", + "collection = client.create_collection(\"test\")\n", + "print(\"ChromaDB 正常\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "7f397f42-1fd0-40bf-92e8-5f73b0a06d3b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "存入 5571 个 chunk\n" + ] + } + ], + "source": [ + "import chromadb\n", + "import hashlib\n", + "\n", + "# 用持久化模式，数据不会丢\n", + "client = chromadb.PersistentClient(path=\"/home/maxta/niw_graph/chroma_db\")\n", + "collection = client.get_or_create_collection(\"niw_pages\")\n", + "\n", + "# 把 chunks 存入 ChromaDB\n", + "for chunk in chunks:\n", + " doc_id = f\"{chunk['case_id']}_page{chunk['page_num']}\"\n", + " collection.add(\n", + " documents=[chunk[\"text\"]],\n", + " metadatas=[{\n", + " \"case_id\": chunk[\"case_id\"],\n", + " \"filename\": chunk[\"filename\"],\n", + " \"page_num\": chunk[\"page_num\"],\n", + " \"total_pages\": chunk[\"total_pages\"],\n", + " \"char_count\": chunk[\"char_count\"]\n", + " }],\n", + " ids=[doc_id]\n", + " )\n", + "\n", + "print(f\"存入 {collection.count()} 个 chunk\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "893164e2-b30d-46e7-bcbb-71ac0880a0bd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "=== 结果 1 ===\n", + "案件: AUG152023_05B5203 | 第 7 页\n", + "eligible for a national interest waiver as a matter of discretion and further discussion of the balancing factors under the third prong would serve no meaningful purpose.\n", + "\n", + "## III. CONCLUSION\n", + "\n", + "As the P\n", + "\n", + "=== 结果 2 ===\n", + "案件: AUG142018_04B5203 | 第 7 页\n", + "endeavor as required by the second prong of the Dhanasar framework, he is not eligible for a national interest waiver and further discussion of the balancing factors under the third prong would serve \n", + "\n", + "=== 结果 3 ===\n", + "案件: MAR062023_01B5203 | 第 6 页\n", + "As the Petitioner has not established the national importance of his proposed endeavor as required by the first prong of the Dhanasar framework, he is not eligible for a national interest waiver and f\n" + ] + } + ], + "source": [ + "# 测试语义检索\n", + "results = collection.query(\n", + " query_texts=[\"Dhanasar three prongs national interest waiver\"],\n", + " n_results=3\n", + ")\n", + "\n", + "for i, (doc, meta) in enumerate(zip(results[\"documents\"][0], results[\"metadatas\"][0])):\n", + " print(f\"\\n=== 结果 {i+1} ===\")\n", + " print(f\"案件: {meta['case_id']} | 第 {meta['page_num']} 页\")\n", + " print(doc[:200])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "6f7b9b94-3791-45ae-8362-78c7fc89a532", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Neo4j is running at pid 2440233\n", + "\n", + "\n" + ] + } + ], + "source": [ + "import subprocess\n", + "result = subprocess.run(['neo4j', 'status'], capture_output=True, text=True)\n", + "print(result.stdout)\n", + "print(result.stderr)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "20e04383-bb04-47a7-a68d-8aa1aaa1f6fd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Neo4j 连接成功\n" + ] + } + ], + "source": [ + "from neo4j import GraphDatabase\n", + "\n", + "driver = GraphDatabase.driver(\n", + " \"bolt://localhost:7687\",\n", + " auth=(\"neo4j\", \"1qazXSW@\")\n", + ")\n", + "\n", + "with driver.session() as session:\n", + " result = session.run(\"RETURN 'Neo4j 连接成功' AS msg\")\n", + " print(result.single()[\"msg\"])\n", + "\n", + "driver.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9b007dad-31cf-40d6-b3be-e08cebf171e1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "节点统计：\n", + " Issue: 3 个\n", + " LegalNorm: 3 个\n", + " EvidenceType: 11 个\n", + " Case: 1191 个\n", + " Party: 1 个\n", + " LegalFact: 3 个\n", + " Claim: 1 个\n", + " Argument: 1 个\n", + " Chunk: 5571 个\n" + ] + } + ], + "source": [ + "from neo4j import GraphDatabase\n", + "\n", + "driver = GraphDatabase.driver(\"bolt://localhost:7687\", auth=(\"neo4j\", \"1qazXSW@\"))\n", + "\n", + "with driver.session() as session:\n", + " # 创建约束（保证唯一性）\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (c:Case) REQUIRE c.case_id IS UNIQUE\")\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (e:EvidenceType) REQUIRE e.name IS UNIQUE\")\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (n:LegalNorm) REQUIRE n.citation IS UNIQUE\")\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (i:Issue) REQUIRE i.name IS UNIQUE\")\n", + " \n", + " # 创建 Dhanasar 三要素节点（Issue）\n", + " session.run(\"\"\"\n", + " MERGE (i1:Issue {name: 'Prong1_SubstantialMerit'})\n", + " SET i1.description = 'The proposed endeavor has substantial merit and national importance'\n", + " MERGE (i2:Issue {name: 'Prong2_WellPositioned'})\n", + " SET i2.description = 'The individual is well-positioned to advance the endeavor'\n", + " MERGE (i3:Issue {name: 'Prong3_OnBalance'})\n", + " SET i3.description = 'On balance, waiving the job offer requirement benefits the United States'\n", + " \"\"\")\n", + " \n", + " # 创建核心法律规范节点（LegalNorm）\n", + " session.run(\"\"\"\n", + " MERGE (n1:LegalNorm {citation: 'Matter of Dhanasar, 26 I&N Dec. 884 (AAO 2016)'})\n", + " SET n1.type = 'precedent', n1.governs = 'NIW standard'\n", + " MERGE (n2:LegalNorm {citation: 'INA Section 203(b)(2)'})\n", + " SET n2.type = 'statute', n2.governs = 'EB-2 NIW eligibility'\n", + " MERGE (n3:LegalNorm {citation: '8 CFR 204.5'})\n", + " SET n3.type = 'regulation', n3.governs = 'EB-2 petition requirements'\n", + " \"\"\")\n", + " \n", + " # 创建证据类型节点（EvidenceType）\n", + " evidence_types = [\n", + " (\"HighCitationPapers\", \"高被引论文\", \"academic\"),\n", + " (\"PeerReview\", \"同行评审邀请\", \"academic\"),\n", + " (\"JournalPublication\", \"期刊发表\", \"academic\"),\n", + " (\"ConferencePaper\", \"会议论文\", \"academic\"),\n", + " (\"Patent\", \"专利\", \"innovation\"),\n", + " (\"MediaCoverage\", \"媒体报道\", \"public\"),\n", + " (\"GovernmentAdoption\", \"政府机构采用\", \"impact\"),\n", + " (\"Award\", \"奖项\", \"recognition\"),\n", + " (\"LeadershipRole\", \"重要职位\", \"role\"),\n", + " (\"Membership\", \"学术会员\", \"role\"),\n", + " (\"HighSalary\", \"高薪\", \"economic\"),\n", + " ]\n", + " \n", + " for name, name_cn, category in evidence_types:\n", + " session.run(\"\"\"\n", + " MERGE (e:EvidenceType {name: $name})\n", + " SET e.name_cn = $name_cn, e.category = $category\n", + " \"\"\", name=name, name_cn=name_cn, category=category)\n", + " \n", + " # 验证\n", + " result = session.run(\"MATCH (n) RETURN labels(n)[0] AS type, count(*) AS cnt\")\n", + " print(\"节点统计：\")\n", + " for record in result:\n", + " print(f\" {record['type']}: {record['cnt']} 个\")\n", + "\n", + "driver.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a346c0a1-1f3f-4469-9352-ca00a46f499f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "案件关系：\n", + " -HAS_EVIDENCE-> EvidenceType: PeerReview\n", + " -HAS_EVIDENCE-> EvidenceType: JournalPublication\n", + " -ADDRESSES_ISSUE-> Issue: Prong1_SubstantialMerit\n", + " -CITES_NORM-> LegalNorm: None\n" + ] + } + ], + "source": [ + "driver = GraphDatabase.driver(\"bolt://localhost:7687\", auth=(\"neo4j\", \"1qazXSW@\"))\n", + "\n", + "with driver.session() as session:\n", + " # 创建案件节点\n", + " session.run(\"\"\"\n", + " MERGE (c:Case {case_id: '38047970'})\n", + " SET c.date = 'MAY 22, 2025',\n", + " c.outcome = 'remanded',\n", + " c.filename = 'test_B5.pdf',\n", + " c.applicant_description = 'a nanotechnology scientist and researcher in the field of renewable energy'\n", + " \"\"\")\n", + " \n", + " # 连接法律规范\n", + " session.run(\"\"\"\n", + " MATCH (c:Case {case_id: '38047970'})\n", + " MATCH (n:LegalNorm {citation: 'Matter of Dhanasar, 26 I&N Dec. 884 (AAO 2016)'})\n", + " MERGE (c)-[:CITES_NORM]->(n)\n", + " \"\"\")\n", + " \n", + " # 连接 Dhanasar 要素\n", + " session.run(\"\"\"\n", + " MATCH (c:Case {case_id: '38047970'})\n", + " MATCH (i:Issue {name: 'Prong1_SubstantialMerit'})\n", + " MERGE (c)-[:ADDRESSES_ISSUE {result: 'passed'}]->(i)\n", + " \"\"\")\n", + " \n", + " # 连接证据类型\n", + " for evidence in [\"JournalPublication\", \"PeerReview\"]:\n", + " session.run(\"\"\"\n", + " MATCH (c:Case {case_id: '38047970'})\n", + " MATCH (e:EvidenceType {name: $evidence})\n", + " MERGE (c)-[:HAS_EVIDENCE]->(e)\n", + " \"\"\", evidence=evidence)\n", + " \n", + " # 验证\n", + " result = session.run(\"\"\"\n", + " MATCH (c:Case {case_id: '38047970'})-[r]->(n)\n", + " RETURN type(r) AS rel, labels(n)[0] AS target, n.name AS name\n", + " \"\"\")\n", + " print(\"案件关系：\")\n", + " for record in result:\n", + " print(f\" -{record['rel']}-> {record['target']}: {record['name']}\")\n", + "\n", + "driver.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "d71441a1-b4ad-4eac-ac0a-ad29e60ff71a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "文件级索引存入完成\n", + "niw_files collection: 1191 个文件\n", + "niw_pages collection: 5571 个页面\n" + ] + } + ], + "source": [ + "# 给 test_B5.pdf 生成文件级摘要，存入 ChromaDB 的独立 collection\n", + "import chromadb\n", + "\n", + "client = chromadb.PersistentClient(path=\"/home/maxta/niw_graph/chroma_db\")\n", + "\n", + "# 创建文件级索引 collection（粗筛层）\n", + "file_collection = client.get_or_create_collection(\"niw_files\")\n", + "\n", + "# 把所有 chunk 合并成文件摘要\n", + "full_text = \"\\n\".join([c[\"text\"] for c in chunks])\n", + "\n", + "# 提取摘要（取第一页 + 最后一页，覆盖案件基本信息和结论）\n", + "summary = chunks[0][\"text\"][:500] + \"\\n...\\n\" + chunks[-1][\"text\"][:500]\n", + "\n", + "file_collection.add(\n", + " documents=[summary],\n", + " metadatas=[{\n", + " \"case_id\": \"38047970\",\n", + " \"filename\": \"test_B5.pdf\",\n", + " \"total_pages\": len(chunks),\n", + " \"outcome\": \"remanded\",\n", + " \"applicant_description\": \"a nanotechnology scientist and researcher in the field of renewable energy\"\n", + " }],\n", + " ids=[\"file_38047970\"]\n", + ")\n", + "\n", + "print(f\"文件级索引存入完成\")\n", + "print(f\"niw_files collection: {file_collection.count()} 个文件\")\n", + "print(f\"niw_pages collection: {client.get_collection('niw_pages').count()} 个页面\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "150d35e3-f38d-4147-b3ec-dc3d6a7f88a6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== 第一层：文件级粗筛 ===\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'outcome'", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[15]\u001b[39m\u001b[32m, line 11\u001b[39m\n\u001b[32m 7\u001b[39m n_results=\u001b[32m3\u001b[39m\n\u001b[32m 8\u001b[39m )\n\u001b[32m 9\u001b[39m print(\u001b[33m\"=== 第一层：文件级粗筛 ===\"\u001b[39m)\n\u001b[32m 10\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m meta \u001b[38;5;28;01min\u001b[39;00m file_results[\u001b[33m\"metadatas\"\u001b[39m][\u001b[32m0\u001b[39m]:\n\u001b[32m---> \u001b[39m\u001b[32m11\u001b[39m print(f\" 案件: {meta[\u001b[33m'case_id'\u001b[39m]} | 结果: {meta[\u001b[33m'outcome'\u001b[39m]} | {meta[\u001b[33m'applicant_description'\u001b[39m][:\u001b[32m50\u001b[39m]}\")\n\u001b[32m 12\u001b[39m \n\u001b[32m 13\u001b[39m \u001b[38;5;66;03m# 第二层：在相关文件里精筛页面\u001b[39;00m\n\u001b[32m 14\u001b[39m page_collection = client.get_collection(\u001b[33m\"niw_pages\"\u001b[39m)\n", + "\u001b[31mKeyError\u001b[39m: 'outcome'" + ] + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6d10fa38-8a64-4a03-bd9d-d084e92bb1ba", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "当前图谱节点统计：\n", + " Chunk: 5571 个\n", + " Case: 1191 个\n", + " EvidenceType: 11 个\n", + " Issue: 3 个\n", + " LegalNorm: 3 个\n", + " LegalFact: 3 个\n", + " Party: 1 个\n", + " Claim: 1 个\n", + " Argument: 1 个\n" + ] + } + ], + "source": [ + "driver = GraphDatabase.driver(\"bolt://localhost:7687\", auth=(\"neo4j\", \"1qazXSW@\"))\n", + "\n", + "with driver.session() as session:\n", + " # 补充约束\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (p:Party) REQUIRE p.party_id IS UNIQUE\")\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (f:LegalFact) REQUIRE f.fact_id IS UNIQUE\")\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (c:Claim) REQUIRE c.claim_id IS UNIQUE\")\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (a:Argument) REQUIRE a.arg_id IS UNIQUE\")\n", + "\n", + " # Party 节点（申请人）\n", + " session.run(\"\"\"\n", + " MERGE (p:Party {party_id: 'party_38047970'})\n", + " SET p.type = 'Petitioner',\n", + " p.description = 'a nanotechnology scientist and researcher in the field of renewable energy',\n", + " p.case_id = '38047970'\n", + " \"\"\")\n", + "\n", + " # LegalFact 节点（从案件中提取的原子事实）\n", + " facts = [\n", + " (\"fact_38047970_1\", \"Petitioner is a nanotechnology scientist in renewable energy field\"),\n", + " (\"fact_38047970_2\", \"Petitioner disseminates research through peer-reviewed journals and conferences\"),\n", + " (\"fact_38047970_3\", \"Proposed endeavor has potential prospective impact on renewable energy field\"),\n", + " ]\n", + " for fact_id, text in facts:\n", + " session.run(\"\"\"\n", + " MERGE (f:LegalFact {fact_id: $fact_id})\n", + " SET f.text = $text, f.case_id = '38047970'\n", + " \"\"\", fact_id=fact_id, text=text)\n", + "\n", + " # Claim 节点（申请人主张）\n", + " session.run(\"\"\"\n", + " MERGE (c:Claim {claim_id: 'claim_38047970_prong1'})\n", + " SET c.text = 'Proposed endeavor has substantial merit and national importance',\n", + " c.dhanasar_prong = 'Prong1',\n", + " c.result = 'passed',\n", + " c.case_id = '38047970'\n", + " \"\"\")\n", + "\n", + " # Argument 节点\n", + " session.run(\"\"\"\n", + " MERGE (a:Argument {arg_id: 'arg_38047970_1'})\n", + " SET a.text = 'Research in renewable energy with dissemination through journals satisfies national importance',\n", + " a.case_id = '38047970'\n", + " \"\"\")\n", + "\n", + " # 建立关系\n", + " # Party → Case\n", + " session.run(\"\"\"\n", + " MATCH (p:Party {party_id: 'party_38047970'})\n", + " MATCH (c:Case {case_id: '38047970'})\n", + " MERGE (p)-[:FILED]->(c)\n", + " \"\"\")\n", + "\n", + " # Evidence → LegalFact\n", + " session.run(\"\"\"\n", + " MATCH (e:EvidenceType {name: 'JournalPublication'})\n", + " MATCH (f:LegalFact {fact_id: 'fact_38047970_2'})\n", + " MERGE (e)-[:PROVES {weight: 0.8}]->(f)\n", + " \"\"\")\n", + "\n", + " # LegalFact → Claim\n", + " session.run(\"\"\"\n", + " MATCH (f:LegalFact {fact_id: 'fact_38047970_3'})\n", + " MATCH (c:Claim {claim_id: 'claim_38047970_prong1'})\n", + " MERGE (f)-[:SUPPORTS {necessity: 'required'}]->(c)\n", + " \"\"\")\n", + "\n", + " # Argument → LegalFact + LegalNorm\n", + " session.run(\"\"\"\n", + " MATCH (a:Argument {arg_id: 'arg_38047970_1'})\n", + " MATCH (f:LegalFact {fact_id: 'fact_38047970_2'})\n", + " MERGE (a)-[:CITES_FACT]->(f)\n", + " \"\"\")\n", + " session.run(\"\"\"\n", + " MATCH (a:Argument {arg_id: 'arg_38047970_1'})\n", + " MATCH (n:LegalNorm {citation: 'Matter of Dhanasar, 26 I&N Dec. 884 (AAO 2016)'})\n", + " MERGE (a)-[:CITES_NORM]->(n)\n", + " \"\"\")\n", + "\n", + " # 验证所有节点\n", + " result = session.run(\"MATCH (n) RETURN labels(n)[0] AS type, count(*) AS cnt ORDER BY cnt DESC\")\n", + " print(\"当前图谱节点统计：\")\n", + " for record in result:\n", + " print(f\" {record['type']}: {record['cnt']} 个\")\n", + "\n", + "driver.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "15347944-b5aa-4814-9422-e00f62e05aa8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "最终图谱节点统计：\n", + " Chunk: 5571 个\n", + " Case: 1191 个\n", + " EvidenceType: 11 个\n", + " Issue: 3 个\n", + " LegalNorm: 3 个\n", + " LegalFact: 3 个\n", + " Party: 1 个\n", + " Claim: 1 个\n", + " Argument: 1 个\n" + ] + } + ], + "source": [ + "driver = GraphDatabase.driver(\"bolt://localhost:7687\", auth=(\"neo4j\", \"1qazXSW@\"))\n", + "\n", + "with driver.session() as session:\n", + " session.run(\"CREATE CONSTRAINT IF NOT EXISTS FOR (c:Chunk) REQUIRE c.chunk_id IS UNIQUE\")\n", + " \n", + " for chunk in chunks:\n", + " chunk_id = f\"{chunk['case_id']}_page{chunk['page_num']}\"\n", + " session.run(\"\"\"\n", + " MERGE (ch:Chunk {chunk_id: $chunk_id})\n", + " SET ch.case_id = $case_id,\n", + " ch.filename = $filename,\n", + " ch.page_num = $page_num,\n", + " ch.total_pages = $total_pages,\n", + " ch.text = $text,\n", + " ch.char_count = $char_count\n", + " \"\"\", chunk_id=chunk_id, **chunk)\n", + " \n", + " # Chunk 连接到 Case\n", + " session.run(\"\"\"\n", + " MATCH (ch:Chunk {chunk_id: $chunk_id})\n", + " MATCH (c:Case {case_id: $case_id})\n", + " MERGE (ch)-[:BELONGS_TO]->(c)\n", + " \"\"\", chunk_id=chunk_id, case_id=chunk[\"case_id\"])\n", + " \n", + " result = session.run(\"MATCH (n) RETURN labels(n)[0] AS type, count(*) AS cnt ORDER BY cnt DESC\")\n", + " print(\"最终图谱节点统计：\")\n", + " for record in result:\n", + " print(f\" {record['type']}: {record['cnt']} 个\")\n", + "\n", + "driver.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "34c585bb-c998-4739-83dc-ad4cee7da548", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "共 3393 个，已处理 1150，待处理 2243\n" + ] + } + ], + "source": [ + "import os, json, time, requests\n", + "import pdfplumber\n", + "import pypdfium2 as pdfium\n", + "import tempfile\n", + "import chromadb\n", + "from neo4j import GraphDatabase\n", + "from tqdm import tqdm\n", + "\n", + "# 配置\n", + "MINERU_API = \"http://localhost:8000\"\n", + "PDF_DIR = \"/home/maxta/niw_graph/niw_graph_mvp/data/raw\"\n", + "OUTPUT_DIR = \"/home/maxta/niw_graph/data/processed\"\n", + "CHROMA_PATH = \"/home/maxta/niw_graph/chroma_db\"\n", + "LOG_PATH = \"/home/maxta/niw_graph/data/processed/mineru_log.json\"\n", + "\n", + "os.makedirs(OUTPUT_DIR, exist_ok=True)\n", + "\n", + "# 初始化数据库\n", + "chroma_client = chromadb.PersistentClient(path=CHROMA_PATH)\n", + "file_collection = chroma_client.get_or_create_collection(\"niw_files\")\n", + "page_collection = chroma_client.get_or_create_collection(\"niw_pages\")\n", + "driver = GraphDatabase.driver(\"bolt://localhost:7687\", auth=(\"neo4j\", \"1qazXSW@\"))\n", + "\n", + "# 读取日志\n", + "if os.path.exists(LOG_PATH):\n", + " with open(LOG_PATH) as f:\n", + " log = json.load(f)\n", + "else:\n", + " log = {\"done\": [], \"failed\": []}\n", + "\n", + "pdfs = [f for f in os.listdir(PDF_DIR) if f.endswith(\".pdf\") and f != \"test_B5.pdf\"]\n", + "todo = [f for f in pdfs if f not in log[\"done\"]]\n", + "print(f\"共 {len(pdfs)} 个，已处理 {len(log['done'])}，待处理 {len(todo)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "09a43a56-016c-43b6-85e9-2a1451411883", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████████████████████████████████████| 3/3 [00:00<00:00, 31300.78it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✗ NOV082023_04B5203.pdf - name 'process_pdf' is not defined\n", + "✗ MAY012020_01B5203.pdf - name 'process_pdf' is not defined\n", + "✗ JAN052024_05B5203.pdf - name 'process_pdf' is not defined\n", + "\n", + "测试完成：成功 1150，失败 3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "from tqdm import tqdm\n", + "\n", + "# 先测试前3个确认没问题\n", + "test_batch = todo[:3]\n", + "\n", + "for filename in tqdm(test_batch):\n", + " pdf_path = os.path.join(PDF_DIR, filename)\n", + " try:\n", + " success = process_pdf(pdf_path, filename)\n", + " if success:\n", + " log[\"done\"].append(filename)\n", + " print(f\"✓ {filename}\")\n", + " else:\n", + " log[\"failed\"].append({\"file\": filename, \"error\": \"no chunks\"})\n", + " print(f\"✗ {filename} - 无内容\")\n", + " except Exception as e:\n", + " log[\"failed\"].append({\"file\": filename, \"error\": str(e)})\n", + " print(f\"✗ {filename} - {e}\")\n", + "\n", + "with open(LOG_PATH, \"w\") as f:\n", + " json.dump(log, f)\n", + "\n", + "print(f\"\\n测试完成：成功 {len(log['done'])}，失败 {len(log['failed'])}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "b361caa9-e78e-4f2a-a5fd-52d5f85f030e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "测试：test.pdf\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def process_pdf(pdf_path, filename):\n", + " print(f\"测试：{filename}\")\n", + " return True\n", + "\n", + "process_pdf(\"test\", \"test.pdf\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "788e8776-42bf-49cb-b001-c0599e5d591a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "函数定义完成\n" + ] + } + ], + "source": [ + "def parse_single_page(pdf_path, page_num):\n", + " src = pdfium.PdfDocument(pdf_path)\n", + " new_pdf = pdfium.PdfDocument.new()\n", + " new_pdf.import_pages(src, [page_num])\n", + " \n", + " with tempfile.NamedTemporaryFile(suffix=\".pdf\", delete=False) as tmp:\n", + " tmp_path = tmp.name\n", + " new_pdf.save(tmp_path)\n", + " \n", + " try:\n", + " with open(tmp_path, \"rb\") as f:\n", + " files = {\"files\": (f\"page_{page_num}.pdf\", f, \"application/pdf\")}\n", + " resp = requests.post(\n", + " f\"{MINERU_API}/tasks\",\n", + " files=files,\n", + " data={\"parse_method\": \"txt\", \"backend\": \"pipeline\"},\n", + " timeout=30\n", + " )\n", + " task_id = resp.json()[\"task_id\"]\n", + " \n", + " for _ in range(30):\n", + " time.sleep(2)\n", + " status = requests.get(f\"{MINERU_API}/tasks/{task_id}\").json()\n", + " if status[\"status\"] == \"completed\":\n", + " result = requests.get(f\"{MINERU_API}/tasks/{task_id}/result\").json()\n", + " filename = list(result[\"results\"].keys())[0]\n", + " return result[\"results\"][filename][\"md_content\"]\n", + " elif status[\"status\"] == \"failed\":\n", + " return None\n", + " return None\n", + " finally:\n", + " os.unlink(tmp_path)\n", + "\n", + "\n", + "def process_pdf(pdf_path, filename):\n", + " case_id = filename.replace(\".pdf\", \"\")\n", + " \n", + " with pdfplumber.open(pdf_path) as pdf:\n", + " total_pages = len(pdf.pages)\n", + " \n", + " chunks = []\n", + " \n", + " for page_num in range(total_pages):\n", + " md = parse_single_page(pdf_path, page_num)\n", + " if md and len(md.strip()) > 50:\n", + " chunks.append({\n", + " \"case_id\": case_id,\n", + " \"filename\": filename,\n", + " \"page_num\": page_num + 1,\n", + " \"total_pages\": total_pages,\n", + " \"text\": md,\n", + " \"char_count\": len(md)\n", + " })\n", + " \n", + " if not chunks:\n", + " return False\n", + " \n", + " # 存入 ChromaDB 页级\n", + " page_collection.add(\n", + " documents=[c[\"text\"] for c in chunks],\n", + " metadatas=[{k: v for k, v in c.items() if k != \"text\"} for c in chunks],\n", + " ids=[f\"{c['case_id']}_page{c['page_num']}\" for c in chunks]\n", + " )\n", + " \n", + " # 存入 ChromaDB 文件级\n", + " summary = chunks[0][\"text\"][:500] + \"\\n...\\n\" + chunks[-1][\"text\"][:300]\n", + " file_collection.add(\n", + " documents=[summary],\n", + " metadatas=[{\n", + " \"case_id\": case_id,\n", + " \"filename\": filename,\n", + " \"total_pages\": total_pages,\n", + " }],\n", + " ids=[f\"file_{case_id}\"]\n", + " )\n", + " \n", + " # 存入 Neo4j\n", + " with driver.session() as session:\n", + " session.run(\"\"\"\n", + " MERGE (c:Case {case_id: $case_id})\n", + " SET c.filename = $filename, c.total_pages = $total_pages\n", + " \"\"\", case_id=case_id, filename=filename, total_pages=total_pages)\n", + " \n", + " for chunk in chunks:\n", + " chunk_id = f\"{case_id}_page{chunk['page_num']}\"\n", + " session.run(\"\"\"\n", + " MERGE (ch:Chunk {chunk_id: $chunk_id})\n", + " SET ch.case_id = $case_id, ch.page_num = $page_num,\n", + " ch.text = $text, ch.char_count = $char_count\n", + " WITH ch\n", + " MATCH (c:Case {case_id: $case_id})\n", + " MERGE (ch)-[:BELONGS_TO]->(c)\n", + " \"\"\", chunk_id=chunk_id, case_id=case_id,\n", + " page_num=chunk[\"page_num\"], text=chunk[\"text\"],\n", + " char_count=chunk[\"char_count\"])\n", + " \n", + " return True\n", + "\n", + "print(\"函数定义完成\")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "bc6bf5fa-3439-4c48-9e58-f8d4dbd882c2", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 33%|███████████████ | 1/3 [00:10<00:21, 10.76s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✓ NOV082023_04B5203.pdf\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 67%|██████████████████████████████ | 2/3 [00:21<00:10, 10.63s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✓ MAY012020_01B5203.pdf\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████| 3/3 [00:37<00:00, 12.63s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✓ JAN052024_05B5203.pdf\n", + "\n", + "测试：成功 1153，失败 3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "test_batch = todo[:3]\n", + "\n", + "for filename in tqdm(test_batch):\n", + " pdf_path = os.path.join(PDF_DIR, filename)\n", + " try:\n", + " success = process_pdf(pdf_path, filename)\n", + " if success:\n", + " log[\"done\"].append(filename)\n", + " print(f\"✓ {filename}\")\n", + " else:\n", + " log[\"failed\"].append({\"file\": filename, \"error\": \"no chunks\"})\n", + " print(f\"✗ {filename} - 无内容\")\n", + " except Exception as e:\n", + " log[\"failed\"].append({\"file\": filename, \"error\": str(e)})\n", + " print(f\"✗ {filename} - {e}\")\n", + "\n", + "with open(LOG_PATH, \"w\") as f:\n", + " json.dump(log, f)\n", + "\n", + "print(f\"\\n测试：成功 {len(log['done'])}，失败 {len(log['failed'])}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fd8da9a-9379-482b-83b8-7c035d7d9167", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "待处理：2240 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 2%|▊ | 47/2240 [07:52<5:22:38, 8.83s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1200 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 4%|█▋ | 97/2240 [15:33<5:53:00, 9.88s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1250 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 7%|██▍ | 147/2240 [23:09<5:04:10, 8.72s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1300 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 9%|███▎ | 197/2240 [31:46<6:12:29, 10.94s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1350 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 11%|████▏ | 247/2240 [40:23<7:02:31, 12.72s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1400 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 13%|█████ | 297/2240 [48:08<5:02:43, 9.35s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1450 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 15%|█████▉ | 347/2240 [56:28<5:13:01, 9.92s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1500 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 18%|██████▍ | 397/2240 [1:04:04<4:32:05, 8.86s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1550 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 20%|███████▏ | 447/2240 [1:12:02<5:04:29, 10.19s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "已完成 1600 个\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 20%|███████▏ | 450/2240 [1:12:36<5:05:20, 10.24s/it]" + ] + } + ], + "source": [ + "# 重新读取日志，过滤掉旧的失败记录\n", + "log[\"failed\"] = []\n", + "todo_full = [f for f in pdfs if f not in log[\"done\"]]\n", + "print(f\"待处理：{len(todo_full)} 个\")\n", + "\n", + "for filename in tqdm(todo_full):\n", + " pdf_path = os.path.join(PDF_DIR, filename)\n", + " try:\n", + " success = process_pdf(pdf_path, filename)\n", + " if success:\n", + " log[\"done\"].append(filename)\n", + " else:\n", + " log[\"failed\"].append({\"file\": filename, \"error\": \"no chunks\"})\n", + " except Exception as e:\n", + " log[\"failed\"].append({\"file\": filename, \"error\": str(e)})\n", + " \n", + " # 每50个保存一次\n", + " if len(log[\"done\"]) % 50 == 0:\n", + " with open(LOG_PATH, \"w\") as f:\n", + " json.dump(log, f)\n", + " print(f\"已完成 {len(log['done'])} 个\")\n", + "\n", + "with open(LOG_PATH, \"w\") as f:\n", + " json.dump(log, f)\n", + "print(f\"完成：{len(log['done'])} 成功，{len(log['failed'])} 失败\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df58bc45-c910-42d8-b1c2-1791dcd25a7c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}