feat: add real PDF ingestion workflow

This commit is contained in:
Codex
2026-08-02 00:29:40 +08:00
parent 0eb2a86f36
commit 4d97a30616
18 changed files with 986 additions and 93 deletions
+26
View File
@@ -1,5 +1,6 @@
from dataclasses import dataclass
from collections.abc import Mapping
from io import BytesIO
from typing import Any
import boto3
@@ -70,3 +71,28 @@ class S3Storage:
object_key=object_key,
expires_in=int(self._value("s3_presigned_url_ttl_seconds", 900)),
)
def put_input(self, object_key: str, content: bytes, content_type: str) -> None:
self._put(self._value("s3_bucket_inputs"), object_key, content, content_type)
def put_output(self, object_key: str, content: bytes, content_type: str) -> None:
self._put(self._value("s3_bucket_derived"), object_key, content, content_type)
def get_output(self, object_key: str) -> tuple[bytes, str]:
if not self.configured:
raise RuntimeError("MinIO/S3 is not configured.")
response = self._client().get_object(
Bucket=self._value("s3_bucket_derived"),
Key=object_key,
)
return response["Body"].read(), response.get("ContentType", "application/octet-stream")
def _put(self, bucket: str, object_key: str, content: bytes, content_type: str) -> None:
if not self.configured:
raise RuntimeError("MinIO/S3 is not configured.")
self._client().upload_fileobj(
BytesIO(content),
bucket,
object_key,
ExtraArgs={"ContentType": content_type},
)