feat: add real PDF ingestion workflow
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Mapping
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
@@ -70,3 +71,28 @@ class S3Storage:
|
||||
object_key=object_key,
|
||||
expires_in=int(self._value("s3_presigned_url_ttl_seconds", 900)),
|
||||
)
|
||||
|
||||
def put_input(self, object_key: str, content: bytes, content_type: str) -> None:
|
||||
self._put(self._value("s3_bucket_inputs"), object_key, content, content_type)
|
||||
|
||||
def put_output(self, object_key: str, content: bytes, content_type: str) -> None:
|
||||
self._put(self._value("s3_bucket_derived"), object_key, content, content_type)
|
||||
|
||||
def get_output(self, object_key: str) -> tuple[bytes, str]:
|
||||
if not self.configured:
|
||||
raise RuntimeError("MinIO/S3 is not configured.")
|
||||
response = self._client().get_object(
|
||||
Bucket=self._value("s3_bucket_derived"),
|
||||
Key=object_key,
|
||||
)
|
||||
return response["Body"].read(), response.get("ContentType", "application/octet-stream")
|
||||
|
||||
def _put(self, bucket: str, object_key: str, content: bytes, content_type: str) -> None:
|
||||
if not self.configured:
|
||||
raise RuntimeError("MinIO/S3 is not configured.")
|
||||
self._client().upload_fileobj(
|
||||
BytesIO(content),
|
||||
bucket,
|
||||
object_key,
|
||||
ExtraArgs={"ContentType": content_type},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user