Wie streamen S3 -Objekte in Java Springboot in S3 in S3? ⇐ Java

[Anonymous]

Ich verwende EC2 mit 1 GB RAM und 8 GB Speicher. Und dieser Code funktioniert sehr schnell, wenn die Gesamtgröße der Dateien unter 500-600 MB liegt, langsamer, wenn etwa 1 GB und wenn die Größe 10 GB beträgt, wird er immer weniger meines 200-Mbit / s-Internets verwendet und verwendet rund 200-1000 kbit / s? Gibt es eine Möglichkeit, es zu verwenden, ohne meinen EC2 -Speicher und -speicher zu verwenden? < /p>

Code: Select all

 private void streamFilesFromS3ToZipToS3(ZippingTask zippingTask) {
try {
// Update task status to PROCESSING
zippingTask.setStatus(ZipTaskStatus.PROCESSING);
zippingTaskService.save(zippingTask);

String folderS3Path = zippingTask.getFolder().getS3Path();
String folderName = zippingTask.getFolder().getFolderName();
String uuid = UUID.randomUUID().toString();
String zipKey = String.format("preSignedZips/%s/%s.zip", uuid, folderName);

List s3Objects = s3Handler.getAllObjectsUnderPrefix(folderS3Path);
int zippedFiles = 0;

// Create and upload zip to S3
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
try (ZipOutputStream zipOut = new ZipOutputStream(byteArrayOutputStream)) {
for (S3Object s3Object : s3Objects) {
String key = s3Object.key();
//                    if (!key.endsWith("/") ) { // Skip folders
if (!key.equals(zippingTask.getFolder().getS3Path())) { // Skip folders
try (InputStream objectData = s3Handler.getObjectInputStream(key)) {
ZipEntry zipEntry = new ZipEntry(key.substring(folderS3Path.length()));
zipOut.putNextEntry(zipEntry);
IOUtils.copy(objectData, zipOut);
zipOut.closeEntry();

log.info("Zipped file: {}", key);

zippingTask.setProgress((++zippedFiles) * 100 / s3Objects.size());
zippingTaskService.save(zippingTask);
}
}
}
}

// Upload zip to S3
s3Handler.uploadInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()), zipKey);
//            String zipPreSignedUrl = s3Handler.generatePreSignedUrlForSingleFile(zipKey);
String zipPreSignedUrl = s3Handler.generatePreSignedUrlForSingleFileWithExpirationDuration(zipKey, Duration.ofDays(1));

// Update task status
zippingTask.setStatus(ZipTaskStatus.COMPLETE);
zippingTask.setZipS3Key(zipKey);
zippingTask.setPresignedUrl(zipPreSignedUrl);
zippingTaskService.save(zippingTask);

// Send email
String userEmail = zippingTask.getUser().getEmail();
emailServices.sendZipCompletionEmail(userEmail, folderName, zipPreSignedUrl);

} catch (Exception e) {
log.error("Error processing zip task: {}", e.getMessage());
zippingTask.setStatus(ZipTaskStatus.ERROR);
zippingTask.setErrorMessage(e.getMessage());
zippingTaskService.save(zippingTask);
}
}
< /code>
    public void uploadInputStream(ByteArrayInputStream byteArrayInputStream, String zipKey) {
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(bucketName)
.key(zipKey)
.build();

RequestBody requestBody = RequestBody.fromInputStream(byteArrayInputStream, byteArrayInputStream.available());
s3Client.putObject(putObjectRequest, requestBody);
}