AWS: S3 operations

To list all objects use:

private Try listAllFiles(String bucket, String key) {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key);
        return Try.of(() -> {
            final List objectListingList = new LinkedList();

            ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest);
            objectListingList.add(objectListing);

            while (objectListing != null && objectListing.isTruncated()) {
                objectListing = amazonS3.listNextBatchOfObjects(objectListing);
                objectListingList.add(objectListing);
            }

            return objectListingList.stream()
                .flatMap(objectListing1 -> objectListing1.getObjectSummaries().stream())
                .collect(Collectors.toList());
        });
    }

Read file:

 

public Try getBytes(String bucket, String key) {
        log.info("Reading data from: " + bucket + "/" + key);
        try (InputStream s3is = amazonS3.getObject(bucket, key).getObjectContent();
             ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
            byte[] buf = new byte[8192];
            int bytesRead;
            while ((bytesRead = s3is.read(buf)) != -1) {
                outputStream.write(buf, 0, bytesRead);
            }
            return Try.success(outputStream.toString());
        } catch (IOException e) {
            return Try.failure(e);
        }
    }

 

Gzip the content:

 

private Try saveGzip(String bucket, String forecastFaultsKey, byte[] ff) {
        return Try.of(() -> {
            ByteArrayOutputStream ffBaos = new ByteArrayOutputStream();
            try (OutputStream os = new GZIPOutputStream(ffBaos)) {
                os.write(ff);
            } catch (IOException e) {
                log.error(e.getMessage());
                throw new RuntimeException(e);
            }

            final byte[] ffCompressed = ffBaos.toByteArray();

            ObjectMetadata ob = new ObjectMetadata();
            ob.setContentLength(ffCompressed.length);
            ob.setContentType("application/octet-stream");

            amazonS3.putObject(bucket, forecastFaultsKey, new ByteArrayInputStream(ffCompressed), ob);

            return null;
        });
    }

private Try readGzip(String bucket, String key) {
        log.info("Reading data from: " + bucket + "/" + key);
        try (InputStream s3is = new GZIPInputStream(amazonS3.getObject(bucket, key).getObjectContent());
             ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
            byte[] buf = new byte[8192];
            int bytesRead;
            while ((bytesRead = s3is.read(buf)) != -1) {
                outputStream.write(buf, 0, bytesRead);
            }
            return Try.success(outputStream.toString());
        } catch (IOException e) {
            return Try.failure(e);
        }
    }
Advertisements