Odtnhj

I need to extract a bunch of zip files stored on s3 and add them to a tar archive and store that archive on s3. it is likely that that the sum of the zip files will greater than the 512mb local storage allowed from lambda functions. I have a partial souldtion that gets the objects from s3 extracts them and puts them in a s3 object without using the lambda local storage.

Extract object Thread

public class ExtractObject implements Runnable

 private String objectName;
 private String uuid;
 private final byte buffer = new byte[1024];

 public ExtractAdvert(String name, String uuid) 
 this.objectName= name;
 this.uuid= uuid;
 

 @Override
 public void run() 
 final String srcBucket = "my-bucket-name";
 final AmazonS3 s3Client = new AmazonS3Client();

 try 
 S3Object s3Object = s3Client.getObject(new GetObjectRequest(srcBucket, objectName));
 ZipInputStream zis = new ZipInputStream(s3Object.getObjectContent());
 ZipEntry entry = zis.getNextEntry();

 while(entry != null) 
 String fileName = entry.getName();
 String mimeType = FileMimeType.fromExtension(FilenameUtils.getExtension(fileName)).mimeType();
 System.out.println("Extracting " + fileName + ", compressed: " + entry.getCompressedSize() + " bytes, extracted: " + entry.getSize() + " bytes, mimetype: " + mimeType);
 ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
 int len;
 while ((len = zis.read(buffer)) > 0) 
 outputStream.write(buffer, 0, len);
 
 InputStream is = new ByteArrayInputStream(outputStream.toByteArray());
 ObjectMetadata meta = new ObjectMetadata();
 meta.setContentLength(outputStream.size());
 meta.setContentType(mimeType);
 System.out.println("##### " + srcBucket + ", " + FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName);

 // Add this to tar archive instead of putting back to s3
 s3Client.putObject(srcBucket, FilenameUtils.getFullPath(objectName) + "tmp" + File.separator + uuid + File.separator + fileName, is, meta);
 is.close();
 outputStream.close();
 entry = zis.getNextEntry();
 
 zis.closeEntry();
 zis.close();
 catch (IOException ioe) 
 System.out.println(ioe.getMessage());
 
 

this runs for each object that needs to be extracted and saves them in a s3 object in the structure needed for the tar file.

I think what i need is instead of putting the object back to s3 is to keep it in memory and add it to a tar archive. and upload that but after a lot of looking around and trial and error i have not created a successful tar file.
The main issue is i can't use the tmp directory in lambda.

Edit
should i be creating the tar file as i go instead of putting objects to s3? (see comment // Add this to tar archive instead of putting back to s3)
if so how do i create a tar stream without a storing it locally?

EDIT 2: Attempt at taring the files

ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
ListObjectsV2Result result;

ByteArrayOutputStream baos = new ByteArrayOutputStream();
TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);

do 
 result = s3Client.listObjectsV2(req);

 for (S3ObjectSummary objectSummary : result.getObjectSummaries()) 

 if(objectSummary.getKey().startsWith("tmp/") ) 
 System.out.printf(" - %s (size: %d)n", objectSummary.getKey(), objectSummary.getSize());
 S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
 InputStream is = s3Object.getObjectContent(); 
 System.out.println("Pre Create entry");
 TarArchiveEntry archiveEntry = new TarArchiveEntry(IOUtils.toByteArray(is));
 // Getting following exception above
 // IllegalArgumentException: Invalid byte 111 at offset 7 in ' positio' len=8
 System.out.println("Pre put entry");
 tarOut.putArchiveEntry(archiveEntry);
 System.out.println("Post put entry");
 
 

 String token = result.getNextContinuationToken();
 System.out.println("Next Continuation Token: " + token);
 req.setContinuationToken(token);
 while (result.isTruncated());

ObjectMetadata metadata = new ObjectMetadata();
InputStream is = new ByteArrayInputStream(baos.toByteArray());
s3Client.putObject(new PutObjectRequest(bucketName, bucketFolder + "tar-file", is, metadata));

I have found a solution to this and it very similar to my attempt in Edit 2 above.

private final String bucketName = "bucket-name";
private final String bucketFolder = "tmp/";
private final String tarKey = "tar-dir/tared-file.tar";

private void createTar() throws IOException, ArchiveException 
 ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName);
 ListObjectsV2Result result;

 ByteArrayOutputStream baos = new ByteArrayOutputStream();
 TarArchiveOutputStream tarOut = new TarArchiveOutputStream(baos);

 do 
 result = s3Client.listObjectsV2(req);

 for (S3ObjectSummary objectSummary : result.getObjectSummaries()) 
 if (objectSummary.getKey().startsWith(bucketFolder)) 
 S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, objectSummary.getKey()));
 InputStream is = s3Object.getObjectContent();

 String s3Key = objectSummary.getKey();
 String tarPath = s3Key.substring(s3Key.indexOf('/') + 1, s3Key.length());
 s3Key.lastIndexOf('.'));

 byte ba = IOUtils.toByteArray(is);

 TarArchiveEntry archiveEntry = new TarArchiveEntry(tarPath);
 archiveEntry.setSize(ba.length);
 tarOut.putArchiveEntry(archiveEntry);
 tarOut.write(ba);
 tarOut.closeArchiveEntry();
 
 

 String token = result.getNextContinuationToken();
 System.out.println("Next Continuation Token: " + token);
 req.setContinuationToken(token);
 while (result.isTruncated());

 ObjectMetadata metadata = new ObjectMetadata();
 InputStream is = baos.toInputStream();
 metadata.setContentLength(baos.size());
 s3Client.putObject(new PutObjectRequest(bucketName, tarKey, is, metadata));