Newfeature1

.github/workflows/workflow.yml

@@ -31,7 +31,7 @@ jobs:
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws-region: us-east-1 #AWS region where you want to use CodeGuru
+        aws-region: eu-west-1 #AWS region where you want to use CodeGuru
 
     - name: CodeGuru Reviewer
       uses: aws-actions/codeguru-reviewer@v1.1

pom.xml

@@ -0,0 +1,48 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>sample-applications</groupId>
+  <artifactId>Guru-Sample-application</artifactId>
+  <version>0.0.1-SNAPSHOT</version>
+  <name>Guru-Sample-application</name>
+	<properties>
+		<maven.compiler.source>1.8</maven.compiler.source>
+		<maven.compiler.target>1.8</maven.compiler.target>
+	</properties>
+  <dependencies>
+  	<dependency>
+  		<groupId>com.amazonaws</groupId>
+  		<artifactId>aws-lambda-java-core</artifactId>
+  		<version>1.2.0</version>
+  	</dependency>
+  	<dependency>
+  		<groupId>com.amazonaws</groupId>
+  		<artifactId>aws-lambda-java-events</artifactId>
+  		<version>2.2.6</version>
+  	</dependency>
+  	<dependency>
+  		<groupId>com.amazonaws</groupId>
+  		<artifactId>aws-java-sdk-s3</artifactId>
+  		<version>1.11.624</version>
+  	</dependency>
+  	<dependency>
+  		<groupId>org.apache.commons</groupId>
+  		<artifactId>commons-lang3</artifactId>
+  		<version>3.8</version>
+  	</dependency>
+        <dependency>
+                <groupId>javax.servlet</groupId>
+                <artifactId>javax.servlet-api</artifactId>
+                <version>3.0.1</version>
+                <scope>provided</scope>
+        </dependency>
+  </dependencies>
+  <build>
+  	<plugins>
+  		<plugin>
+  			<groupId>org.apache.maven.plugins</groupId>
+  			<artifactId>maven-shade-plugin</artifactId>
+  			<version>3.2.1</version>
+  		</plugin>
+  	</plugins>
+  </build>
+</project>

src/main/java/EventHandler.java

@@ -0,0 +1,174 @@
+package com.shipmentEvents.handlers;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestHandler;
+import com.amazonaws.services.lambda.runtime.LambdaLogger;
+import com.amazonaws.services.lambda.runtime.events.ScheduledEvent;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.shipmentEvents.util.Constants;
+
+import org.apache.commons.lang3.tuple.MutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+
+
+public class EventHandler implements RequestHandler<ScheduledEvent, String> {
+
+    /**
+     * Shipment events for a carrier are uploaded to separate S3 buckets based on the source of events. E.g., events originating from
+     * the hand-held scanner are stored in a separate bucket than the ones from mobile App. The Lambda processes events from multiple
+     * sources and updates the latest status of the package in a summary S3 bucket every 15 minutes.
+     * 
+     * The events are stored in following format:
+     * - Each status update is a file, where the name of the file is tracking number + random id.
+     * - Each file has status and time-stamp as the first 2 lines respectively.
+     * - The time at which the file is stored in S3 is not an indication of the time-stamp of the event.
+     * - Once the status is marked as DELIVERED, we can stop tracking the package.
+     * 
+     * A Sample files looks as below:
+     *  FILE-NAME-> '8787323232232332--55322798-dd29-4a04-97f4-93e18feed554'
+     *   >status:IN TRANSIT
+     *   >timestamp: 1573410202
+     *   >Other fields like...tracking history and address
+     */
+    public String handleRequest(ScheduledEvent scheduledEvent, Context context) {
+
+        final LambdaLogger logger = context.getLogger();
+        try {
+            processShipmentUpdates(logger);
+            return "SUCCESS";
+        } catch (final Exception ex) {
+            logger.log(String.format("Failed to process shipment Updates in %s due to %s", scheduledEvent.getAccount(), ex.getMessage()));
+            throw new RuntimeException(ex);
+        }
+    }
+
+
+    private void processShipmentUpdates(final LambdaLogger logger) throws InterruptedException {
+
+        final List<String> bucketsToProcess = Constants.BUCKETS_TO_PROCESS;
+        final Map<String, Pair<Long, String>> latestStatusForTrackingNumber = new HashMap<String, Pair<Long, String>>();
+        final Map<String, List<KeyVersion>> filesToDelete = new HashMap<String, List<DeleteObjectsRequest.KeyVersion>>(); 
+        for (final String bucketName : bucketsToProcess) {
+            final List<KeyVersion> filesProcessed = processEventsInBucket(bucketName, logger, latestStatusForTrackingNumber);
+            filesToDelete.put(bucketName, filesProcessed);
+        }
+        final AmazonS3 s3Client = EventHandler.getS3Client();
+
+        //Create a new file in the Constants.SUMMARY_BUCKET
+        logger.log("Map of statuses -> " + latestStatusForTrackingNumber);
+        String summaryUpdateName = Long.toString(System.currentTimeMillis());
+
+        EventHandler.getS3Client().putObject(Constants.SUMMARY_BUCKET, summaryUpdateName, latestStatusForTrackingNumber.toString());
+
+        long expirationTime = System.currentTimeMillis() + Duration.ofMinutes(1).toMillis();
+        while(System.currentTimeMillis() < expirationTime) {
+            if (s3Client.doesObjectExist(Constants.SUMMARY_BUCKET, summaryUpdateName)) {
+                break;
+            }
+            logger.log("waiting for file to be created " + summaryUpdateName);
+            Thread.sleep(1000);
+        }
+
+        // Before we delete the shipment updates make sure the summary update file exists
+        if (EventHandler.getS3Client().doesObjectExist(Constants.SUMMARY_BUCKET, summaryUpdateName)) {
+            deleteProcessedFiles(filesToDelete);
+            logger.log("All updates successfully processed");
+        } else {
+            throw new RuntimeException("Failed to write summary status, will be retried in 15 minutes");
+        }
+
+    }
+
+    private List<KeyVersion> processEventsInBucket(String bucketName, LambdaLogger logger, Map<String, Pair<Long, String>> latestStatusForTrackingNumber) {
+        final AmazonS3 s3Client = EventHandler.getS3Client();
+        logger.log("Processing Bucket: " + bucketName);
+
+        ObjectListing files = s3Client.listObjects(bucketName);
+        List<KeyVersion> filesProcessed = new ArrayList<DeleteObjectsRequest.KeyVersion>();
+
+        for (Iterator<?> iterator = files.getObjectSummaries().iterator(); iterator.hasNext(); ) {
+            S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
+            logger.log("Reading Object: " + summary.getKey());
+
+            String trackingNumber = summary.getKey().split("--")[0];
+            Pair<Long, String> lastKnownStatus = latestStatusForTrackingNumber.get(trackingNumber);
+
+            // Check if this shipment has already been delivered, skip this file
+            if (lastKnownStatus != null && "DELIVERED".equals(lastKnownStatus.getRight())) {
+                continue;
+            }
+
+            String fileContents = s3Client.getObjectAsString(bucketName, summary.getKey());
+
+            if (!isValidFile(fileContents)) {
+                logger.log(String.format("Skipping invalid file %s", summary.getKey()));
+                continue;
+            }
+
+            if (!fileContents.contains("\n")) {
+
+            }
+            String[] lines = fileContents.split("\n");
+            String line1 = lines[0];
+            String line2 = lines[1];
+
+            String status = line1.split(":")[1];
+            Long timeStamp = Long.parseLong(line2.split(":")[1]);
+
+
+            if (null == lastKnownStatus || lastKnownStatus.getLeft() < timeStamp) {
+                lastKnownStatus = new MutablePair<Long, String>(timeStamp, status);
+                latestStatusForTrackingNumber.put(trackingNumber, lastKnownStatus);
+            }
+
+            //Add to list of processed files
+            filesProcessed.add(new KeyVersion(summary.getKey()));
+            logger.log("logging file contents" + fileContents);
+        }
+        return filesProcessed;
+    }
+
+
+    private void deleteProcessedFiles(Map<String, List<KeyVersion>> filesToDelete) {
+      final AmazonS3 s3Client = EventHandler.getS3Client();
+      for (Entry<String, List<KeyVersion>> entry : filesToDelete.entrySet()) {
+          final DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(entry.getKey()).withKeys(entry.getValue()).withQuiet(false);
+          s3Client.deleteObjects(deleteRequest);
+      }
+    }
+
+    private boolean isValidFile(String fileContents) {
+        if (!fileContents.contains("\n")) {
+            return false;
+        }
+        String[] lines = fileContents.split("\n");
+        for (String l: lines) {
+            if (!l.contains(":")) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public static AmazonS3 getS3Client() {
+        return AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
+    }
+
+
+}
+
+

src/main/java/com/shipmentEvents/util/Constants.java

@@ -0,0 +1,10 @@
+package com.shipmentEvents.util;
+
+import java.util.Arrays;
+import java.util.List;
+
+public final class Constants {
+
+    public static final List<String> BUCKETS_TO_PROCESS = Arrays.asList("shipment-events-from-scanner", "shipment-events-from-mobile-app");
+    public static final String SUMMARY_BUCKET = "shipment-events-summary"; 
+}

src/main/java/com/shipmentEvents/util/S3ClientUtil.java

@@ -0,0 +1,13 @@
+package com.shipmentEvents.util;
+
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+
+public class S3ClientUtil {
+
+    public static AmazonS3 getS3Client() {
+        return AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
+    }
+
+}