Merge pull request #141 from teaxyz/feature/upload-packages-to-ipfs

TS implementation: upload packages to ipfs

lambdas/package.json

@@ -9,15 +9,21 @@
   "engines": {
     "node": ">=14.15.0"
   },
+  "resolutions": {
+    "node-fetch": "2.6.7"
+  },
   "dependencies": {
     "@middy/core": "^3.4.0",
     "@middy/http-json-body-parser": "^3.4.0",
     "airtable": "^0.11.5",
     "algoliasearch": "^4.14.2",
-    "aws-sdk": "^2.1234.0",
+    "aws-lambda": "^1.0.7",
+    "aws-sdk": "^2.1239.0",
     "axios": "^1.1.3",
     "compare-versions": "^5.0.1",
-    "lodash": "^4.17.21"
+    "ipfs-http-client": "40.0.0",
+    "lodash": "^4.17.21",
+    "node-fetch": "2.6.7"
   },
   "devDependencies": {
     "@serverless/typescript": "^3.0.0",

lambdas/serverless.ts

@@ -1,6 +1,7 @@
 import type { AWS } from '@serverless/typescript';
 
 import buildPackages from '@functions/buildPackages';
+import ipfsUpload from '@functions/ipfsUpload';
 
 const serverlessConfiguration: AWS = {
   service: 'lambdas',
@@ -43,6 +44,7 @@ const serverlessConfiguration: AWS = {
   // import the function via paths
   functions: {
     buildPackages,
+    ipfsUpload,
   },
   package: { individually: true },
   custom: {

lambdas/src/functions/ipfsUpload/handler.ts

@@ -0,0 +1,43 @@
+import AWS from 'aws-sdk';
+import { S3CreateEvent } from 'aws-lambda';
+import ipfs from 'ipfs-http-client';
+
+const s3 = new AWS.S3();
+
+const ipfsUpload = async (event: S3CreateEvent) => {
+  const http = ipfs(process.env.IPFS_IP4_ADDRESS);
+  
+  // S3 interaction
+  for (const record of event.Records){
+    try {
+      const { key } = record.s3.object;
+      const isPackage =  key.split('/').length > 1;
+      const isCid = key.includes('.cid');
+      if(!isPackage || isCid) continue
+
+      // const objectURL = "https://" + process.env.BUCKET + ".s3.amazonaws.com/" + record.s3.object.key
+      const fileName = decodeURIComponent(record.s3.object.key.replace(/\+/g, " "))
+      const objectURL = "https://s3.amazonaws.com/" + process.env.AWS_DIST_BUCKET + "/" + fileName
+
+      const cid = await http.add(objectURL)
+      console.log({objectURL, cid})
+      const cidHash = cid[0].hash
+      console.log({cidHash})
+  
+      console.log("process.env.AWS_DIST_BUCKET: ", process.env.AWS_DIST_BUCKET)
+      console.log("Path: ", record.s3.object.key + ".cid")
+      
+      const s3Ack = await s3.putObject({
+        Bucket: process.env.AWS_DIST_BUCKET,
+        Key: fileName + ".cid",
+        Body: cidHash
+      }).promise()
+      
+      console.log({s3Ack})
+    } catch (error) {
+      console.error(error);
+    }
+  }
+}
+
+export const main = ipfsUpload;

lambdas/src/functions/ipfsUpload/index.ts

@@ -0,0 +1,36 @@
+// import schema from './schema';
+import { handlerPath } from '@libs/handler-resolver';
+
+export default {
+  handler: `${handlerPath(__dirname)}/handler.main`,
+  events: [
+    {
+      s3: {
+        bucket: '${ssm:AW5_S3_BUCKET}',
+        existing: true,
+        event: 's3:ObjectCreated:*',
+        rules: [
+          {
+            suffix: '.gz'
+          },
+        ]
+      },
+    },
+    {
+      s3: {
+        bucket: '${ssm:AW5_S3_BUCKET}',
+        existing: true,
+        event: 's3:ObjectCreated:*',
+        rules: [
+          {
+            suffix: '.xz'
+          },
+        ]
+      },
+    },
+  ],
+  environment: {
+    AWS_DIST_BUCKET: '${ssm:AW5_S3_BUCKET}',
+    IPFS_IP4_ADDRESS: '${ssm:/ipfs/ip4_address}',
+  }
+};

lambdas/src/functions/ipfsUpload/mock.json

@@ -0,0 +1,25 @@
+{
+  "Records": [
+    {
+      "s3": {
+        "object": {
+          "key": "zlib.net/v1.2.12.tar.gz"
+        }
+      }
+    },
+    {
+      "s3": {
+        "object": {
+          "key": "test/test.cid"
+        }
+      }
+    },
+    {
+      "s3": {
+        "object": {
+          "key": "packages.json"
+        }
+      }
+    }
+  ]
+}