Merge pull request #145 from teaxyz/bugfix/more-suffix-ipfs-uploads

add ext .sha256sum. .tgz and .txt files to ipfs uploads
enable handling large filesize for ipfs uploads

lambdas/package.json

@@ -10,7 +10,7 @@
     "node": ">=14.15.0"
   },
   "resolutions": {
-    "node-fetch": "^1.6.3"
+    "node-fetch": "2.6.1"
   },
   "dependencies": {
     "@middy/core": "^3.4.0",
@@ -23,7 +23,7 @@
     "compare-versions": "^5.0.1",
     "ipfs-http-client": "^33.1.1",
     "lodash": "^4.17.21",
-    "node-fetch": "^1.6.3"
+    "node-fetch": "2.6.1"
   },
   "devDependencies": {
     "@serverless/typescript": "^3.0.0",

lambdas/src/functions/ipfsUpload/handler.ts

@@ -11,7 +11,7 @@ const ipfsUpload = async (event: S3CreateEvent) => {
   // S3 interaction
   for (const record of event.Records){
     try {
-      const { key } = record.s3.object;
+      const { key, size } = record.s3.object;
       const isPackage =  key.split('/').length > 1;
       if(!isPackage) continue
 
@@ -19,29 +19,33 @@ const ipfsUpload = async (event: S3CreateEvent) => {
       const fileName = decodeURIComponent(record.s3.object.key.replace(/\+/g, " "))
 
       // download file to /tmp      
-      const { Body } = await s3.getObject({
+      const s3ReadStream = s3.getObject({
         Bucket: process.env.AWS_DIST_BUCKET,
         Key: key,
-      }).promise();
+      }).createReadStream();
 
       console.log(`adding key:${key} to ipfs...`);
-      const cid = await http.add(Body);
+      const cid = await http.add(s3ReadStream, {
+        progress: (progress) => {
+          const progressPercent = Math.floor(progress/size * 100);
+          console.log(`ipfs upload progress ${progressPercent}% ${fileName}`);
+        }
+      });
+      console.log('file added to ipfs.');
 
       const cidHash = cid[0].hash
-  
+      console.log(`adding cid file: ${key}.cid`);
-      console.log(`added cid file: ${key}.cid`);
+      await s3.putObject({
-      
-      const s3Ack = await s3.putObject({
         Bucket: process.env.AWS_DIST_BUCKET,
         Key: fileName + ".cid",
         Body: cidHash
       }).promise()
-      
+      console.log('cid file added.');
-      console.log({s3Ack})
     } catch (error) {
       console.error(error);
     }
   }
 }
 
+
 export const main = ipfsUpload;

lambdas/src/functions/ipfsUpload/index.ts

@@ -28,6 +28,42 @@ export default {
         ]
       },
     },
+    {
+      s3: {
+        bucket: '${ssm:AW5_S3_BUCKET}',
+        existing: true,
+        event: 's3:ObjectCreated:*',
+        rules: [
+          {
+            suffix: '.sha256sum'
+          },
+        ]
+      },
+    },
+    {
+      s3: {
+        bucket: '${ssm:AW5_S3_BUCKET}',
+        existing: true,
+        event: 's3:ObjectCreated:*',
+        rules: [
+          {
+            suffix: '.txt'
+          },
+        ]
+      },
+    },
+    {
+      s3: {
+        bucket: '${ssm:AW5_S3_BUCKET}',
+        existing: true,
+        event: 's3:ObjectCreated:*',
+        rules: [
+          {
+            suffix: '.tgz'
+          },
+        ]
+      },
+    },
   ],
   environment: {
     AWS_DIST_BUCKET: '${ssm:AW5_S3_BUCKET}',

lambdas/yarn.lock

@@ -1409,13 +1409,6 @@ emoji-regex@^8.0.0:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
   integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
 
-encoding@^0.1.11:
-  version "0.1.13"
-  resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9"
-  integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==
-  dependencies:
-    iconv-lite "^0.6.2"
-
 end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
@@ -2181,13 +2174,6 @@ iconv-lite@^0.4.24:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
-iconv-lite@^0.6.2:
-  version "0.6.3"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501"
-  integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==
-  dependencies:
-    safer-buffer ">= 2.1.2 < 3.0.0"
-
 ieee754@1.1.13:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
@@ -2555,7 +2541,7 @@ is-shared-array-buffer@^1.0.2:
   dependencies:
     call-bind "^1.0.2"
 
-is-stream@^1.0.1, is-stream@^1.1.0:
+is-stream@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
   integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==
@@ -3279,13 +3265,10 @@ node-dir@^0.1.17:
   dependencies:
     minimatch "^3.0.2"
 
-node-fetch@^1.6.3, node-fetch@^2.6.7:
+node-fetch@2.6.1, node-fetch@^2.6.7:
-  version "1.7.3"
+  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"
-  integrity sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==
+  integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
-  dependencies:
-    encoding "^0.1.11"
-    is-stream "^1.0.1"
 
 node-forge@^0.10.0:
   version "0.10.0"
@@ -3823,7 +3806,7 @@ safe-regex-test@^1.0.0:
     get-intrinsic "^1.1.3"
     is-regex "^1.1.4"
 
-"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.1.0:
+"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==