From 2885e9448aa335f2cb5f1806c2bdcfb230635ea3 Mon Sep 17 00:00:00 2001 From: neil Date: Fri, 28 Oct 2022 12:31:20 +0800 Subject: [PATCH] bugfix: enable uploading large file size via stream. added .tgz suffix trigger --- lambdas/package.json | 4 +-- lambdas/src/functions/ipfsUpload/handler.ts | 26 ++++++++++-------- lambdas/src/functions/ipfsUpload/index.ts | 12 +++++++++ lambdas/yarn.lock | 29 +++++---------------- 4 files changed, 35 insertions(+), 36 deletions(-) diff --git a/lambdas/package.json b/lambdas/package.json index a3a5a8e..53da409 100644 --- a/lambdas/package.json +++ b/lambdas/package.json @@ -10,7 +10,7 @@ "node": ">=14.15.0" }, "resolutions": { - "node-fetch": "^1.6.3" + "node-fetch": "2.6.1" }, "dependencies": { "@middy/core": "^3.4.0", @@ -23,7 +23,7 @@ "compare-versions": "^5.0.1", "ipfs-http-client": "^33.1.1", "lodash": "^4.17.21", - "node-fetch": "^1.6.3" + "node-fetch": "2.6.1" }, "devDependencies": { "@serverless/typescript": "^3.0.0", diff --git a/lambdas/src/functions/ipfsUpload/handler.ts b/lambdas/src/functions/ipfsUpload/handler.ts index 8e51b20..307dbbc 100644 --- a/lambdas/src/functions/ipfsUpload/handler.ts +++ b/lambdas/src/functions/ipfsUpload/handler.ts @@ -11,37 +11,41 @@ const ipfsUpload = async (event: S3CreateEvent) => { // S3 interaction for (const record of event.Records){ try { - const { key } = record.s3.object; + const { key, size } = record.s3.object; const isPackage = key.split('/').length > 1; if(!isPackage) continue // const objectURL = "https://" + process.env.BUCKET + ".s3.amazonaws.com/" + record.s3.object.key const fileName = decodeURIComponent(record.s3.object.key.replace(/\+/g, " ")) - // download file to /tmp - const { Body } = await s3.getObject({ + // download file to /tmp + const s3ReadStream = s3.getObject({ Bucket: process.env.AWS_DIST_BUCKET, Key: key, - }).promise(); + }).createReadStream(); console.log(`adding key:${key} to ipfs...`); - const cid = await http.add(Body); + const cid = await http.add(s3ReadStream, { + progress: (progress) => { + const progressPercent = Math.floor(progress/size * 100); + console.log(`ipfs upload progress ${progressPercent}% ${fileName}`); + } + }); + console.log('file added to ipfs.'); const cidHash = cid[0].hash - - console.log(`added cid file: ${key}.cid`); - - const s3Ack = await s3.putObject({ + console.log(`adding cid file: ${key}.cid`); + await s3.putObject({ Bucket: process.env.AWS_DIST_BUCKET, Key: fileName + ".cid", Body: cidHash }).promise() - - console.log({s3Ack}) + console.log('cid file added.'); } catch (error) { console.error(error); } } } + export const main = ipfsUpload; \ No newline at end of file diff --git a/lambdas/src/functions/ipfsUpload/index.ts b/lambdas/src/functions/ipfsUpload/index.ts index 507a982..05af0ef 100644 --- a/lambdas/src/functions/ipfsUpload/index.ts +++ b/lambdas/src/functions/ipfsUpload/index.ts @@ -52,6 +52,18 @@ export default { ] }, }, + { + s3: { + bucket: '${ssm:AW5_S3_BUCKET}', + existing: true, + event: 's3:ObjectCreated:*', + rules: [ + { + suffix: '.tgz' + }, + ] + }, + }, ], environment: { AWS_DIST_BUCKET: '${ssm:AW5_S3_BUCKET}', diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 0542dda..1da828c 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -1409,13 +1409,6 @@ emoji-regex@^8.0.0: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== -encoding@^0.1.11: - version "0.1.13" - resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9" - integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== - dependencies: - iconv-lite "^0.6.2" - end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -2181,13 +2174,6 @@ iconv-lite@^0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" -iconv-lite@^0.6.2: - version "0.6.3" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" - integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== - dependencies: - safer-buffer ">= 2.1.2 < 3.0.0" - ieee754@1.1.13: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" @@ -2555,7 +2541,7 @@ is-shared-array-buffer@^1.0.2: dependencies: call-bind "^1.0.2" -is-stream@^1.0.1, is-stream@^1.1.0: +is-stream@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ== @@ -3279,13 +3265,10 @@ node-dir@^0.1.17: dependencies: minimatch "^3.0.2" -node-fetch@^1.6.3, node-fetch@^2.6.7: - version "1.7.3" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" - integrity sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ== - dependencies: - encoding "^0.1.11" - is-stream "^1.0.1" +node-fetch@2.6.1, node-fetch@^2.6.7: + version "2.6.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" + integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== node-forge@^0.10.0: version "0.10.0" @@ -3823,7 +3806,7 @@ safe-regex-test@^1.0.0: get-intrinsic "^1.1.3" is-regex "^1.1.4" -"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.1.0: +"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==