ivabus/pantry
1
0
Fork 0
mirror of https://github.com/ivabus/pantry synced 2024-11-13 03:55:18 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: add retry to s3 uploads (#102)

Browse source 
* fix: add retry to s3 uploads

* fix: replace deprecated imports

* bumped again

* bump tea cli dependency

* add missing import

* :/

* ignore incorrect ts error
This commit is contained in:
Lino Le Van 2023-01-17 12:05:12 -08:00 committed by GitHub
parent abd9ee8da0
commit ab5c410cfa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 21 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
import-map.json
View file

@ -1,20 +1,20 @@
{ {
"imports": { "imports": {
"prefab": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/prefab/index.ts", "prefab": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/prefab/index.ts",
"semver": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/utils/semver.ts", "semver": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/utils/semver.ts",
"hooks": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/hooks/index.ts", "hooks": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/hooks/index.ts",
"utils": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/utils/index.ts", "utils": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/utils/index.ts",
"path": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/vendor/Path.ts", "path": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/vendor/Path.ts",
"prefab/": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/prefab/", "prefab/": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/prefab/",
"types": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/types.ts", "types": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/types.ts",
"hooks/": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/hooks/", "hooks/": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/hooks/",
"utils/": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.1/src/utils/", "utils/": "https://raw.githubusercontent.com/teaxyz/cli/v0.19.4/src/utils/",
"deno/": "https://deno.land/std@0.156.0/", "deno/": "https://deno.land/std@0.173.0/",
"is_what": "https://deno.land/x/is_what@v4.1.7/src/index.ts", "is_what": "https://deno.land/x/is_what@v4.1.7/src/index.ts",
"cliffy/": "https://deno.land/x/cliffy@v0.25.2/", "cliffy/": "https://deno.land/x/cliffy@v0.25.2/",
"s3": "https://deno.land/x/s3@0.5.0/mod.ts", "s3": "https://deno.land/x/s3@0.5.0/mod.ts",
"outdent": "https://deno.land/x/outdent@v0.8.0/mod.ts", "outdent": "https://deno.land/x/outdent@v0.8.0/mod.ts",
"rimbu/": "https://deno.land/x/rimbu@0.12.3/", "sha256": "https://deno.land/std@0.160.0/hash/sha256.ts",
"retried": "https://deno.land/x/retried@1.0.1/mod.ts" "rimbu/": "https://deno.land/x/rimbu@0.12.3/"
} }
} }

7
scripts/fixup-checksums.ts
View file

@ -10,8 +10,9 @@ args:
---*/ ---*/
import { S3, S3Object } from "s3" import { S3, S3Object } from "s3"
import { Sha256 } from "deno/hash/sha256.ts" import { crypto, toHashString } from "deno/crypto/mod.ts";
import { readerFromStreamReader, readAll } from "deno/streams/conversion.ts" import { readerFromStreamReader } from "deno/streams/reader_from_stream_reader.ts"
import { readAll } from "deno/streams/read_all.ts"
import Path from "path" import Path from "path"
const s3 = new S3({ const s3 = new S3({
@ -33,7 +34,7 @@ for await (const pkg of bucket.listAllObjects({ batchSize: 200 })) {
const reader = (await bucket.getObject(keys.bottle.string))!.body.getReader() const reader = (await bucket.getObject(keys.bottle.string))!.body.getReader()
const contents = await readAll(readerFromStreamReader(reader)) const contents = await readAll(readerFromStreamReader(reader))
const sha256sum = new Sha256().update(contents).toString() const sha256sum = toHashString(await crypto.subtle.digest("SHA-256", contents))
const body = new TextEncoder().encode(`${sha256sum} ${keys.bottle.basename()}`) const body = new TextEncoder().encode(`${sha256sum} ${keys.bottle.basename()}`)
await bucket.putObject(keys.checksum.string, body) await bucket.putObject(keys.checksum.string, body)

4
scripts/upload-sync.ts
View file

@ -13,7 +13,7 @@ args:
import { readAll, readerFromStreamReader } from "deno/streams/mod.ts" import { readAll, readerFromStreamReader } from "deno/streams/mod.ts"
import { useCache, useOffLicense } from "hooks" import { useCache, useOffLicense } from "hooks"
import { Package } from "types" import { Package } from "types"
import { Sha256 } from "deno/hash/sha256.ts" import { crypto, toHashString } from "deno/crypto/mod.ts";
import { S3 } from "s3" import { S3 } from "s3"
import Path from "path" import Path from "path"
@ -37,7 +37,7 @@ for (const stowed of await useCache().ls()) {
// path.read() returns a string; this is easier to get a UInt8Array // path.read() returns a string; this is easier to get a UInt8Array
const contents = await Deno.readFile(stowed.path.string) const contents = await Deno.readFile(stowed.path.string)
const sha256sum = new Sha256().update(contents).toString() const sha256sum = toHashString(await crypto.subtle.digest("SHA-256", contents))
if (!inRepo || repoChecksum !== sha256sum) { if (!inRepo || repoChecksum !== sha256sum) {
const basename = url.path().basename() const basename = url.path().basename()

4
scripts/upload.ts
View file

@ -16,6 +16,7 @@ import { useCache, useFlags, useOffLicense, usePrefix } from "hooks"
import { Package, PackageRequirement } from "types" import { Package, PackageRequirement } from "types"
import SemVer, * as semver from "semver" import SemVer, * as semver from "semver"
import { basename, dirname } from "deno/path/mod.ts" import { basename, dirname } from "deno/path/mod.ts"
import { retry } from "deno/async/retry.ts"
import { decode as base64Decode } from "deno/encoding/base64.ts" import { decode as base64Decode } from "deno/encoding/base64.ts"
import Path from "path" import Path from "path"
import { set_output } from "./utils/gha.ts" import { set_output } from "./utils/gha.ts"
@ -77,7 +78,8 @@ async function put(key: string, body: string | Path | Uint8Array, bucket: S3Buck
} else if (typeof body === "string") { } else if (typeof body === "string") {
body = encode(body) body = encode(body)
} }
return bucket.putObject(key, body) // @ts-ignore typescript doesn't narrow the types properly here
return retry(()=>bucket.putObject(key, body))
} }
//------------------------------------------------------------------------- main //------------------------------------------------------------------------- main