init step for ingesting/indexing package details (#144)

* #143 init step for ingesting package details

* #143 create reusable workflow for triggering package ingestion/indexing

* space

* #143 create task after uploading bottles

* cleanup

---------

Co-authored-by: neil <neil@neils-MacBook-Pro.local>

.github/workflows/index-data.yml

@@ -0,0 +1,25 @@
+name: index-data
+
+on:
+  workflow_call:
+    inputs:
+      projects:
+        required: true
+        type: string
+
+env:
+  TEA_PANTRY_PATH: ${{ github.workspace }}/pantry
+
+jobs:
+  queue_detail_ingestion:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: teaxyz/setup@v0
+      - run: ./scripts/index-packages.ts ${{ inputs.projects }}
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-east-1
+          TEA_PANTRY_PATH: ${{ env.TEA_PANTRY_PATH }}
+          SQS_GENERATE_PACKAGE_DETAILS_URL: ${{ secrets.SQS_GENERATE_PACKAGE_DETAILS_URL }}

.github/workflows/new-version.yml

@@ -24,7 +24,12 @@ jobs:
     with:
       new-version: true
     secrets: inherit
-
+  index_data:
+    needs: [bottle]
+    uses: ./.github/workflows/index-data.yml
+    with:
+      projects: ${{ inputs.projects }}
+    secrets: inherit
   complain:
     if: failure()
     needs: [build, bottle]

scripts/index-packages.ts

@@ -0,0 +1,42 @@
+#!/usr/bin/env -S tea -E
+
+/*---
+args:
+  - deno
+  - run
+  - --allow-read
+  - --allow-env
+  - --allow-net
+  - --allow-sys
+  - --import-map={{ srcroot }}/import-map.json
+---*/
+
+import * as semver from "semver"
+import { usePantry } from "hooks"
+import * as ARGV from "./utils/args.ts"
+import  { SQSClient, SendMessageCommand } from "npm:@aws-sdk/client-sqs"
+
+const sqsClient = new SQSClient({ region: 'us-east-1' })
+const pantry = usePantry()
+
+const pkgs = await ARGV.toArray(ARGV.pkgs())
+for(const { project } of pkgs) {
+  try {
+    const yml = await pantry.getYAML({ project, constraint: new semver.Range('*') }).parse()
+
+    const taskMessage = {
+      project,
+      github: yml?.versions?.github || "",
+      // TODO: add other useable data here eventually
+    }
+    const res = await sqsClient.send(new SendMessageCommand({
+      MessageGroupId: 'project',
+      MessageDeduplicationId: project,
+      MessageBody: JSON.stringify(taskMessage),
+      QueueUrl: Deno.env.get("SQS_GENERATE_PACKAGE_DETAILS_URL")!,
+    }))
+    console.log(`SQS task for pkg:${project} messageId:${res.MessageId}`)
+  } catch (error) {
+    console.error(error);
+  }
+}