Merge pull request #122 from teaxyz/feature/init-packages-json-builder

init sls lambdas for building packages.json
This commit is contained in:
Neil 2022-10-20 21:42:22 +08:00 committed by GitHub
commit 2ea0c95814
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 4326 additions and 3 deletions

30
.github/prepare-thumb-images.sh vendored Executable file
View file

@ -0,0 +1,30 @@
#!/bin/bash
# cmd: $ ./prepare-thumb-images.sh srcJson targetPath
# sample: $ .github/prepare-thumb-images.sh src/data/packages.json src/static/Images/packages/
mkdir -p $2
packages="cat $1"
temp_packages="./src/data/temp.json"
cp $1 $temp_packages
for row in $($packages | jq -r '.[] | @base64'); do
_jq() {
echo ${row} | base64 --decode | jq -r ${1}
}
dl_url=$(_jq '.thumb_image_url')
if [[ $dl_url == *"https://dl.airtable.com/.attachments"* ]]; then
filename=$(basename -- "$dl_url")
extension="${filename##*.}"
filename="${filename%.*}"
slug=$(_jq '.slug')
outputPath=$2/$(_jq '.slug').$extension
new_thumb_image_url="/Images/packages/$slug.$extension"
curl $dl_url -o $2/$slug.$extension
updated_packages=$(jq '(.[] | select(.slug == "'$slug'") | .thumb_image_url) |= "'$new_thumb_image_url'"' ./src/data/temp.json)
echo $updated_packages > $temp_packages
echo "update $slug"
fi
done
# replace packages.json with the updated version with localized image links
mv $temp_packages $1

View file

@ -6,8 +6,29 @@ concurrency:
env: env:
TEA_SECRET: ${{ secrets.TEA_SECRET }} TEA_SECRET: ${{ secrets.TEA_SECRET }}
AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
jobs: jobs:
deploy_sls:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- uses: borales/actions-yarn@v3.0.0
with:
cmd: --cwd ./lambdas install
- name: serverless deploy
uses: serverless/github-action@v3
with:
args: -c "cd ./lambdas && serverless deploy"
entrypoint: /bin/sh
deploy: deploy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
@ -43,10 +64,15 @@ jobs:
- name: Create Package Detail Pages from packages.json - name: Create Package Detail Pages from packages.json
uses: getneil/jq-action@v1 uses: getneil/jq-action@v1
id: version
with: with:
cmd: '.github/build-package-pages.sh src/data/packages.json src/content/packages' cmd: '.github/build-package-pages.sh src/data/packages.json src/content/packages'
- name: Download package thumbnail images
uses: getneil/jq-action@v1
with:
cmd: '.github/prepare-thumb-images.sh src/data/packages.json ./src/static/Images/packages/'
- run: .github/mk-pantry-accessible.sh ${{ secrets.TEMP_JACOBS_GITHUB_PAT }} - run: .github/mk-pantry-accessible.sh ${{ secrets.TEMP_JACOBS_GITHUB_PAT }}
- uses: teaxyz/setup@v0 - uses: teaxyz/setup@v0

1
.gitignore vendored
View file

@ -27,3 +27,4 @@ cdk.out
# initially committed _sample.md only the rest will be ignored # initially committed _sample.md only the rest will be ignored
src/content/packages/* src/content/packages/*
src/static/Images/packages/*

View file

@ -16,7 +16,8 @@ The tea logo and wordmark are registered trademarks of tea.inc.
* Repeated components are in [`./src/layouts/partials/`]. * Repeated components are in [`./src/layouts/partials/`].
## Getting Started ## Getting Started
Just once assuming there are no updates in `/src/data/packages.json`. This will create the package detail pages. The detail pages of each package are not committed to the repository for the simple reason of that would be too much to much repeating data.
Execute the following command just once per version of `/src/data/packages.json`. This will create the package detail pages in `/src/content/packages/[package_slug].md`.
```sh ```sh
.github/build-package-pages.sh src/data/packages.json src/content/packages .github/build-package-pages.sh src/data/packages.json src/content/packages
``` ```

9
lambdas/.gitignore vendored Normal file
View file

@ -0,0 +1,9 @@
# package directories
node_modules
jspm_packages
# Serverless directories
.serverless
# esbuild directories
.esbuild

1
lambdas/.nvmrc Normal file
View file

@ -0,0 +1 @@
lts/fermium

90
lambdas/README.md Normal file
View file

@ -0,0 +1,90 @@
# Serverless - AWS Node.js Typescript
This project has been generated using the `aws-nodejs-typescript` template from the [Serverless framework](https://www.serverless.com/).
For detailed instructions, please refer to the [documentation](https://www.serverless.com/framework/docs/providers/aws/).
## Installation/deployment instructions
Depending on your preferred package manager, follow the instructions below to deploy your project.
> **Requirements**: NodeJS `lts/fermium (v.14.15.0)`. If you're using [nvm](https://github.com/nvm-sh/nvm), run `nvm use` to ensure you're using the same Node version in local and in your lambda's runtime.
### Using Yarn
- Run `yarn` to install the project dependencies
- Run `yarn sls deploy` to deploy this stack to AWS
## Test your service
This template contains a single lambda function triggered by an HTTP request made on the provisioned API Gateway REST API `/hello` route with `POST` method. The request body must be provided as `application/json`. The body structure is tested by API Gateway against `src/functions/hello/schema.ts` JSON-Schema definition: it must contain the `name` property.
- requesting any other path than `/hello` with any other method than `POST` will result in API Gateway returning a `403` HTTP error code
- sending a `POST` request to `/hello` with a payload **not** containing a string property named `name` will result in API Gateway returning a `400` HTTP error code
- sending a `POST` request to `/hello` with a payload containing a string property named `name` will result in API Gateway returning a `200` HTTP status code with a message saluting the provided name and the detailed event processed by the lambda
> :warning: As is, this template, once deployed, opens a **public** endpoint within your AWS account resources. Anybody with the URL can actively execute the API Gateway endpoint and the corresponding lambda. You should protect this endpoint with the authentication method of your choice.
### Locally
In order to test the hello function locally, run the following command:
- `npx sls invoke local -f hello --path src/functions/hello/mock.json` if you're using NPM
- `yarn sls invoke local -f hello --path src/functions/hello/mock.json` if you're using Yarn
Check the [sls invoke local command documentation](https://www.serverless.com/framework/docs/providers/aws/cli-reference/invoke-local/) for more information.
### Remotely
Copy and replace your `url` - found in Serverless `deploy` command output - and `name` parameter in the following `curl` command in your terminal or in Postman to test your newly deployed application.
```
curl --location --request POST 'https://myApiEndpoint/dev/hello' \
--header 'Content-Type: application/json' \
--data-raw '{
"name": "Frederic"
}'
```
## Template features
### Project structure
The project code base is mainly located within the `src` folder. This folder is divided in:
- `functions` - containing code base and configuration for your lambda functions
- `libs` - containing shared code base between your lambdas
```
.
├── src
│ ├── functions # Lambda configuration and source code folder
│ │ ├── hello
│ │ │ ├── handler.ts # `Hello` lambda source code
│ │ │ ├── index.ts # `Hello` lambda Serverless configuration
│ │ │ ├── mock.json # `Hello` lambda input parameter, if any, for local invocation
│ │ │ └── schema.ts # `Hello` lambda input event JSON-Schema
│ │ │
│ │ └── index.ts # Import/export of all lambda configurations
│ │
│ └── libs # Lambda shared code
│ └── apiGateway.ts # API Gateway specific helpers
│ └── handlerResolver.ts # Sharable library for resolving lambda handlers
│ └── lambda.ts # Lambda middleware
├── package.json
├── serverless.ts # Serverless service file
├── tsconfig.json # Typescript compiler configuration
├── tsconfig.paths.json # Typescript paths
└── webpack.config.js # Webpack configuration
```
### 3rd party libraries
- [json-schema-to-ts](https://github.com/ThomasAribart/json-schema-to-ts) - uses JSON-Schema definitions used by API Gateway for HTTP request validation to statically generate TypeScript types in your lambda's handler code base
- [middy](https://github.com/middyjs/middy) - middleware engine for Node.Js lambda. This template uses [http-json-body-parser](https://github.com/middyjs/middy/tree/master/packages/http-json-body-parser) to convert API Gateway `event.body` property, originally passed as a stringified JSON, to its corresponding parsed object
- [@serverless/typescript](https://github.com/serverless/typescript) - provides up-to-date TypeScript definitions for your `serverless.ts` service file
### Advanced usage
Any tsconfig.json can be used, but if you do, set the environment variable `TS_NODE_CONFIG` for building the application, eg `TS_NODE_CONFIG=./tsconfig.app.json npx serverless webpack`

36
lambdas/package.json Normal file
View file

@ -0,0 +1,36 @@
{
"name": "lambdas",
"version": "1.0.0",
"description": "Serverless aws-nodejs-typescript template",
"main": "serverless.ts",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"engines": {
"node": ">=14.15.0"
},
"dependencies": {
"@middy/core": "^3.4.0",
"@middy/http-json-body-parser": "^3.4.0",
"airtable": "^0.11.5",
"algoliasearch": "^4.14.2",
"aws-sdk": "^2.1234.0",
"axios": "^1.1.3",
"compare-versions": "^5.0.1",
"lodash": "^4.17.21"
},
"devDependencies": {
"@serverless/typescript": "^3.0.0",
"@types/aws-lambda": "^8.10.71",
"@types/node": "^14.14.25",
"esbuild": "^0.14.11",
"json-schema-to-ts": "^1.5.0",
"serverless": "^3.0.0",
"serverless-esbuild": "^1.23.3",
"ts-node": "^10.4.0",
"tsconfig-paths": "^3.9.0",
"typescript": "^4.1.3"
},
"author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)",
"license": "MIT"
}

62
lambdas/serverless.ts Normal file
View file

@ -0,0 +1,62 @@
import type { AWS } from '@serverless/typescript';
import buildPackages from '@functions/buildPackages';
const serverlessConfiguration: AWS = {
service: 'lambdas',
frameworkVersion: '3',
plugins: ['serverless-esbuild'],
provider: {
name: 'aws',
runtime: 'nodejs14.x',
apiGateway: {
minimumCompressionSize: 1024,
shouldStartNameWithService: true,
},
environment: {
AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
NODE_OPTIONS: '--enable-source-maps --stack-trace-limit=1000',
AWS_DIST_BUCKET: '',
},
iam: {
deploymentRole: `arn:aws:iam::${process.env.AWS_ACCOUNT_ID || '640264234305'}:role/CloudFormationExecutionRole`,
role: {
statements: [
{
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::dist.tea.xyz",
"arn:aws:s3:::dist.tea.xyz/*",
"arn:aws:s3:::dist.tea.xyz/*/*",
],
"Action": [
"s3:GetBucketAcl",
"s3:List",
"s3:ListBucket",
"s3:PutObject"
]
}
]
}
}
},
// import the function via paths
functions: {
buildPackages,
},
package: { individually: true },
custom: {
esbuild: {
bundle: true,
minify: false,
sourcemap: true,
exclude: ['aws-sdk'],
target: 'node14',
define: { 'require.resolve': undefined },
platform: 'node',
concurrency: 10,
},
},
};
module.exports = serverlessConfiguration;

View file

@ -0,0 +1,92 @@
import type { Package, AirtablePackage, S3Package } from '@libs/types';
import _ from 'lodash';
import { getAllS3Packages, writePackagesToS3 } from '@libs/dist_tea_xyz';
import { getAllAirtablePackages, insertPackagesToAirtable } from '@libs/airtable';
import { getBestMatchingIndexedPackage } from '@libs/algolia';
type NewAirtablePackage = Partial<AirtablePackage>;
const buildPackages = async () => {
try {
const [
allS3Packages,
airtablePackages
] = await Promise.all([
getAllS3Packages(),
getAllAirtablePackages(),
]);
const {
newPackages,
packagesJson,
} = await getFinalPackagesData(allS3Packages, airtablePackages);
await Promise.all([
insertPackagesToAirtable(newPackages),
writePackagesToS3(packagesJson),
]);
} catch (error) {
console.error(error);
}
};
interface FinalPackageOutput {
newPackages: NewAirtablePackage[],
packagesJson: Package[],
}
interface AirtablePackageDict {
[slug: string]: AirtablePackage
}
const getFinalPackagesData = async (s3Packages: S3Package[], airtablePackages: AirtablePackage[]): Promise<FinalPackageOutput> => {
const newPackages: NewAirtablePackage[] = [];
const packagesJson: Package[] = [];
const airtablePackagesDict: AirtablePackageDict = airtablePackages.reduce(
(dict: AirtablePackageDict, p: AirtablePackage) => {
dict[p.slug] = p;
return dict;
},
{}
);
for(const s3Package of s3Packages) {
const airtablePackage = airtablePackagesDict[s3Package.slug];
if (airtablePackage) {
const finalPackage: Package = {
...airtablePackage,
installs: 0, // temporary get this from tea db/ipfs eventually
thumb_image_url: airtablePackage.thumb_image_url || '/Images/package-thumb-nolabel4.jpg',
}
packagesJson.push(finalPackage);
} else {
const matchingIndexedPackage = await getBestMatchingIndexedPackage(s3Package.full_name);
const desc = matchingIndexedPackage ? matchingIndexedPackage.desc : '';
const homepage = s3Package.homepage || _.get(matchingIndexedPackage, 'homepage', '');
const newPackage: NewAirtablePackage = {
...s3Package,
desc,
homepage,
}
const tempPackage: Package = {
...s3Package,
homepage,
desc,
installs: 0, // TODO: get from algolia
thumb_image_url: '',
}
newPackages.push(newPackage);
packagesJson.push(tempPackage);
}
}
return {
newPackages,
packagesJson,
}
}
export const main = buildPackages;

View file

@ -0,0 +1,14 @@
// import schema from './schema';
import { handlerPath } from '@libs/handler-resolver';
export default {
handler: `${handlerPath(__dirname)}/handler.main`,
events: [],
environment: {
AIRTABLE_API_KEY: '${ssm:/airtable/api_key}',
AIRTABLE_PACKAGES_BASE: '${ssm:/airtable/packages_base}',
AWS_DIST_BUCKET: '${ssm:AW5_S3_BUCKET}',
ALGOLIA_APP_ID: '${ssm:/algolia/app_id}',
ALGOLIA_SEARCH_API_KEY: '${ssm:/algolia/search_api_key}',
}
};

View file

@ -0,0 +1,6 @@
{
"headers": {
"Content-Type": "application/json"
},
"body": "{\"name\": \"Frederic\"}"
}

View file

@ -0,0 +1,7 @@
export default {
type: "object",
properties: {
name: { type: 'string' }
},
required: ['name']
} as const;

View file

@ -0,0 +1 @@
export { default as buildPackages } from './buildPackages';

View file

@ -0,0 +1,90 @@
import { base, FieldSet } from 'airtable';
import _ from 'lodash';
import type { AirtablePackage } from './types';
const airtablePackagesBase = base(process.env.AIRTABLE_PACKAGES_BASE);
export const getAllAirtablePackages = async (): Promise<AirtablePackage[]> => {
const allRecords = await airtablePackagesBase('packages')
.select({
maxRecords: 100,
view: '_api'
}).all();
const packages: AirtablePackage[] = allRecords.map((record) => {
return {
airtable_record_id: record.id,
..._.pick(record.fields, [
'slug',
'homepage',
'maintainer',
'name',
'version',
'last_modified',
'full_name',
]),
maintainer: record.fields?.maintainer || '',
desc: record.fields?.desc || '',
thumb_image_url: _.get(record.fields, 'thumb_image[0].url', '/Images/package-thumb-nolabel3.jpg')
} as AirtablePackage;
});
/**
* // SAMPLE RECORD SHAPE w/ thumb_image is uploaded
{
slug: 'unicode_org',
name: 'unicode.org',
full_name: 'unicode.org',
homepage: 'https://unicode.org',
version: '71.1.1',
last_modified: '2022-09-26T19:46:25.000Z',
thumb_image: [
{
id: 'attQVgaRUXOYinsWy',
width: 640,
height: 534,
url: 'https://dl.airtable.com/.attachments/f2465c36a0060919368e2f53305694f9/cfab76a8/gen-art-1.png',
filename: 'gen-art-1.png',
size: 184878,
type: 'image/png',
thumbnails: [Object]
}
]
}
TODO IMAGE UPLOAD to S3/CDN:
if thumb_image_url is empty
get thumb_image data
upload to s3
update thumb_image_url in airtable
*/
return packages;
}
type NewPackageRecord = {
fields: Partial<FieldSet>
}
export const insertPackagesToAirtable = async (newPackages: Partial<AirtablePackage>[]) => {
console.log(`airtable: inserting new packages(${newPackages.length})`);
try {
const newRecords: NewPackageRecord[] = newPackages.map((fields) => {
return {
fields: {
...fields,
last_modified: fields.last_modified.toString(),
}
}
});
// airtable can only insert 10 at a time
const insertBatches = _.chunk(newRecords, 10);
for(const batch of insertBatches) {
await airtablePackagesBase('packages').create(batch);
}
console.info(`airtable: new packages(${newPackages.length}) inserted`)
} catch (error) {
console.error(error);
console.log(`airtable: failed to insert packages(${newPackages.length})!`);
}
}

View file

@ -0,0 +1,32 @@
import algoliasearch from 'algoliasearch';
import _ from 'lodash';
import type { AlgoliaIndexedPackage } from './types';
const appId = process.env.ALGOLIA_APP_ID;
const searchApiKey = process.env.ALGOLIA_SEARCH_API_KEY;
const client = algoliasearch(appId, searchApiKey);
const packagesIndex = client.initIndex('dev_packages');
export const getBestMatchingIndexedPackage = async (name: string): Promise<AlgoliaIndexedPackage | void> => {
const { hits: [bestMatch] } = await packagesIndex.search(name);
if (bestMatch) {
/**
* sample hit:
* {
name: 'pyyaml',
full_name: 'pyyaml',
desc: 'YAML framework for Python',
homepage: 'https://pyyaml.org',
version: '6.0',
objectID: 'pyyaml',
_highlightResult: [Object]
}
*/
return {
..._.omit(bestMatch, ['_highlightResult']),
} as unknown as AlgoliaIndexedPackage;
}
}

View file

@ -0,0 +1,12 @@
import type { APIGatewayProxyEvent, APIGatewayProxyResult, Handler } from "aws-lambda"
import type { FromSchema } from "json-schema-to-ts";
type ValidatedAPIGatewayProxyEvent<S> = Omit<APIGatewayProxyEvent, 'body'> & { body: FromSchema<S> }
export type ValidatedEventAPIGatewayProxyEvent<S> = Handler<ValidatedAPIGatewayProxyEvent<S>, APIGatewayProxyResult>
export const formatJSONResponse = (response: Record<string, unknown>) => {
return {
statusCode: 200,
body: JSON.stringify(response)
}
}

View file

@ -0,0 +1,85 @@
import S3 from 'aws-sdk/clients/s3';
import { compareVersions, validate } from 'compare-versions';
import _ from 'lodash';
import type { S3Package } from './types';
const Bucket = process.env.AWS_DIST_BUCKET;
const s3 = new S3();
export const getAllS3Packages = async (): Promise<S3Package[]> => {
const allS3PackagesWithDups = await getKeysFromS3();
const sortedByVersion = allS3PackagesWithDups
.sort((a, b) => compareVersions(a.version, b.version))
.reverse();
const uniquePackages = _(sortedByVersion)
.uniqBy('name')
.value();
return uniquePackages;
}
const getKeysFromS3 = async (ContinuationToken?: string) : Promise<S3Package[]> => {
const res = await s3.listObjectsV2({
Bucket,
MaxKeys: 2147483647,
...(ContinuationToken ? { ContinuationToken } : {}),
}).promise();
const s3Packages: S3Package[] = res.Contents
.filter((data: S3.Object) => data.Key.split('/').length >= 2)
.map(convertS3ContentTOS3Package);
if (res.IsTruncated && res.NextContinuationToken) {
const nextPackages = await getKeysFromS3(res.NextContinuationToken);
s3Packages.push(...nextPackages);
}
return s3Packages.filter((p) => validate(p.version));
}
const convertS3ContentTOS3Package = (data: S3.Object) : S3Package => {
const pathElements = data.Key.replace('github.com/', '').split('/');
const [rawVersion] = pathElements.pop().split('.tar');
const version = rawVersion.replace('v', '')
.replace('.sha256sum','')
.replace('ersions.txt', '');
const [maintainerOrPackageName, packageName] = pathElements;
const isMaintainer = !packageName ? false :
!['linux','darwin'].includes(packageName);
const fullName = isMaintainer ? [maintainerOrPackageName, packageName].join('/') : maintainerOrPackageName;
return {
slug: fullName.replace(/[^\w\s]/gi, '_').toLocaleLowerCase(),
name: isMaintainer ? packageName : maintainerOrPackageName,
full_name: fullName,
maintainer: isMaintainer ? maintainerOrPackageName : '',
version,
last_modified: data.LastModified,
homepage: getPossibleHomepage(maintainerOrPackageName) || getPossibleHomepage(packageName) || ''
}
}
const getPossibleHomepage = (name: string) => {
return name && name.split('.').length > 1 ? `https://${name}` : ''
}
export const writePackagesToS3 = async (packages: S3Package[]) => {
console.log("uploading!")
const buf = Buffer.from(JSON.stringify(packages));
const data = {
Bucket: 'dist.tea.xyz',
Key: 'mock_packages.json',
Body: buf,
ContentEncoding: 'base64',
ContentType: 'application/json'
};
await s3.putObject(data).promise();
console.log("uploaded!")
}

View file

@ -0,0 +1,3 @@
export const handlerPath = (context: string) => {
return `${context.split(process.cwd())[1].substring(1).replace(/\\/g, '/')}`
};

View file

@ -0,0 +1,6 @@
import middy from "@middy/core"
import middyJsonBodyParser from "@middy/http-json-body-parser"
export const middyfy = (handler) => {
return middy(handler).use(middyJsonBodyParser())
}

26
lambdas/src/libs/types.ts Normal file
View file

@ -0,0 +1,26 @@
export interface S3Package {
slug: string,
version: string,
full_name: string,
name: string,
maintainer: string,
homepage: string,
// key: string,
last_modified: Date | string,
}
export type AirtablePackage = S3Package & {
airtable_record_id: string,
thumb_image_url: string,
desc: string,
}
export type Package = Omit<AirtablePackage, 'airtable_record_id'> & {
airtable_record_id?: string,
installs: number,
}
export type AlgoliaIndexedPackage = Omit<S3Package, 'slug | maintainer | last_modified'> & {
objectID: string;
desc: string;
}

25
lambdas/tsconfig.json Normal file
View file

@ -0,0 +1,25 @@
{
"extends": "./tsconfig.paths.json",
"compilerOptions": {
"lib": ["ESNext"],
"moduleResolution": "node",
"noUnusedLocals": true,
"noUnusedParameters": true,
"removeComments": true,
"sourceMap": true,
"target": "ES2020",
"outDir": "lib",
"allowSyntheticDefaultImports": true
},
"include": ["src/**/*.ts", "serverless.ts"],
"exclude": [
"node_modules/**/*",
".serverless/**/*",
".webpack/**/*",
"_warmup/**/*",
".vscode/**/*"
],
"ts-node": {
"require": ["tsconfig-paths/register"]
}
}

View file

@ -0,0 +1,9 @@
{
"compilerOptions": {
"baseUrl": ".",
"paths": {
"@functions/*": ["src/functions/*"],
"@libs/*": ["src/libs/*"]
}
}
}

3659
lambdas/yarn.lock Normal file

File diff suppressed because it is too large Load diff