mirror of
https://github.com/ivabus/www
synced 2024-11-10 06:25:14 +03:00
remove lambdas moved to kettle repo
This commit is contained in:
neil
parent
7d23dedca1
commit
189d6fd410
45
.github/workflows/staging.yml
vendored
45
.github/workflows/staging.yml
vendored
|
|
@ -9,52 +9,7 @@ env:
|
|||
AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}
|
||||
|
||||
jobs:
|
||||
validation:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
website: ${{steps.website.outputs.src}}
|
||||
lambdas: ${{steps.lambdas.outputs.lambdas}}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dorny/paths-filter@v2
|
||||
id: website
|
||||
with:
|
||||
filters: |
|
||||
src:
|
||||
- 'src/**'
|
||||
- uses: dorny/paths-filter@v2
|
||||
id: lambdas
|
||||
with:
|
||||
filters: |
|
||||
lambdas:
|
||||
- 'lambdas/**'
|
||||
|
||||
serverless:
|
||||
needs: validation
|
||||
if: needs.validation.outputs.lambdas == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: us-east-1
|
||||
|
||||
- uses: borales/actions-yarn@v3.0.0
|
||||
with:
|
||||
cmd: --cwd ./lambdas install
|
||||
|
||||
- name: serverless deploy
|
||||
uses: serverless/github-action@v3
|
||||
with:
|
||||
args: -c "cd ./lambdas && serverless deploy"
|
||||
entrypoint: /bin/sh
|
||||
|
||||
deploy:
|
||||
needs: validation
|
||||
if: needs.validation.outputs.website == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: inject slug/short variables
|
||||
|
|
|
|||
9
lambdas/.gitignore
vendored
9
lambdas/.gitignore
vendored
|
|
@ -1,9 +0,0 @@
|
|||
# package directories
|
||||
node_modules
|
||||
jspm_packages
|
||||
|
||||
# Serverless directories
|
||||
.serverless
|
||||
|
||||
# esbuild directories
|
||||
.esbuild
|
||||
|
|
@ -1 +0,0 @@
|
|||
lts/fermium
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
# Serverless - AWS Node.js Typescript
|
||||
|
||||
This project has been generated using the `aws-nodejs-typescript` template from the [Serverless framework](https://www.serverless.com/).
|
||||
|
||||
For detailed instructions, please refer to the [documentation](https://www.serverless.com/framework/docs/providers/aws/).
|
||||
|
||||
## Installation/deployment instructions
|
||||
|
||||
Depending on your preferred package manager, follow the instructions below to deploy your project.
|
||||
|
||||
> **Requirements**: NodeJS `lts/fermium (v.14.15.0)`. If you're using [nvm](https://github.com/nvm-sh/nvm), run `nvm use` to ensure you're using the same Node version in local and in your lambda's runtime.
|
||||
|
||||
### Using Yarn
|
||||
|
||||
- Run `yarn` to install the project dependencies
|
||||
- Run `yarn sls deploy` to deploy this stack to AWS
|
||||
|
||||
## Test your service
|
||||
|
||||
This template contains a single lambda function triggered by an HTTP request made on the provisioned API Gateway REST API `/hello` route with `POST` method. The request body must be provided as `application/json`. The body structure is tested by API Gateway against `src/functions/hello/schema.ts` JSON-Schema definition: it must contain the `name` property.
|
||||
|
||||
- requesting any other path than `/hello` with any other method than `POST` will result in API Gateway returning a `403` HTTP error code
|
||||
- sending a `POST` request to `/hello` with a payload **not** containing a string property named `name` will result in API Gateway returning a `400` HTTP error code
|
||||
- sending a `POST` request to `/hello` with a payload containing a string property named `name` will result in API Gateway returning a `200` HTTP status code with a message saluting the provided name and the detailed event processed by the lambda
|
||||
|
||||
> :warning: As is, this template, once deployed, opens a **public** endpoint within your AWS account resources. Anybody with the URL can actively execute the API Gateway endpoint and the corresponding lambda. You should protect this endpoint with the authentication method of your choice.
|
||||
|
||||
### Locally
|
||||
|
||||
In order to test the hello function locally, run the following command:
|
||||
|
||||
- `npx sls invoke local -f hello --path src/functions/hello/mock.json` if you're using NPM
|
||||
- `yarn sls invoke local -f hello --path src/functions/hello/mock.json` if you're using Yarn
|
||||
|
||||
Check the [sls invoke local command documentation](https://www.serverless.com/framework/docs/providers/aws/cli-reference/invoke-local/) for more information.
|
||||
|
||||
### Remotely
|
||||
|
||||
Copy and replace your `url` - found in Serverless `deploy` command output - and `name` parameter in the following `curl` command in your terminal or in Postman to test your newly deployed application.
|
||||
|
||||
```
|
||||
curl --location --request POST 'https://myApiEndpoint/dev/hello' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw '{
|
||||
"name": "Frederic"
|
||||
}'
|
||||
```
|
||||
|
||||
## Template features
|
||||
|
||||
### Project structure
|
||||
|
||||
The project code base is mainly located within the `src` folder. This folder is divided in:
|
||||
|
||||
- `functions` - containing code base and configuration for your lambda functions
|
||||
- `libs` - containing shared code base between your lambdas
|
||||
|
||||
```
|
||||
.
|
||||
├── src
|
||||
│ ├── functions # Lambda configuration and source code folder
|
||||
│ │ ├── hello
|
||||
│ │ │ ├── handler.ts # `Hello` lambda source code
|
||||
│ │ │ ├── index.ts # `Hello` lambda Serverless configuration
|
||||
│ │ │ ├── mock.json # `Hello` lambda input parameter, if any, for local invocation
|
||||
│ │ │ └── schema.ts # `Hello` lambda input event JSON-Schema
|
||||
│ │ │
|
||||
│ │ └── index.ts # Import/export of all lambda configurations
|
||||
│ │
|
||||
│ └── libs # Lambda shared code
|
||||
│ └── apiGateway.ts # API Gateway specific helpers
|
||||
│ └── handlerResolver.ts # Sharable library for resolving lambda handlers
|
||||
│ └── lambda.ts # Lambda middleware
|
||||
│
|
||||
├── package.json
|
||||
├── serverless.ts # Serverless service file
|
||||
├── tsconfig.json # Typescript compiler configuration
|
||||
├── tsconfig.paths.json # Typescript paths
|
||||
└── webpack.config.js # Webpack configuration
|
||||
```
|
||||
|
||||
### 3rd party libraries
|
||||
|
||||
- [json-schema-to-ts](https://github.com/ThomasAribart/json-schema-to-ts) - uses JSON-Schema definitions used by API Gateway for HTTP request validation to statically generate TypeScript types in your lambda's handler code base
|
||||
- [middy](https://github.com/middyjs/middy) - middleware engine for Node.Js lambda. This template uses [http-json-body-parser](https://github.com/middyjs/middy/tree/master/packages/http-json-body-parser) to convert API Gateway `event.body` property, originally passed as a stringified JSON, to its corresponding parsed object
|
||||
- [@serverless/typescript](https://github.com/serverless/typescript) - provides up-to-date TypeScript definitions for your `serverless.ts` service file
|
||||
|
||||
### Advanced usage
|
||||
|
||||
Any tsconfig.json can be used, but if you do, set the environment variable `TS_NODE_CONFIG` for building the application, eg `TS_NODE_CONFIG=./tsconfig.app.json npx serverless webpack`
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
{
|
||||
"name": "lambdas",
|
||||
"version": "1.0.0",
|
||||
"description": "Serverless aws-nodejs-typescript template",
|
||||
"main": "serverless.ts",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.15.0"
|
||||
},
|
||||
"resolutions": {
|
||||
"node-fetch": "2.6.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@middy/core": "^3.4.0",
|
||||
"@middy/http-json-body-parser": "^3.4.0",
|
||||
"airtable": "^0.11.5",
|
||||
"algoliasearch": "^4.14.2",
|
||||
"aws-lambda": "^1.0.7",
|
||||
"aws-sdk": "^2.1239.0",
|
||||
"axios": "^1.1.3",
|
||||
"compare-versions": "^5.0.1",
|
||||
"ipfs-http-client": "^33.1.1",
|
||||
"lodash": "^4.17.21",
|
||||
"node-fetch": "2.6.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@serverless/typescript": "^3.0.0",
|
||||
"@types/aws-lambda": "^8.10.71",
|
||||
"@types/node": "^14.14.25",
|
||||
"esbuild": "^0.14.11",
|
||||
"json-schema-to-ts": "^1.5.0",
|
||||
"serverless": "^3.0.0",
|
||||
"serverless-esbuild": "^1.23.3",
|
||||
"serverless-iam-roles-per-function": "^3.2.0",
|
||||
"ts-node": "^10.4.0",
|
||||
"tsconfig-paths": "^3.9.0",
|
||||
"typescript": "^4.1.3"
|
||||
},
|
||||
"author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)",
|
||||
"license": "MIT"
|
||||
}
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
import type { AWS } from '@serverless/typescript';
|
||||
|
||||
import buildPackages from '@functions/buildPackages';
|
||||
import ipfsUpload from '@functions/ipfsUpload';
|
||||
|
||||
const serverlessConfiguration: AWS = {
|
||||
service: 'lambdas',
|
||||
frameworkVersion: '3',
|
||||
plugins: [
|
||||
'serverless-esbuild',
|
||||
'serverless-iam-roles-per-function'
|
||||
],
|
||||
provider: {
|
||||
name: 'aws',
|
||||
runtime: 'nodejs14.x',
|
||||
apiGateway: {
|
||||
minimumCompressionSize: 1024,
|
||||
shouldStartNameWithService: true,
|
||||
},
|
||||
environment: {
|
||||
AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
|
||||
NODE_OPTIONS: '--enable-source-maps --stack-trace-limit=1000',
|
||||
AWS_DIST_BUCKET: '',
|
||||
},
|
||||
iam: {
|
||||
deploymentRole: `arn:aws:iam::${process.env.AWS_ACCOUNT_ID || '640264234305'}:role/CloudFormationExecutionRole`,
|
||||
role: {
|
||||
statements: []
|
||||
}
|
||||
},
|
||||
vpc: {
|
||||
securityGroupIds: [
|
||||
"${ssm:/vpc/sg/serverless_lambdas}",
|
||||
],
|
||||
subnetIds: [
|
||||
"${ssm:/vpc/subnets/private_1}",
|
||||
"${ssm:/vpc/subnets/private_2}"
|
||||
]
|
||||
}
|
||||
},
|
||||
// import the function via paths
|
||||
functions: {
|
||||
buildPackages,
|
||||
ipfsUpload,
|
||||
},
|
||||
package: { individually: true },
|
||||
custom: {
|
||||
esbuild: {
|
||||
bundle: true,
|
||||
minify: false,
|
||||
sourcemap: true,
|
||||
exclude: ['aws-sdk'],
|
||||
target: 'node14',
|
||||
define: { 'require.resolve': undefined },
|
||||
platform: 'node',
|
||||
concurrency: 10,
|
||||
},
|
||||
'serverless-iam-roles-per-function': {
|
||||
// on inherit: try to configure permission per function correctly
|
||||
// TODO: ci/cd puresec-ish auto-auditing permissions
|
||||
defaultInherit: false,
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = serverlessConfiguration;
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
import type { Package, AirtablePackage, S3Package } from '@libs/types';
|
||||
import _ from 'lodash';
|
||||
|
||||
import { getAllS3Packages, writePackagesToS3 } from '@libs/dist_tea_xyz';
|
||||
import { getAllAirtablePackages, insertPackagesToAirtable } from '@libs/airtable';
|
||||
import { getBestMatchingIndexedPackage } from '@libs/algolia';
|
||||
|
||||
type NewAirtablePackage = Partial<AirtablePackage>;
|
||||
|
||||
|
||||
const buildPackages = async () => {
|
||||
try {
|
||||
const [
|
||||
allS3Packages,
|
||||
airtablePackages
|
||||
] = await Promise.all([
|
||||
getAllS3Packages(),
|
||||
getAllAirtablePackages(),
|
||||
]);
|
||||
|
||||
const {
|
||||
newPackages,
|
||||
packagesJson,
|
||||
} = await getFinalPackagesData(allS3Packages, airtablePackages);
|
||||
|
||||
await Promise.all([
|
||||
insertPackagesToAirtable(newPackages),
|
||||
writePackagesToS3(packagesJson),
|
||||
]);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
};
|
||||
|
||||
interface FinalPackageOutput {
|
||||
newPackages: NewAirtablePackage[],
|
||||
packagesJson: Package[],
|
||||
}
|
||||
interface AirtablePackageDict {
|
||||
[slug: string]: AirtablePackage
|
||||
}
|
||||
const placeholderThumbImg = '/Images/package-thumb-nolabel4.jpg';
|
||||
const getFinalPackagesData = async (s3Packages: S3Package[], airtablePackages: AirtablePackage[]): Promise<FinalPackageOutput> => {
|
||||
const newPackages: NewAirtablePackage[] = [];
|
||||
const packagesJson: Package[] = [];
|
||||
|
||||
const airtablePackagesDict: AirtablePackageDict = airtablePackages.reduce(
|
||||
(dict: AirtablePackageDict, p: AirtablePackage) => {
|
||||
dict[p.slug] = p;
|
||||
return dict;
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
for(const s3Package of s3Packages) {
|
||||
const airtablePackage = airtablePackagesDict[s3Package.slug];
|
||||
if (airtablePackage) {
|
||||
const finalPackage: Package = {
|
||||
...airtablePackage,
|
||||
installs: 0, // temporary get this from tea db/ipfs eventually
|
||||
thumb_image_url: airtablePackage.thumb_image_url || placeholderThumbImg,
|
||||
}
|
||||
packagesJson.push(finalPackage);
|
||||
} else {
|
||||
const matchingIndexedPackage = await getBestMatchingIndexedPackage(s3Package.full_name);
|
||||
const desc = matchingIndexedPackage ? matchingIndexedPackage.desc : '';
|
||||
const homepage = s3Package.homepage || _.get(matchingIndexedPackage, 'homepage', '');
|
||||
|
||||
const newPackage: NewAirtablePackage = {
|
||||
...s3Package,
|
||||
desc,
|
||||
homepage,
|
||||
}
|
||||
const tempPackage: Package = {
|
||||
...s3Package,
|
||||
homepage,
|
||||
desc,
|
||||
installs: 0, // TODO: get from algolia
|
||||
dl_count: 0,
|
||||
thumb_image_url: placeholderThumbImg,
|
||||
}
|
||||
newPackages.push(newPackage);
|
||||
packagesJson.push(tempPackage);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
newPackages,
|
||||
packagesJson,
|
||||
}
|
||||
}
|
||||
|
||||
export const main = buildPackages;
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
// import schema from './schema';
|
||||
import { handlerPath } from '@libs/handler-resolver';
|
||||
|
||||
export default {
|
||||
handler: `${handlerPath(__dirname)}/handler.main`,
|
||||
events: [],
|
||||
environment: {
|
||||
AIRTABLE_API_KEY: '${ssm:/airtable/api_key}',
|
||||
AIRTABLE_PACKAGES_BASE: '${ssm:/airtable/packages_base}',
|
||||
AWS_DIST_BUCKET: '${ssm:AW5_S3_BUCKET}',
|
||||
ALGOLIA_APP_ID: '${ssm:/algolia/app_id}',
|
||||
ALGOLIA_SEARCH_API_KEY: '${ssm:/algolia/search_api_key}',
|
||||
},
|
||||
iamRoleStatements: [
|
||||
{
|
||||
Effect: 'Allow',
|
||||
Action: [
|
||||
's3:GetBucketAcl',
|
||||
's3:List',
|
||||
's3:ListBucket',
|
||||
's3:PutObject'
|
||||
],
|
||||
Resource: [
|
||||
"arn:aws:s3:::${ssm:AW5_S3_BUCKET}",
|
||||
"arn:aws:s3:::${ssm:AW5_S3_BUCKET}/*",
|
||||
"arn:aws:s3:::${ssm:AW5_S3_BUCKET}/*/*",
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
{
|
||||
"headers": {
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"body": "{\"name\": \"Frederic\"}"
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
export default {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: { type: 'string' }
|
||||
},
|
||||
required: ['name']
|
||||
} as const;
|
||||
|
|
@ -1 +0,0 @@
|
|||
export { default as buildPackages } from './buildPackages';
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
import AWS from 'aws-sdk';
|
||||
import { S3CreateEvent } from 'aws-lambda';
|
||||
import ipfs from 'ipfs-http-client';
|
||||
import fs from 'fs';
|
||||
|
||||
const s3 = new AWS.S3();
|
||||
|
||||
const ipfsUpload = async (event: S3CreateEvent) => {
|
||||
const http = ipfs(process.env.IPFS_IP4_ADDRESS);
|
||||
|
||||
// S3 interaction
|
||||
for (const record of event.Records){
|
||||
try {
|
||||
const { key, size } = record.s3.object;
|
||||
const isPackage = key.split('/').length > 1;
|
||||
if(!isPackage) continue
|
||||
|
||||
// const objectURL = "https://" + process.env.BUCKET + ".s3.amazonaws.com/" + record.s3.object.key
|
||||
const fileName = decodeURIComponent(record.s3.object.key.replace(/\+/g, " "))
|
||||
|
||||
// download file to /tmp
|
||||
const s3ReadStream = s3.getObject({
|
||||
Bucket: process.env.AWS_DIST_BUCKET,
|
||||
Key: key,
|
||||
}).createReadStream();
|
||||
|
||||
console.log(`adding key:${key} to ipfs...`);
|
||||
const cid = await http.add(s3ReadStream, {
|
||||
progress: (progress) => {
|
||||
const progressPercent = Math.floor(progress/size * 100);
|
||||
console.log(`ipfs upload progress ${progressPercent}% ${fileName}`);
|
||||
}
|
||||
});
|
||||
console.log('file added to ipfs.');
|
||||
|
||||
const cidHash = cid[0].hash
|
||||
console.log(`adding cid file: ${key}.cid`);
|
||||
await s3.putObject({
|
||||
Bucket: process.env.AWS_DIST_BUCKET,
|
||||
Key: fileName + ".cid",
|
||||
Body: cidHash
|
||||
}).promise()
|
||||
console.log('cid file added.');
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export const main = ipfsUpload;
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
// import schema from './schema';
|
||||
import { handlerPath } from '@libs/handler-resolver';
|
||||
|
||||
export default {
|
||||
handler: `${handlerPath(__dirname)}/handler.main`,
|
||||
events: [
|
||||
{
|
||||
s3: {
|
||||
bucket: '${ssm:AW5_S3_BUCKET}',
|
||||
existing: true,
|
||||
event: 's3:ObjectCreated:*',
|
||||
rules: [
|
||||
{
|
||||
suffix: '.gz'
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
{
|
||||
s3: {
|
||||
bucket: '${ssm:AW5_S3_BUCKET}',
|
||||
existing: true,
|
||||
event: 's3:ObjectCreated:*',
|
||||
rules: [
|
||||
{
|
||||
suffix: '.xz'
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
{
|
||||
s3: {
|
||||
bucket: '${ssm:AW5_S3_BUCKET}',
|
||||
existing: true,
|
||||
event: 's3:ObjectCreated:*',
|
||||
rules: [
|
||||
{
|
||||
suffix: '.sha256sum'
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
{
|
||||
s3: {
|
||||
bucket: '${ssm:AW5_S3_BUCKET}',
|
||||
existing: true,
|
||||
event: 's3:ObjectCreated:*',
|
||||
rules: [
|
||||
{
|
||||
suffix: '.txt'
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
{
|
||||
s3: {
|
||||
bucket: '${ssm:AW5_S3_BUCKET}',
|
||||
existing: true,
|
||||
event: 's3:ObjectCreated:*',
|
||||
rules: [
|
||||
{
|
||||
suffix: '.tgz'
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
],
|
||||
environment: {
|
||||
AWS_DIST_BUCKET: '${ssm:AW5_S3_BUCKET}',
|
||||
IPFS_IP4_ADDRESS: '${ssm:/ipfs/ip4_address}',
|
||||
},
|
||||
iamRoleStatements: [
|
||||
{
|
||||
Effect: 'Allow',
|
||||
Action: [
|
||||
's3:GetObject',
|
||||
's3:PutObject',
|
||||
],
|
||||
Resource: [
|
||||
"arn:aws:s3:::${ssm:AW5_S3_BUCKET}",
|
||||
"arn:aws:s3:::${ssm:AW5_S3_BUCKET}/*",
|
||||
"arn:aws:s3:::${ssm:AW5_S3_BUCKET}/*/*",
|
||||
]
|
||||
}
|
||||
]
|
||||
};
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
{
|
||||
"Records": [
|
||||
{
|
||||
"s3": {
|
||||
"object": {
|
||||
"key": "zlib.net/v1.2.12.tar.gz"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"s3": {
|
||||
"object": {
|
||||
"key": "test/test.cid"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"s3": {
|
||||
"object": {
|
||||
"key": "packages.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
import { base, FieldSet } from 'airtable';
|
||||
import _ from 'lodash';
|
||||
import type { AirtablePackage } from './types';
|
||||
|
||||
|
||||
const airtablePackagesBase = base(process.env.AIRTABLE_PACKAGES_BASE);
|
||||
|
||||
|
||||
export const getAllAirtablePackages = async (): Promise<AirtablePackage[]> => {
|
||||
const allRecords = await airtablePackagesBase('packages')
|
||||
.select({
|
||||
maxRecords: 100,
|
||||
view: '_api'
|
||||
}).all();
|
||||
|
||||
const packages: AirtablePackage[] = allRecords.map((record) => {
|
||||
return {
|
||||
airtable_record_id: record.id,
|
||||
..._.pick(record.fields, [
|
||||
'slug',
|
||||
'homepage',
|
||||
'maintainer',
|
||||
'name',
|
||||
'version',
|
||||
'last_modified',
|
||||
'full_name',
|
||||
'dl_count',
|
||||
]),
|
||||
maintainer: record.fields?.maintainer || '',
|
||||
desc: record.fields?.desc || '',
|
||||
thumb_image_url: _.get(record.fields, 'thumb_image[0].url', '/Images/package-thumb-nolabel3.jpg')
|
||||
} as AirtablePackage;
|
||||
});
|
||||
return packages;
|
||||
}
|
||||
|
||||
type NewPackageRecord = {
|
||||
fields: Partial<FieldSet>
|
||||
}
|
||||
|
||||
export const insertPackagesToAirtable = async (newPackages: Partial<AirtablePackage>[]) => {
|
||||
console.log(`airtable: inserting new packages(${newPackages.length})`);
|
||||
try {
|
||||
const newRecords: NewPackageRecord[] = newPackages.map((fields) => {
|
||||
return {
|
||||
fields: {
|
||||
...fields,
|
||||
last_modified: fields.last_modified.toString(),
|
||||
dl_count: 0,
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// airtable can only insert 10 at a time
|
||||
const insertBatches = _.chunk(newRecords, 10);
|
||||
for(const batch of insertBatches) {
|
||||
await airtablePackagesBase('packages').create(batch);
|
||||
}
|
||||
console.info(`airtable: new packages(${newPackages.length}) inserted`)
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
console.log(`airtable: failed to insert packages(${newPackages.length})!`);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
import algoliasearch from 'algoliasearch';
|
||||
import _ from 'lodash';
|
||||
|
||||
import type { AlgoliaIndexedPackage } from './types';
|
||||
|
||||
const appId = process.env.ALGOLIA_APP_ID;
|
||||
const searchApiKey = process.env.ALGOLIA_SEARCH_API_KEY;
|
||||
|
||||
const client = algoliasearch(appId, searchApiKey);
|
||||
|
||||
const packagesIndex = client.initIndex('dev_packages');
|
||||
|
||||
export const getBestMatchingIndexedPackage = async (name: string): Promise<AlgoliaIndexedPackage | void> => {
|
||||
const { hits: [bestMatch] } = await packagesIndex.search(name);
|
||||
if (bestMatch) {
|
||||
/**
|
||||
* sample hit:
|
||||
* {
|
||||
name: 'pyyaml',
|
||||
full_name: 'pyyaml',
|
||||
desc: 'YAML framework for Python',
|
||||
homepage: 'https://pyyaml.org',
|
||||
version: '6.0',
|
||||
objectID: 'pyyaml',
|
||||
_highlightResult: [Object]
|
||||
}
|
||||
*/
|
||||
return {
|
||||
..._.omit(bestMatch, ['_highlightResult']),
|
||||
} as unknown as AlgoliaIndexedPackage;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
import type { APIGatewayProxyEvent, APIGatewayProxyResult, Handler } from "aws-lambda"
|
||||
import type { FromSchema } from "json-schema-to-ts";
|
||||
|
||||
type ValidatedAPIGatewayProxyEvent<S> = Omit<APIGatewayProxyEvent, 'body'> & { body: FromSchema<S> }
|
||||
export type ValidatedEventAPIGatewayProxyEvent<S> = Handler<ValidatedAPIGatewayProxyEvent<S>, APIGatewayProxyResult>
|
||||
|
||||
export const formatJSONResponse = (response: Record<string, unknown>) => {
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: JSON.stringify(response)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
import S3 from 'aws-sdk/clients/s3';
|
||||
import { compareVersions, validate } from 'compare-versions';
|
||||
import _ from 'lodash';
|
||||
import type { S3Package } from './types';
|
||||
|
||||
const Bucket = process.env.AWS_DIST_BUCKET;
|
||||
const s3 = new S3();
|
||||
|
||||
export const getAllS3Packages = async (): Promise<S3Package[]> => {
|
||||
const allS3PackagesWithDups = await getKeysFromS3();
|
||||
|
||||
const sortedByVersion = allS3PackagesWithDups
|
||||
.sort((a, b) => compareVersions(a.version, b.version))
|
||||
.reverse();
|
||||
|
||||
const uniquePackages = _(sortedByVersion)
|
||||
.uniqBy('name')
|
||||
.value();
|
||||
|
||||
return uniquePackages;
|
||||
}
|
||||
|
||||
const getKeysFromS3 = async (ContinuationToken?: string) : Promise<S3Package[]> => {
|
||||
const res = await s3.listObjectsV2({
|
||||
Bucket,
|
||||
MaxKeys: 2147483647,
|
||||
...(ContinuationToken ? { ContinuationToken } : {}),
|
||||
}).promise();
|
||||
|
||||
const s3Packages: S3Package[] = res.Contents
|
||||
.filter((data: S3.Object) => data.Key.split('/').length >= 2)
|
||||
.map(convertS3ContentTOS3Package);
|
||||
|
||||
if (res.IsTruncated && res.NextContinuationToken) {
|
||||
const nextPackages = await getKeysFromS3(res.NextContinuationToken);
|
||||
s3Packages.push(...nextPackages);
|
||||
}
|
||||
|
||||
return s3Packages.filter((p) => validate(p.version));
|
||||
}
|
||||
|
||||
const convertS3ContentTOS3Package = (data: S3.Object) : S3Package => {
|
||||
const pathElements = data.Key.replace('github.com/', '').split('/');
|
||||
const [rawVersion] = pathElements.pop().split('.tar');
|
||||
const version = rawVersion.replace('v', '')
|
||||
.replace('.sha256sum','')
|
||||
.replace('ersions.txt', '');
|
||||
|
||||
const [maintainerOrPackageName, packageName] = pathElements;
|
||||
const isMaintainer = !packageName ? false :
|
||||
!['linux','darwin'].includes(packageName);
|
||||
|
||||
const fullName = isMaintainer ? [maintainerOrPackageName, packageName].join('/') : maintainerOrPackageName;
|
||||
|
||||
return {
|
||||
slug: fullName.replace(/[^\w\s]/gi, '_').toLocaleLowerCase(),
|
||||
name: isMaintainer ? packageName : maintainerOrPackageName,
|
||||
full_name: fullName,
|
||||
maintainer: isMaintainer ? maintainerOrPackageName : '',
|
||||
version,
|
||||
last_modified: data.LastModified,
|
||||
homepage: getPossibleHomepage(maintainerOrPackageName) || getPossibleHomepage(packageName) || ''
|
||||
}
|
||||
}
|
||||
|
||||
const getPossibleHomepage = (name: string) => {
|
||||
return name && name.split('.').length > 1 ? `https://${name}` : ''
|
||||
}
|
||||
|
||||
export const writePackagesToS3 = async (packages: S3Package[]) => {
|
||||
console.log("uploading!")
|
||||
|
||||
const buf = Buffer.from(JSON.stringify(packages));
|
||||
|
||||
const data = {
|
||||
Bucket: 'dist.tea.xyz',
|
||||
Key: 'mock_packages.json',
|
||||
Body: buf,
|
||||
ContentEncoding: 'base64',
|
||||
ContentType: 'application/json'
|
||||
};
|
||||
|
||||
await s3.putObject(data).promise();
|
||||
console.log("uploaded!")
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
export const handlerPath = (context: string) => {
|
||||
return `${context.split(process.cwd())[1].substring(1).replace(/\\/g, '/')}`
|
||||
};
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
import middy from "@middy/core"
|
||||
import middyJsonBodyParser from "@middy/http-json-body-parser"
|
||||
|
||||
export const middyfy = (handler) => {
|
||||
return middy(handler).use(middyJsonBodyParser())
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
export interface S3Package {
|
||||
slug: string,
|
||||
version: string,
|
||||
full_name: string,
|
||||
name: string,
|
||||
maintainer: string,
|
||||
homepage: string,
|
||||
// key: string,
|
||||
last_modified: Date | string,
|
||||
}
|
||||
|
||||
export type AirtablePackage = S3Package & {
|
||||
airtable_record_id: string,
|
||||
thumb_image_url: string,
|
||||
desc: string,
|
||||
dl_count: number,
|
||||
}
|
||||
|
||||
export type Package = Omit<AirtablePackage, 'airtable_record_id'> & {
|
||||
airtable_record_id?: string,
|
||||
installs: number,
|
||||
}
|
||||
|
||||
export type AlgoliaIndexedPackage = Omit<S3Package, 'slug | maintainer | last_modified'> & {
|
||||
objectID: string;
|
||||
desc: string;
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
{
|
||||
"extends": "./tsconfig.paths.json",
|
||||
"compilerOptions": {
|
||||
"lib": ["ESNext"],
|
||||
"moduleResolution": "node",
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"removeComments": true,
|
||||
"sourceMap": true,
|
||||
"target": "ES2020",
|
||||
"outDir": "lib",
|
||||
"allowSyntheticDefaultImports": true
|
||||
},
|
||||
"include": ["src/**/*.ts", "serverless.ts"],
|
||||
"exclude": [
|
||||
"node_modules/**/*",
|
||||
".serverless/**/*",
|
||||
".webpack/**/*",
|
||||
"_warmup/**/*",
|
||||
".vscode/**/*"
|
||||
],
|
||||
"ts-node": {
|
||||
"require": ["tsconfig-paths/register"]
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@functions/*": ["src/functions/*"],
|
||||
"@libs/*": ["src/libs/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
4617
lambdas/yarn.lock
4617
lambdas/yarn.lock
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue