mirror of
https://github.com/ivabus/pantry
synced 2024-09-20 00:30:48 +03:00
moved indexing to www
This commit is contained in:
Max Howell
parent
89d42f0cfc
commit
79006687f7
128
.github/scripts/utils/indexer.ts
vendored
128
.github/scripts/utils/indexer.ts
vendored
|
|
@ -1,128 +0,0 @@
|
|||
import * as fs from "node:fs"
|
||||
|
||||
interface Package {
|
||||
project: string,
|
||||
birthtime: Date,
|
||||
name?: string
|
||||
description?: string
|
||||
labels?: string[]
|
||||
}
|
||||
|
||||
export async function getKettleRemoteMetadata() {
|
||||
const headers = { Authorization: 'public' }
|
||||
const rsp = await fetch(`https://app.pkgx.dev/v1/packages/`, {headers})
|
||||
const foo = await rsp.json() as {project: string, short_description: string}[]
|
||||
return foo.reduce((acc, {project, short_description}) => {
|
||||
acc[project] = short_description
|
||||
return acc
|
||||
}, {} as Record<string, string>)
|
||||
}
|
||||
|
||||
const descriptions = await getKettleRemoteMetadata();
|
||||
|
||||
async function getPackageYmlCreationDates(): Promise<Package[]> {
|
||||
const cmdString = "git log --pretty=format:'%H %aI' --name-only --diff-filter=A -- '**/package.yml'";
|
||||
const process = Deno.run({
|
||||
cmd: ["bash", "-c", cmdString],
|
||||
stdout: "piped",
|
||||
});
|
||||
|
||||
const output = new TextDecoder().decode(await process.output());
|
||||
await process.status();
|
||||
process.close();
|
||||
|
||||
const lines = output.trim().split('\n');
|
||||
const rv: Package[] = []
|
||||
let currentCommitDate: string | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.includes(' ')) { // Detect lines with commit hash and date
|
||||
currentCommitDate = line.split(' ')[1];
|
||||
} else if (line.endsWith('package.yml')) {
|
||||
const project = line.slice(9, -12)
|
||||
|
||||
if (!fs.existsSync(line)) {
|
||||
// the file used to exist but has been deleted
|
||||
console.warn("skipping yanked: ", project)
|
||||
continue
|
||||
}
|
||||
|
||||
const birthtime = new Date(currentCommitDate!)
|
||||
const name = await get_name(line, project)
|
||||
|
||||
let description: string | undefined = descriptions[project]?.trim()
|
||||
if (!description) description = undefined
|
||||
|
||||
let labels: string[] | undefined = [...await get_labels(line)]
|
||||
if (labels.length == 0) labels = undefined
|
||||
|
||||
rv.push({ project, birthtime, name, description, labels })
|
||||
}
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
const pkgs = await getPackageYmlCreationDates();
|
||||
|
||||
// sort by birthtime
|
||||
pkgs.sort((a, b) => b.birthtime.getTime() - a.birthtime.getTime());
|
||||
|
||||
console.log(JSON.stringify(pkgs, null, 2));
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
import { parse } from "https://deno.land/std@0.204.0/yaml/mod.ts";
|
||||
import { isArray } from "https://deno.land/x/is_what@v4.1.15/src/index.ts";
|
||||
import get_pkg_name from "https://raw.githubusercontent.com/pkgxdev/www/main/src/utils/pkg-name.ts";
|
||||
|
||||
async function get_name(path: string, project: string): Promise<string | undefined> {
|
||||
const txt = await Deno.readTextFileSync(path)
|
||||
const yml = await parse(txt) as Record<string, any>
|
||||
if (yml['display-name']) {
|
||||
return yml['display-name']
|
||||
} else if (isArray(yml.provides) && yml.provides.length == 1) {
|
||||
return yml.provides[0].slice(4)
|
||||
} else {
|
||||
return get_pkg_name(project)
|
||||
}
|
||||
}
|
||||
|
||||
import { parse_pkgs_node } from "pkgx/hooks/usePantry.ts"
|
||||
|
||||
async function get_labels(path: string) {
|
||||
const txt = await Deno.readTextFileSync(path)
|
||||
const yml = await parse(txt) as Record<string, any>
|
||||
const deps = parse_pkgs_node(yml.dependencies) //NOTE will ignore other platforms than linux, but should be fine
|
||||
|
||||
const labels = new Set<string>(deps.compact(({ project }) => {
|
||||
switch (project) {
|
||||
case 'nodejs.org':
|
||||
case 'npmjs.com':
|
||||
return 'node'
|
||||
case 'python.org':
|
||||
case 'pip.pypa.io':
|
||||
return 'python'
|
||||
case 'ruby-lang.org':
|
||||
case 'rubygems.org':
|
||||
return 'ruby'
|
||||
case 'rust-lang.org':
|
||||
case 'rust-lang.org/cargo':
|
||||
return 'rust'
|
||||
}
|
||||
}))
|
||||
|
||||
console.error(path)
|
||||
|
||||
if (yml.build?.dependencies) for (const dep of parse_pkgs_node(yml.build.dependencies)) {
|
||||
switch (dep.project) {
|
||||
case 'rust-lang.org':
|
||||
case 'rust-lang.org/cargo':
|
||||
labels.add('rust')
|
||||
break
|
||||
case 'go.dev':
|
||||
labels.add('go')
|
||||
}
|
||||
}
|
||||
|
||||
return labels
|
||||
}
|
||||
39
.github/workflows/indexer.yml
vendored
39
.github/workflows/indexer.yml
vendored
|
|
@ -1,39 +0,0 @@
|
|||
name: indexer
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/scripts/utils/indexer.ts
|
||||
- .github/workflows/indexer.yml
|
||||
push:
|
||||
branches: main
|
||||
paths:
|
||||
- projects/**/package.yml
|
||||
- .github/scripts/utils/indexer.ts
|
||||
- .github/workflows/indexer.yml
|
||||
|
||||
concurrency:
|
||||
group: indexer/${{ github.ref || 'main' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
new-pkgs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: pkgxdev/setup@v2
|
||||
with:
|
||||
+: deno
|
||||
|
||||
- run: mkdir out
|
||||
- run: deno run -A .github/scripts/utils/indexer.ts > out/pkgs.json
|
||||
|
||||
- uses: JamesIves/github-pages-deploy-action@v4
|
||||
with:
|
||||
folder: out
|
||||
Loading…
Reference in a new issue