Hydrate a local package cache (#422)

* cache packages to hydrate

* remove unused subscribe function

* packages -> packageMap

modules/desktop/electron/libs/ipc.ts

@@ -2,7 +2,7 @@ import { ipcMain, app } from "electron";
 import { createReadStream, statSync } from "fs";
 import { getInstalledPackages } from "./tea-dir";
 import { readSessionData, writeSessionData } from "./auth";
-import type { Session } from "../../src/libs/types";
+import type { Packages, Session } from "../../src/libs/types";
 import * as log from "electron-log";
 import { post } from "./v1-client";
 import { deepReadDir, deletePackageFolder } from "./tea-dir";
@@ -13,7 +13,7 @@ import { installPackage, openTerminal } from "./cli";
 import { getUpdater } from "./auto-updater";
 import fetch from "node-fetch";
 
-import { syncPackageTopicSubscriptions } from "./push-notification";
+import { loadPackageCache, writePackageCache } from "./package";
 let teaProtocolPath = ""; // this should be empty string
 
 export const setProtocolPath = (path: string) => {
@@ -156,4 +156,21 @@ export default function initializeHandlers() {
 			return error;
 		}
 	);
+
+	ipcMain.handle("write-package-cache", async (_, data) => {
+		try {
+			await writePackageCache(data as Packages);
+		} catch (error) {
+			log.error(error);
+		}
+	});
+
+	ipcMain.handle("load-package-cache", async () => {
+		try {
+			return await loadPackageCache();
+		} catch (error) {
+			log.error(error);
+			return { version: "1", packages: {} };
+		}
+	});
 }

modules/desktop/electron/libs/package.ts

@@ -1,3 +1,42 @@
+import path from "path";
+import { mkdirp } from "mkdirp";
+import fs from "fs";
+import * as log from "electron-log";
+import { getTeaPath } from "./tea-dir";
+import { Packages } from "../../src/libs/types";
+
+const pkgsFilePath = path.join(getTeaPath(), "tea.xyz/gui/pkgs.json");
+const pkgsFolder = path.join(getTeaPath(), "tea.xyz/gui");
+
+export async function writePackageCache(pkgs: Packages) {
+	try {
+		if (!pkgs || !Object.keys(pkgs.packages).length) {
+			return;
+		}
+
+		log.info(`writing data for ${Object.keys(pkgs.packages).length} packages to ${pkgsFilePath}`);
+		await mkdirp(pkgsFolder);
+		fs.writeFileSync(pkgsFilePath, JSON.stringify(pkgs), {
+			encoding: "utf-8"
+		});
+	} catch (error) {
+		log.error(error);
+	}
+}
+
+export async function loadPackageCache(): Promise<Packages> {
+	try {
+		log.info(`loading package cache from ${pkgsFilePath}`);
+		const pkgData = fs.readFileSync(pkgsFilePath);
+		return JSON.parse(pkgData.toString()) as Packages;
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			log.error(err);
+		}
+		return { version: "1", packages: {} };
+	}
+}
+
 export const nameToSlug = (name: string) => {
 	// github.com/Pypa/twine -> github_com_pypa_twine
 	const [nameOnly] = name.split("@");

modules/desktop/src/app.d.ts

@@ -12,7 +12,7 @@ declare namespace App {
 declare namespace svelte.JSX {
 	// eslint-disable-next-line @typescript-eslint/no-unused-vars
 	interface HTMLAttributes<T> {
-		onclick_outside: () => void;
-		leave_delay: () => void;
+		onclick_outside?: () => void;
+		leave_delay?: () => void;
 	}
 }

modules/desktop/src/components/packages/packages.svelte

@@ -10,7 +10,7 @@
 	import Package from "./package.svelte";
 	import { packagesStore } from "$libs/stores";
 
-	const { packages: allPackages } = packagesStore;
+	const { packageList: allPackages } = packagesStore;
 	export let packageFilter: SideMenuOptions = SideMenuOptions.all;
 
 	export let sortBy: "popularity" | "most recent" = "most recent";

modules/desktop/src/components/side-menu/side-menu.svelte

@@ -5,11 +5,11 @@
   import MenuButton from './menu-button.svelte';
 	import { t } from '$libs/translations'; 
   import { goto } from '$app/navigation';
-  const { packages } = packagesStore;
+  const { packageList } = packagesStore;
 
 	export let activeOption:SideMenuOptions;
 
-  $: needsUpdateCount = $packages.filter((p) => p.state === PackageStates.NEEDS_UPDATE).length;
+  $: needsUpdateCount = $packageList.filter((p) => p.state === PackageStates.NEEDS_UPDATE).length;
 </script>
 
 <aside class="border border-t-0 border-b-0 border-l-0 border-gray p-2">
@@ -79,4 +79,4 @@
 	hr {
 		border-top: 1px solid #272626;
 	}
-</style>
+</style>

modules/desktop/src/libs/native-electron.ts

@@ -12,7 +12,7 @@
  */
 
 import type { Package, Review, AirtablePost, Bottle } from "@tea/ui/types";
-import { type GUIPackage, type DeviceAuth, type Session, AuthStatus } from "./types";
+import { type GUIPackage, type DeviceAuth, type Session, AuthStatus, type Packages } from "./types";
 
 import * as mock from "./native-mock";
 import { PackageStates, type InstalledPackage } from "./types";
@@ -235,3 +235,19 @@ export const deletePackage = async (args: { fullName: string; version: string })
 		log.error(error);
 	}
 };
+
+export const loadPackageCache = async () => {
+	try {
+		return await ipcRenderer.invoke("load-package-cache");
+	} catch (error) {
+		log.error(error);
+	}
+};
+
+export const writePackageCache = async (pkgs: Packages) => {
+	try {
+		await ipcRenderer.invoke("write-package-cache", pkgs);
+	} catch (error) {
+		log.error(error);
+	}
+};

modules/desktop/src/libs/native-mock.ts

@@ -6,7 +6,7 @@
  *  * make cors work with api.tea.xyz/v1
  */
 import type { Package, Review, AirtablePost, Bottle } from "@tea/ui/types";
-import type { GUIPackage, Course, Category, Session } from "./types";
+import type { GUIPackage, Course, Category, Session, Packages } from "./types";
 import { PackageStates } from "./types";
 import { loremIpsum } from "lorem-ipsum";
 import _ from "lodash";
@@ -28,7 +28,8 @@ const packages: Package[] = [
 		desc: "Fast and user friendly build system",
 		thumb_image_url: "https://tea.xyz/Images/packages/mesonbuild_com.jpg",
 		installs: 0,
-		categories: ["foundation_essentials"]
+		categories: ["foundation_essentials"],
+		created: "2022-10-06T15:45:08.000Z"
 	},
 	{
 		slug: "pixman_org",
@@ -43,7 +44,8 @@ const packages: Package[] = [
 		desc: "Pixman is a library that provides low-level pixel manipulation features such as image compositing and trapezoid rasterization.",
 		thumb_image_url: "https://tea.xyz/Images/packages/pixman_org.jpg",
 		installs: 0,
-		categories: ["foundation_essentials"]
+		categories: ["foundation_essentials"],
+		created: "2022-09-26T19:37:47.000Z"
 	},
 	{
 		slug: "freedesktop_org_pkg_config",
@@ -58,7 +60,8 @@ const packages: Package[] = [
 		desc: "Manage compile and link flags for libraries",
 		thumb_image_url: "https://tea.xyz/Images/packages/freedesktop_org_pkg_config.jpg",
 		installs: 0,
-		categories: ["foundation_essentials"]
+		categories: ["foundation_essentials"],
+		created: "2022-10-20T01:32:15.000Z"
 	},
 	{
 		slug: "gnu_org_gettext",
@@ -73,7 +76,8 @@ const packages: Package[] = [
 		desc: "GNU internationalization (i18n) and localization (l10n) library",
 		thumb_image_url: "https://tea.xyz/Images/packages/gnu_org_gettext.jpg",
 		installs: 0,
-		categories: ["foundation_essentials"]
+		categories: ["foundation_essentials"],
+		created: "2022-10-20T01:23:46.000Z"
 	},
 	{
 		slug: "ipfs_tech",
@@ -88,7 +92,8 @@ const packages: Package[] = [
 		desc: "Peer-to-peer hypermedia protocol",
 		thumb_image_url: "https://tea.xyz/Images/packages/ipfs_tech.jpg",
 		installs: 0,
-		categories: ["foundation_essentials"]
+		categories: ["foundation_essentials"],
+		created: "2022-10-19T21:36:52.000Z"
 	},
 	{
 		slug: "nixos_org_patchelf",
@@ -103,7 +108,8 @@ const packages: Package[] = [
 		desc: "PatchELF is a simple utility for modifying existing ELF executables and libraries.",
 		thumb_image_url: "https://tea.xyz/Images/packages/nixos_org_patchelf.jpg",
 		installs: 0,
-		categories: ["top_packages", "foundation_essentials"]
+		categories: ["top_packages", "foundation_essentials"],
+		created: "2022-09-27T04:50:44.000Z"
 	},
 	{
 		slug: "tea_xyz",
@@ -118,7 +124,8 @@ const packages: Package[] = [
 		desc: "Website of tea.xyz",
 		thumb_image_url: "https://tea.xyz/Images/packages/tea_xyz.jpg",
 		installs: 0,
-		categories: ["top_packages", "foundation_essentials"]
+		categories: ["top_packages", "foundation_essentials"],
+		created: "2022-10-19T19:13:51.000Z"
 	},
 	{
 		slug: "charm_sh_gum",
@@ -133,7 +140,8 @@ const packages: Package[] = [
 		desc: "",
 		thumb_image_url: "https://tea.xyz/Images/packages/charm_sh_gum.jpg",
 		installs: 0,
-		categories: ["top_packages", "foundation_essentials"]
+		categories: ["top_packages", "foundation_essentials"],
+		created: "2022-10-21T02:15:16.000Z"
 	},
 	{
 		slug: "pyyaml_org",
@@ -148,7 +156,8 @@ const packages: Package[] = [
 		desc: "YAML framework for Python",
 		thumb_image_url: "https://tea.xyz/Images/packages/pyyaml_org.jpg",
 		installs: 0,
-		categories: ["top_packages", "foundation_essentials"]
+		categories: ["top_packages", "foundation_essentials"],
+		created: "2022-10-03T15:35:14.000Z"
 	},
 	{
 		slug: "tea_xyz_gx_cc",
@@ -163,7 +172,8 @@ const packages: Package[] = [
 		desc: "",
 		thumb_image_url: "https://tea.xyz/Images/packages/tea_xyz_gx_cc.jpg",
 		installs: 0,
-		categories: ["top_packages", "foundation_essentials"]
+		categories: ["top_packages", "foundation_essentials"],
+		created: "2022-10-19T16:47:44.000Z"
 	}
 ];
 
@@ -352,3 +362,11 @@ export const setBadgeCount = async (count: number) => {
 export const deletePackage = async (args: { fullName: string; version: string }) => {
 	console.log("delete package", args);
 };
+
+export const loadPackageCache = async () => {
+	return { version: "1", packages: {} };
+};
+
+export const writePackageCache = async (pkgs: Packages) => {
+	console.log("write package cache", pkgs);
+};

modules/desktop/src/libs/stores/pkgs.ts

@@ -1,5 +1,5 @@
-import { writable } from "svelte/store";
-import type { GUIPackage, InstalledPackage } from "../types";
+import { derived, writable } from "svelte/store";
+import type { GUIPackage, InstalledPackage, Packages } from "../types";
 import { PackageStates } from "../types";
 import Fuse from "fuse.js";
 import {
@@ -9,33 +9,46 @@ import {
 	installPackage,
 	deletePackage,
 	getPackageBottles,
-	setBadgeCount
+	setBadgeCount,
+	loadPackageCache,
+	writePackageCache
 } from "@native";
 
 import { getReadme, getContributors, getRepoAsPackage } from "$libs/github";
 import type { Package } from "@tea/ui/types";
 import { trackInstall, trackInstallFailed } from "$libs/analytics";
 import { addInstalledVersion } from "$libs/packages/pkg-utils";
+import withDebounce from "$libs/utils/debounce";
 
 const log = window.require("electron-log");
 
 export default function initPackagesStore() {
 	let initialized = false;
+
 	const syncProgress = writable<number>(0); // TODO: maybe use this in the UI someday
-	const packages = writable<GUIPackage[]>([]);
+
+	const packageMap = writable<Packages>({ version: "0", packages: {} });
+	const packageList = derived(packageMap, ($packages) => Object.values($packages.packages));
+
 	const requireTeaCli = writable<boolean>(false);
 
 	let packagesIndex: Fuse<GUIPackage>;
 
+	const updateAllPackages = (guiPkgs: GUIPackage[]) => {
+		packageMap.update((pkgs) => {
+			guiPkgs.forEach((pkg) => {
+				const oldPkg = pkgs.packages[pkg.full_name];
+				pkgs.packages[pkg.full_name] = { ...oldPkg, ...pkg };
+			});
+			setBadgeCountFromPkgs(pkgs);
+			return pkgs;
+		});
+	};
+
 	const updatePackage = (full_name: string, props: Partial<GUIPackage>) => {
-		packages.update((pkgs) => {
-			const i = pkgs.findIndex((pkg) => pkg.full_name === full_name);
-			if (i >= 0) {
-				pkgs[i] = {
-					...pkgs[i],
-					...props
-				};
-			}
+		packageMap.update((pkgs) => {
+			const pkg = pkgs.packages[full_name];
+			pkgs.packages[full_name] = { ...pkg, ...props };
 			setBadgeCountFromPkgs(pkgs);
 			return pkgs;
 		});
@@ -83,7 +96,12 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 
 	const init = async function () {
 		log.info("packages store: try initialize");
+
 		if (!initialized) {
+			const cachedPkgs: Packages = await loadPackageCache();
+			log.info(`Loaded ${Object.keys(cachedPkgs.packages).length} packages from cache`);
+			packageMap.set(cachedPkgs);
+
 			log.info("packages store: initializing...");
 			initialized = true;
 			const pkgs = await getDistPackages();
@@ -93,8 +111,9 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 			}));
 
 			// set packages data so that i can render something in the UI already
-			packages.set(guiPkgs);
+			updateAllPackages(guiPkgs);
 			log.info("initialized packages store with ", guiPkgs.length);
+
 			packagesIndex = new Fuse(guiPkgs, {
 				keys: ["name", "full_name", "desc", "categories"]
 			});
@@ -121,7 +140,8 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 					syncProgress.set(+((i + 1) / installedPkgs.length).toFixed(2));
 				}
 
-				setBadgeCountFromPkgs(guiPkgs);
+				// TODO: I think this can just get deleted, updatePackage should be enough
+				//setBadgeCountFromPkgs(guiPkgs);
 			} catch (error) {
 				log.error(error);
 			}
@@ -164,7 +184,9 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 	const fetchPackageBottles = async (pkgName: string) => {
 		// TODO: this api should take an architecture argument or else an architecture filter should be applied downstreawm
 		const bottles = await getPackageBottles(pkgName);
-		updatePackage(pkgName, { bottles });
+		if (bottles?.length) {
+			updatePackage(pkgName, { bottles });
+		}
 	};
 
 	const deletePkg = async (pkg: GUIPackage, version: string) => {
@@ -179,11 +201,15 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 		}
 	};
 
+	const writePackageCacheWithDebounce = withDebounce(writePackageCache);
+	packageMap.subscribe(async (pkgs) => {
+		writePackageCacheWithDebounce(pkgs);
+	});
+
 	return {
-		packages,
+		packageList,
 		syncProgress,
 		requireTeaCli,
-		subscribe: packages.subscribe,
 		search: async (term: string, limit = 5): Promise<GUIPackage[]> => {
 			if (!term || !packagesIndex) return [];
 			// TODO: if online, use algolia else use Fuse
@@ -191,15 +217,6 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 			const matchingPackages: GUIPackage[] = res.map((v) => v.item);
 			return matchingPackages;
 		},
-		subscribeToPackage: (slug: string, cb: (pkg: GUIPackage) => void) => {
-			packages.subscribe((pkgs) => {
-				const foundPackage = pkgs.find((p) => p.slug === slug) as GUIPackage;
-				if (foundPackage) {
-					cb(foundPackage);
-					syncPackageData(foundPackage);
-				}
-			});
-		},
 		fetchPackageBottles,
 		updatePackage,
 		init,
@@ -228,9 +245,11 @@ const withFakeLoader = (pkg: GUIPackage, callback: (progress: number) => void):
 	return fakeTimer;
 };
 
-const setBadgeCountFromPkgs = (pkgs: GUIPackage[]) => {
+const setBadgeCountFromPkgs = (pkgs: Packages) => {
 	try {
-		const needsUpdateCount = pkgs.filter((p) => p.state === PackageStates.NEEDS_UPDATE).length;
+		const needsUpdateCount = Object.values(pkgs.packages).filter(
+			(p) => p.state === PackageStates.NEEDS_UPDATE
+		).length;
 		setBadgeCount(needsUpdateCount);
 	} catch (error) {
 		log.error(error);

modules/desktop/src/libs/types.ts

@@ -14,6 +14,11 @@ export enum PackageStates {
 	UPDATING = "UPDATING"
 }
 
+export type Packages = {
+	version: string;
+	packages: { [full_name: string]: GUIPackage };
+};
+
 export type GUIPackage = Package & {
 	state: PackageStates;
 	installed_versions?: string[];

modules/desktop/src/libs/utils/debounce.ts

@@ -0,0 +1,29 @@
+const log = window.require("electron-log");
+
+type DebounceableFunc = (...args: any[]) => void;
+
+export type DebounceOptions = {
+	lingerMs?: number;
+};
+
+export default function withDebounce(
+	f: DebounceableFunc,
+	{ lingerMs = 1000 }: DebounceOptions = {}
+) {
+	let timeoutId: ReturnType<typeof setTimeout> | undefined = undefined;
+
+	return (...args: any[]) => {
+		if (timeoutId) {
+			clearTimeout(timeoutId);
+		}
+
+		timeoutId = setTimeout(() => {
+			try {
+				f(...args);
+			} catch (err) {
+				//swallow the error, there is no good way to signal failure to the caller
+				log.error(err);
+			}
+		}, lingerMs);
+	};
+}

modules/desktop/src/routes/+page.svelte

@@ -16,7 +16,7 @@
 
 	const log = window.require("electron-log");
 
-	const { packages, requireTeaCli } = packagesStore;
+	const { packageList, requireTeaCli } = packagesStore;
 
 	const url = $page.url;
 
@@ -28,10 +28,10 @@
 	let updating = false;
 
 	let packagesScrollY = 0;
-	$: currentUpdatingPkg = $packages.find((p) => p.state === PackageStates.UPDATING)
+	$: currentUpdatingPkg = $packageList.find((p) => p.state === PackageStates.UPDATING)
 	$: updatingMessage = `updating ${currentUpdatingPkg?.full_name} (${currentUpdatingPkg?.install_progress_percentage}%)`;
 
-	$: pkgsToUpdate = $packages.filter((p: GUIPackage) => p.state === PackageStates.NEEDS_UPDATE);
+	$: pkgsToUpdate = $packageList.filter((p: GUIPackage) => p.state === PackageStates.NEEDS_UPDATE);
 	async function updateAll() {
 		updating = true;
 		log.info(`updating: ${pkgsToUpdate.length} packages`);
@@ -46,7 +46,7 @@
 		sideMenuOption = SideMenuOptions.all;
 	}
 
-	$: teaPkg = $packages.find((p) => p.full_name === "tea.xyz");
+	$: teaPkg = $packageList.find((p) => p.full_name === "tea.xyz");
 	$: needsUpdateCount = pkgsToUpdate.length;
 
 	

modules/desktop/src/routes/packages/[slug]/+page.svelte

@@ -20,9 +20,9 @@
 	import { packagesStore } from '$libs/stores';
   import { onMount } from 'svelte';
 
-	const { packages } = packagesStore;
+	const { packageList } = packagesStore;
 
-	$: pkg = $packages.find((p) => p.slug === data?.slug);
+	$: pkg = $packageList.find((p) => p.slug === data?.slug);
 
 	// let reviews: Review[];
 	$: bottles = pkg?.bottles || [];
@@ -95,4 +95,4 @@
 	</div>
 {:else}
 	<Preloader/>
-{/if}
+{/if}