replace api loading (#327)

* #326 change packages loading strategy

* #326 refactor implementation of new sync strategy


---------

Co-authored-by: neil molina <neil@neils-MacBook-Pro.local>

modules/desktop/electron/libs/ipc.ts

@@ -2,7 +2,7 @@ import { ipcMain } from "electron";
 
 import { getInstalledPackages } from "./tea-dir";
 import { readSessionData, writeSessionData } from "./auth";
-import type { Session } from "../../src/libs/types";
+import type { Session, InstalledPackage } from "../../src/libs/types";
 import * as log from "electron-log";
 
 import { installPackage, openTerminal } from "./cli";

modules/desktop/electron/libs/tea-dir.ts

@@ -4,6 +4,8 @@ import path from "path";
 import { app } from "electron";
 import semver from "semver";
 import * as log from "electron-log";
+import type { InstalledPackage } from "../../src/libs/types";
+import semverCompare from "semver/functions/compare";
 
 type Dir = {
 	name: string;
@@ -21,7 +23,7 @@ export const getGuiPath = () => {
 	return path.join(getTeaPath(), "tea.xyz/gui");
 };
 
-export async function getInstalledPackages() {
+export async function getInstalledPackages(): Promise<InstalledPackage[]> {
 	const pkgsPath = getTeaPath();
 	log.info("recusively reading:", pkgsPath);
 	const folders = await deepReadDir({
@@ -30,19 +32,32 @@ export async function getInstalledPackages() {
 		filter: (name: string) => !!semver.valid(name)
 	});
 
-	const pkgs = folders
+	const bottles = folders
 		.map((p: string) => p.split(".tea/")[1])
 		.map((p: string) => {
 			const path = p.trim().split("/");
 			const version = path.pop();
 			return {
-				version: semver.clean(version || ""),
+				version: semver.clean(version || "") || "",
 				full_name: path.join("/")
 			};
-		});
+		})
+		.sort((a, b) => semverCompare(b.version, a.version));
 
-	log.info("found installed packages:", pkgs.length);
-	return pkgs;
+	log.info("installed bottles:", bottles.length);
+
+	return bottles.reduce<InstalledPackage[]>((pkgs, bottle) => {
+		const pkg = pkgs.find((v) => v.full_name === bottle.full_name);
+		if (pkg) {
+			pkg.installed_versions.push(bottle.version);
+		} else {
+			pkgs.push({
+				full_name: bottle.full_name,
+				installed_versions: [bottle.version]
+			});
+		}
+		return pkgs;
+	}, []);
 }
 
 const semverTest =

modules/desktop/src/libs/native-electron.ts

@@ -13,57 +13,63 @@
 
 import semverCompare from "semver/functions/compare";
 import type { Package, Review, AirtablePost, Bottle } from "@tea/ui/types";
-import {
-	type GUIPackage,
-	type Course,
-	type Category,
-	type DeviceAuth,
-	type Session,
-	AuthStatus
-} from "./types";
+import { type GUIPackage, type DeviceAuth, type Session, AuthStatus } from "./types";
 
 import * as mock from "./native-mock";
-import { PackageStates } from "./types";
+import { PackageStates, type InstalledPackage } from "./types";
 import { installPackageCommand } from "./native/cli";
 
 import { get as apiGet } from "$libs/v1-client";
+import axios from "axios";
+
 const log = window.require("electron-log");
 const { ipcRenderer, shell } = window.require("electron");
 
 let retryLimit = 0;
-async function getDistPackages(): Promise<Package[]> {
+export async function getDistPackages(): Promise<Package[]> {
 	let packages: Package[] = [];
 	try {
-		const resultingPackages = await apiGet<Package[]>("packages");
-		if (resultingPackages) packages = resultingPackages;
+		const req = await axios.get<Package[]>(
+			"https://s3.amazonaws.com/preview.gui.tea.xyz/packages.json"
+		);
+		log.info("packages received:", req.data.length);
+		packages = req.data;
 	} catch (error) {
 		retryLimit++;
 		log.error("getDistPackagesList:", error);
 		if (retryLimit < 3) packages = await getDistPackages();
 	}
+	retryLimit = 0;
 	return packages;
 }
 
+export async function getInstalledPackages(): Promise<InstalledPackage[]> {
+	let pkgs: InstalledPackage[] = [];
+	try {
+		log.info("getting installed packages");
+		pkgs = (await ipcRenderer.invoke("get-installed-packages")) as InstalledPackage[];
+		log.info("got installed packages:", pkgs.length);
+	} catch (error) {
+		log.error(error);
+	}
+	return pkgs;
+}
+
 export async function getPackages(): Promise<GUIPackage[]> {
 	const [packages, installedPackages] = await Promise.all([
 		getDistPackages(),
-		ipcRenderer.invoke("get-installed-packages") as { version: string; full_name: string }[]
+		ipcRenderer.invoke("get-installed-packages") as InstalledPackage[]
 	]);
 
-	// sorts all packages from highest -> lowest
-	installedPackages.sort((a, b) => semverCompare(b.version, a.version));
-
 	// NOTE: its not ideal to get bottles or set package states here maybe do it async in the package store init
 	// --- it has noticeable slowness
 	log.info(`native: installed ${installedPackages.length} out of ${(packages || []).length}`);
 	return (packages || []).map((pkg) => {
-		const installedVersions = installedPackages
-			.filter((p) => p.full_name === pkg.full_name)
-			.map((p) => p.version);
+		const installedPkg = installedPackages.find((p) => p.full_name === pkg.full_name);
 		return {
 			...pkg,
-			state: installedVersions.length ? PackageStates.INSTALLED : PackageStates.AVAILABLE,
-			installed_versions: installedVersions
+			state: installedPkg ? PackageStates.INSTALLED : PackageStates.AVAILABLE,
+			installed_versions: installedPkg?.installed_versions || []
 		};
 	});
 }

modules/desktop/src/libs/native-mock.ts

@@ -167,6 +167,11 @@ const packages: Package[] = [
 	}
 ];
 
+export const getInstalledPackages = () => [];
+export async function getDistPackages(): Promise<Package[]> {
+	return packages;
+}
+
 export async function getPackages(): Promise<GUIPackage[]> {
 	return packages.map((pkg) => {
 		return {

modules/desktop/src/libs/stores/pkgs.ts

@@ -1,58 +1,39 @@
 import { writable } from "svelte/store";
-import type { GUIPackage } from "../types";
+import type { GUIPackage, InstalledPackage } from "../types";
 import { PackageStates } from "../types";
-import { getPackages } from "@native";
 import Fuse from "fuse.js";
-import { getPackage, getPackageBottles } from "@native";
+import {
+	getPackage,
+	getDistPackages,
+	getPackageBottles,
+	openTerminal,
+	getInstalledPackages
+} from "@native";
 
 import { getReadme, getContributors, getRepoAsPackage } from "$libs/github";
 import semverCompare from "semver/functions/compare";
 import { notificationStore } from "../stores";
-import { openTerminal } from "@native";
 import { NotificationType } from "@tea/ui/types";
+import type { Package } from "@tea/ui/types";
 
 const log = window.require("electron-log");
 
 const installTea = async () => {
-	console.log("installing tea...");
+	log.info("installing tea...");
 	try {
 		openTerminal(`sh <(curl https://tea.xyz)`);
 	} catch (error) {
-		console.log("install failed");
+		log.error("install failed", error);
 	}
 };
 
 export default function initPackagesStore() {
 	let initialized = false;
+	const syncProgress = writable<number>(0); // TODO: maybe use this in the UI someday
 	const packages = writable<GUIPackage[]>([]);
-	// const packages: GUIPackage[] = [];
+
 	let packagesIndex: Fuse<GUIPackage>;
 
-	if (!initialized) {
-		initialized = true;
-		getPackages().then(async (pkgs) => {
-			packages.set(pkgs);
-			packagesIndex = new Fuse(pkgs, {
-				keys: ["name", "full_name", "desc", "categories"]
-			});
-
-			try {
-				const teaCliName = "tea.xyz";
-				for (const pkg of pkgs) {
-					log.info(`syncing ${pkg.full_name}`);
-					if (pkg.full_name === teaCliName) {
-						await syncTeaCliPackage(pkg);
-					} else if (pkg.state === PackageStates.INSTALLED) {
-						await syncPackageBottlesAndState(pkg.full_name);
-					}
-					log.info(`synced ${pkg.full_name}`);
-				}
-			} catch (error) {
-				log.error(error);
-			}
-		});
-	}
-
 	const updatePackage = (full_name: string, props: Partial<GUIPackage>) => {
 		packages.update((pkgs) => {
 			const i = pkgs.findIndex((pkg) => pkg.full_name === full_name);
@@ -103,6 +84,7 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 				const i = pkgs.findIndex((pkg) => pkg.full_name === pkgName);
 				if (i >= 0) {
 					const pkg = pkgs[i];
+					console.log(bottles, pkg);
 
 					const availableVersionsRaw = bottles
 						.map(({ version }) => version)
@@ -134,31 +116,75 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 		});
 	};
 
-	const syncTeaCliPackage = async (cliPkg: GUIPackage) => {
-		const updatedPkg = await syncPackageBottlesAndState(cliPkg.full_name);
-		if (updatedPkg && updatedPkg.state === PackageStates.INSTALLED) return;
-
-		if (!updatedPkg || updatedPkg.state === PackageStates.AVAILABLE) {
-			notificationStore.add({
-				message: "install cli",
-				i18n_key: "package.install-tea-cli",
-				type: NotificationType.ACTION_BANNER,
-				callback: installTea,
-				callback_label: "INSTALL"
-			});
-		} else if (updatedPkg.state === PackageStates.NEEDS_UPDATE) {
+	const checkTeaCLIPackage = async (teaPkg: Package, installedTeaCliPkg?: InstalledPackage) => {
+		const isLatest =
+			installedTeaCliPkg && installedTeaCliPkg.installed_versions[0] === teaPkg.version;
+		log.info("check Tea-CLI if latest:", !isLatest);
+		if (!isLatest) {
 			notificationStore.add({
 				message: "install cli",
 				i18n_key: "package.update-tea-cli",
 				type: NotificationType.ACTION_BANNER,
 				callback: installTea,
-				callback_label: "UPDATE"
+				callback_label: installedTeaCliPkg ? "INSTALL" : "UPDATE"
 			});
 		}
 	};
 
+	const init = async function () {
+		log.info("packages store: try initialize");
+		if (!initialized) {
+			log.info("packages store: initializing...");
+			initialized = true;
+			const pkgs = await getDistPackages();
+			const guiPkgs: GUIPackage[] = pkgs.map((p) => ({
+				...p,
+				state: PackageStates.AVAILABLE
+			}));
+
+			// set packages data so that i can render something in the UI already
+			packages.set(guiPkgs);
+			log.info("initialized packages store with ", guiPkgs.length);
+			packagesIndex = new Fuse(guiPkgs, {
+				keys: ["name", "full_name", "desc", "categories"]
+			});
+			log.info("initialized packages fuse index");
+
+			try {
+				const installedPkgs: InstalledPackage[] = await getInstalledPackages();
+
+				log.info("sync test for tea-cli");
+				const installedTea = installedPkgs.find((p) => p.full_name === "tea.xyz");
+				const distTea = pkgs.find((p) => p.full_name === "tea.xyz");
+				if (distTea) await checkTeaCLIPackage(distTea, installedTea);
+
+				log.info("set NEEDS_UPDATE state to pkgs");
+				let progressCount = 0;
+				for (const iPkg of installedPkgs) {
+					progressCount++;
+					const pkg = guiPkgs.find((p) => p.full_name === iPkg.full_name);
+					if (pkg) {
+						const isUpdated = pkg.version === iPkg.installed_versions[0];
+						updatePackage(pkg.full_name, {
+							installed_versions: iPkg.installed_versions,
+							state: isUpdated ? PackageStates.INSTALLED : PackageStates.NEEDS_UPDATE
+						});
+						log.info(`getting available bottles for ${pkg.full_name}`);
+						await syncPackageBottlesAndState(iPkg.full_name);
+					}
+					log.info(`synced ${iPkg.full_name} ${progressCount}/${installedPkgs.length}`);
+					syncProgress.set(+(progressCount / installedPkgs.length).toFixed(2));
+				}
+			} catch (error) {
+				log.error(error);
+			}
+		}
+		log.info("packages store: initialized!");
+	};
+
 	return {
 		packages,
+		syncProgress,
 		subscribe: packages.subscribe,
 		search: async (term: string, limit = 5): Promise<GUIPackage[]> => {
 			if (!term || !packagesIndex) return [];
@@ -176,6 +202,7 @@ To read more about this package go to [${guiPkg.homepage}](${guiPkg.homepage}).
 				}
 			});
 		},
-		updatePackage
+		updatePackage,
+		init
 	};
 }

modules/desktop/src/libs/types.ts

@@ -3,7 +3,7 @@
 // 		please use the package @tea/ui/src/types.ts
 //		things that go there are shared types/shapes like ie: Package
 
-import type { Package, Developer } from "@tea/ui/types";
+import type { Package, Developer, Bottle } from "@tea/ui/types";
 
 export enum PackageStates {
 	AVAILABLE = "AVAILABLE",
@@ -64,3 +64,5 @@ export enum SideMenuOptions {
 	star_struct = "star_struct",
 	made_by_tea = "made_by_tea"
 }
+
+export type InstalledPackage = Require<Pick<GUIPackage, "full_name" | "installed_versions">>;

modules/desktop/src/routes/+layout.svelte

@@ -5,7 +5,7 @@
 	import { afterNavigate } from '$app/navigation';
 	import TopBar from '$components/top-bar/top-bar.svelte';
 	import SideBar from '$components/side-bar/side-bar.svelte';
-	import { navStore } from '$libs/stores';
+	import { navStore, packagesStore } from '$libs/stores';
 	import { listenToChannel } from "@native";
 
 	import SearchPopupResults from '$components/search-popup-results/search-popup-results.svelte';
@@ -37,6 +37,7 @@
 		// used by the tea:// protocol to suggest a path to open
 		syncPath();
 		listenToChannel("sync-path", syncPath);
+		packagesStore.init();
 	});
 </script>