Merge pull request #35 from ViossaDiskordServer/monorepo

feat: monorepo and api

.gitignore

@@ -23,4 +23,7 @@ dist-ssr
 *.sln
 *.sw?
 
+# Secrets
+*.secret*
 .turbo
+*.sqlite

README.md

@@ -48,18 +48,14 @@ This project uses Turborepo for task management/caching. Install Turborepo globa
 ### Frontend (Viossa.net)
 1. Ensure you're in the root directory of the project (`ViossaDotNet`)
 1. Move into the app's directory: `cd apps/vdn-static`
-1. Now, to run the site, use `turbo dev`. This will set up watchers to build all libraries used by the frontend, as well as hot-refreshing the site as changes are made to it.
+1. To run the site, use `turbo dev`. This will set up watchers to build all libraries used by the frontend, as well as hot-refreshing the site as changes are made to it.
 1. To view the website running locally, visit http://localhost:1224/ in your browser!
 
 ### Backend (Viossa DB)
 1. Ensure you're in the root directory of the project (`ViossaDotNet`)
-2. Move into the app's directory: `cd apps/vdb-backend`
-3. Now, to run the site, use `turbo start`. This will build all of the app's dependencies and then start the application.
-   1. **NOTE:** Backend apps are not watched/hot-refreshed like frontend apps! If you make changes, you must kill the app by spamming Ctrl+C in the terminal it is running in, before rerunning it with the changes applied.
-4. To view a sample response from the backend API, visit http://localhost:1225/sample in your browser!
-
-## The Content
-
-**What will be in the site?**
+1. Move into the app's directory: `cd apps/vdb-backend`
+1. To run the site, use `turbo start`. This will build all of the app's dependencies and then start the application.
+   1. **NOTE:** Backend apps are not watched/hot-refreshed like frontend apps! If you make changes, you must kill the app and re-run it to apply changes.
+1. To view a sample response from the backend API, visit http://localhost:1225/sample in your browser!
 
 [Visit the GitHub Issues page for this repository.](https://github.com/ViossaDiskordServer/ViossaDotNet/issues)

apps/vdb-backend/package.json

@@ -19,11 +19,22 @@
 	"license": "ISC",
 	"packageManager": "pnpm@10.11.0",
 	"dependencies": {
+		"@feathersjs/feathers": "^5.0.6",
+		"@google-cloud/local-auth": "^3.0.1",
 		"@repo/common": "workspace:*",
-		"express": "^5.1.0"
+		"express": "^5.1.0",
+		"google-auth-library": "^9.15.1",
+		"googleapis": "^149.0.0",
+		"node-fetch": "^3.3.2",
+		"reflect-metadata": "^0.2.2",
+		"sqlite3": "^5.1.7",
+		"typeorm": "0.3.26",
+		"zod": "^4.1.8"
 	},
 	"devDependencies": {
+		"@total-typescript/ts-reset": "^0.6.1",
 		"@types/express": "^5.0.3",
+		"@types/node": "^22.5.1",
 		"tsx": "^4.19.4"
 	}
 }

apps/vdb-backend/res/credentials.template.json

@@ -0,0 +1,11 @@
+{
+  "installed": {
+    "client_id": "",
+    "project_id": "",
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_secret": "",
+    "redirect_uris": ["http://localhost"]
+  }
+}

apps/vdb-backend/res/token.template.json

@@ -0,0 +1,6 @@
+{
+	"type": "authorized_user",
+	"client_id": "",
+	"client_secret": "",
+	"refresh_token": ""
+}

apps/vdb-backend/src/auth.ts

@@ -0,0 +1,114 @@
+import { promises as fs } from "fs";
+import * as path from "path";
+import { authenticate } from "@google-cloud/local-auth";
+import { google } from "googleapis";
+import { JWTInput, OAuth2Client } from "google-auth-library";
+import {
+	GoogleAuth,
+	JSONClient,
+} from "google-auth-library/build/src/auth/googleauth.js";
+import z from "zod";
+
+//setup google auth
+const SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly"];
+const TOKEN_PATH = path.join(process.cwd(), "res/token.secret.json");
+const CREDENTIALS_PATH = path.join(
+	process.cwd(),
+	"res/credentials.secret.json",
+);
+
+/**
+ * Load or request or authorization to call APIs.
+ */
+export async function authorize(): Promise<
+	GoogleAuth<JSONClient> | OAuth2Client
+> {
+	const savedClient = await loadSavedCredentialsIfExist();
+	if (savedClient) {
+		return savedClient;
+	}
+
+	const oauthClient = await authenticate({
+		scopes: SCOPES,
+		keyfilePath: CREDENTIALS_PATH,
+	});
+
+	if (oauthClient.credentials) {
+		await saveCredentials(oauthClient);
+	}
+
+	return oauthClient;
+}
+
+const jwtInputZod = z.object({
+	type: z.string(),
+	client_id: z.string(),
+	client_secret: z.string(),
+	refresh_token: z.string(),
+}) satisfies z.ZodType<JWTInput>;
+
+async function loadSavedCredentialsIfExist(): Promise<GoogleAuth<JSONClient> | null> {
+	try {
+		const content = await fs.readFile(TOKEN_PATH);
+		const jwtInputRaw = JSON.parse(content.toString());
+
+		const jwtInputRes = jwtInputZod.safeParse(jwtInputRaw);
+		if (!jwtInputRes.success) {
+			throw new Error("Malformed JWT/token credentials");
+		}
+
+		const jwtInput = jwtInputRes.data;
+		const jsonClient = google.auth.fromJSON(jwtInput);
+		const auth = new GoogleAuth({ authClient: jsonClient });
+		return auth;
+	} catch (err) {
+		return null;
+	}
+}
+
+const credentialsZod = z.object({
+	installed: z.object({
+		client_id: z.string(),
+		project_id: z.string(),
+		auth_uri: z.string(),
+		token_uri: z.string(),
+		auth_provider_x509_cert_url: z.string(),
+		client_secret: z.string(),
+		redirect_uris: z.array(z.string()),
+	}),
+	web: z.optional(
+		z.object({
+			client_id: z.string(),
+			project_id: z.string(),
+			auth_uri: z.string(),
+			token_uri: z.string(),
+			auth_provider_x509_cert_url: z.string(),
+			client_secret: z.string(),
+			redirect_uris: z.array(z.string()),
+		}),
+	),
+});
+
+/**
+ * Serializes credentials to a file compatible with GoogleAuth.fromJSON.
+ */
+async function saveCredentials(client: OAuth2Client): Promise<void> {
+	const content = await fs.readFile(CREDENTIALS_PATH);
+	const keysRaw = JSON.parse(content.toString());
+
+	const keysRes = credentialsZod.safeParse(keysRaw);
+	if (!keysRes.success) {
+		throw new Error("Malformed credentials/keys");
+	}
+
+	const keys = keysRes.data;
+	const key = keys.installed ?? keys.web;
+	const payload = JSON.stringify({
+		type: "authorized_user",
+		client_id: key.client_id,
+		client_secret: key.client_secret,
+		refresh_token: client.credentials.refresh_token,
+	});
+
+	await fs.writeFile(TOKEN_PATH, payload);
+}

apps/vdb-backend/src/config/dbconfig.ts

@@ -0,0 +1,15 @@
+import "reflect-metadata"
+import { DataSource } from "typeorm"
+import {Lemma, WordForm, Media, Example, Definition, Comment, PartOfSpeech, Lect} from "../db/dbmodel.js"
+
+const persistent_path = process.env.VI_DB_PERSISTENT_PATH || "./res";
+
+export const appDataSource = new DataSource({
+  type: "sqlite",
+  database:`${persistent_path}/dev.sqlite`,
+  synchronize: true,
+  logging: false,
+  entities: [Lemma, WordForm, Example, Media, Definition, Comment, PartOfSpeech, Lect],
+  migrations: [],
+  subscribers: [],
+})

apps/vdb-backend/src/db/dbmodel.ts

@@ -0,0 +1,120 @@
+import "reflect-metadata";
+import {
+	Entity,
+	PrimaryGeneratedColumn,
+	Column,
+	PrimaryColumn,
+	BaseEntity,
+	ManyToOne,
+	OneToMany,
+	OneToOne,
+	ManyToMany,
+	JoinTable,
+	ColumnType,
+} from "typeorm";
+
+@Entity()
+export class Lemma extends BaseEntity {
+	@PrimaryColumn({ type: "text" })
+	lemma_name: string;
+
+	@OneToMany(() => WordForm, (word_form) => word_form.lemma, { cascade: true})
+	word_forms: WordForm[];
+
+	@OneToMany(() => Example, (example) => example.lemma)
+	examples: Example[];
+
+	@OneToMany(() => Definition, (definition) => definition.lemma)
+	definitions: Definition[];
+
+	@OneToMany(() => Comment, (comment) => comment.lemma)
+	comments: Comment[];
+
+	@OneToMany(() => Media, (media) => media.lemma)
+	media: Media[];
+
+	@ManyToMany(() => PartOfSpeech)
+	@JoinTable()
+	parts_of_speech: PartOfSpeech[];
+}
+
+@Entity()
+export class Lect extends BaseEntity {
+	@PrimaryColumn({ type: "text" })
+	name: string;
+
+	@OneToMany(() => WordForm, (w) => w.lect)
+	word_forms: WordForm[];
+}
+
+@Entity()
+export class WordForm extends BaseEntity {
+	@PrimaryColumn({ type: "integer" })
+	word_form_id: number;
+
+	@Column({ type: "text" })
+	word_form: string;
+
+	@ManyToOne(() => Lemma, (lemma) => lemma.word_forms)
+	lemma: Lemma;
+
+	@ManyToOne(() => Lect, (lect) => lect.word_forms)
+	lect: Lect;
+}
+
+@Entity()
+export class Example extends BaseEntity {
+	@PrimaryGeneratedColumn()
+	example_id: number;
+
+	@Column({ nullable: false, type: "text" })
+	example_text: string;
+
+	@ManyToOne(() => Lemma, (lemma) => lemma.examples)
+	lemma: Lemma;
+}
+
+@Entity()
+export class Media extends BaseEntity {
+	@PrimaryGeneratedColumn()
+	media_id: number;
+
+	@Column({ nullable: false, type: "text" })
+	media_url: string;
+
+	@ManyToOne(() => Lemma, (lemma) => lemma.media)
+	lemma: Lemma;
+}
+
+@Entity()
+export class Definition extends BaseEntity {
+	@PrimaryGeneratedColumn()
+	definition_id: number;
+
+	@Column({ nullable: false, type: "text" })
+	definition_text: string;
+
+	@ManyToOne(() => Lemma, (lemma) => lemma.definitions)
+	lemma: Lemma;
+}
+
+@Entity()
+export class Comment extends BaseEntity {
+	@PrimaryGeneratedColumn()
+	comment_id: number;
+
+	@Column({ nullable: false, type: "text" })
+	comment_text: string;
+
+	@ManyToOne(() => Lemma, (lemma) => lemma.comments)
+	lemma: Lemma;
+}
+
+@Entity()
+export class PartOfSpeech extends BaseEntity {
+	@PrimaryColumn({ type: "text" })
+	long_form: string;
+
+	@Column({ nullable: false, unique: true, type: "text" })
+short_form: string;
+}

apps/vdb-backend/src/index.ts

@@ -1,15 +1,207 @@
+import "reflect-metadata";
+import fetch from "node-fetch";
 import { SAMPLE } from "@repo/common/sample";
 import express from "express";
+import { google, sheets_v4 } from "googleapis";
+import { OAuth2Client } from "google-auth-library";
+import { appDataSource } from "./config/dbconfig.js";
+import { authorize } from "./auth.js";
+import { Lemma, WordForm, Lect } from "./db/dbmodel.js";
+import {
+	GoogleAuth,
+	JSONClient,
+} from "google-auth-library/build/src/auth/googleauth.js";
+import "@total-typescript/ts-reset";
 
+const RELOAD_SHEET_ON_START = false;
+
+global.fetch = fetch as any;
+
+appDataSource
+	.initialize()
+	.then(async () => {
+		initExpress();
+
+		if (RELOAD_SHEET_ON_START) {
+			const lect_repository = appDataSource.getRepository(Lect);
+			const word_form_repository = appDataSource.getRepository(WordForm);
+			const lemma_repository = appDataSource.getRepository(Lemma);
+
+			await word_form_repository.clear();
+			await lemma_repository.clear();
+			await lect_repository.clear();
+
+			authorize().then(loadSheet).catch(console.error);
+		}
+	})
+	.catch((error) => console.log(error));
+
+function initExpress() {
+	const app = express();
 	const PORT = 1225;
 
-const app = express();
+	const lect_repository = appDataSource.getRepository(Lect);
+	const word_form_repository = appDataSource.getRepository(WordForm);
+	const lemma_repository = appDataSource.getRepository(Lemma);
 
 	app.get("/sample", (_req, res) => {
 		res.status(200).send(SAMPLE);
 	});
 
+	app.get("/search", async (req, res) => {
+		const search_term = req.query.search_term?.toString();
+
+		if (!search_term) {
+			return void res.sendStatus(400);
+		}
+
+		const lemmas: Lemma[] = (
+			await Lemma.find({ relations: { word_forms: { lect: true } } })
+		).filter((e) => {
+			for (const wf of e.word_forms) {
+				if (wf.word_form.includes(search_term)) {
+					return true;
+				}
+			}
+		});
+
+		const lects = await Lect.find();
+		res.status(200).send({
+			terms: lemmas.length,
+			results: lemmas,
+			lects: lects,
+		});
+	});
+
+	app.get("/lect", async (req, res) => {
+		const name = req.query.name?.toString();
+
+		if (!name) {
+			return void res.sendStatus(400);
+		}
+
+		const lect = await Lect.findOne({
+			where: { name: name },
+			relations: { word_forms: { lemma: true } },
+		});
+
+		res.status(200).send({ lect });
+	});
+
+	app.get("/lects", async (_req, res) => {
+		const lects = await Lect.find();
+		res.status(200).send({
+			lects,
+		});
+	});
+
 	app.listen(PORT, () => {
 		console.log(`Backend started @ http://localhost:${PORT} !`);
 		console.log(SAMPLE);
 	});
+}
+
+/**
+ * @param auth The authenticated Google OAuth client.
+ */
+async function loadSheet(auth: OAuth2Client | GoogleAuth<JSONClient>) {
+	const lect_repository = appDataSource.getRepository(Lect);
+	const word_form_repository = appDataSource.getRepository(WordForm);
+	const lemma_repository = appDataSource.getRepository(Lemma);
+	const options: sheets_v4.Options = { version: "v4", auth };
+	const sheets = google.sheets(options);
+	const res = await sheets.spreadsheets.values.get({
+		spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
+		range: "al_ko_mit_govor",
+	});
+
+	const rows = res.data.values;
+
+	const keys_res = await sheets.spreadsheets.values.get({
+		spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
+		range: "klucz",
+	});
+
+	const keys = keys_res.data.values;
+
+	if (!rows || rows.length === 0) {
+		console.error("No data found.");
+		return;
+	}
+
+	if (!keys || keys.length === 0) {
+		console.error("No lemma keys found.");
+		return;
+	}
+
+	const lects = rows.shift();
+
+	if (keys.length != rows.length) {
+		console.error("Lemma count doesn't match number of rows.");
+		return;
+	}
+
+	if (!lects) {
+		console.error("No lects found");
+		return;
+	}
+
+	for (const lect of lects) {
+		let l = new Lect();
+		l.name = lect;
+		l.save();
+		console.log(l);
+	}
+
+	const nikolect = lects.indexOf("Nikomiko");
+	const lemmas = Array<Lemma>();
+	for (let i = 0; i < rows.length; i++) {
+		const row = rows[i];
+
+		const lemma_key = keys[i]?.[0];
+		const lemma = new Lemma();
+
+		if (lemma_key == null || lemma_key.length == 0) {
+			//assign a random UUID to the lemma as punishment for our failures
+			lemma.lemma_name = crypto.randomUUID();
+			console.log(
+				`womp womp, missing lemma name, calling it ${lemma.lemma_name}`,
+			);
+		} else {
+			lemma.lemma_name = lemma_key;
+		}
+
+		lemmas.push(lemma);
+		lemma.word_forms = Array<WordForm>();
+
+		for (let i = 0; i < rows.length; i++) {
+			const cell = row?.[i];
+			if (
+				cell === null ||
+				cell === undefined ||
+				(typeof cell === "string" && cell.length === 0)
+			) {
+				continue;
+			}
+
+			for (let word_form of cell.split(";")) {
+				const f = new WordForm();
+				f.word_form = word_form;
+				f.lect = lects[i];
+				//console.log(f);
+				lemma.word_forms.push(f);
+			}
+		}
+	}
+
+	for (let i = 0; i < lemmas.length; i += 100) {
+		await lemma_repository.save(lemmas.slice(i, i + 100));
+		console.log(
+			`Saved ${Math.min(i + 100, lemmas.length)} / ${
+				lemmas.length
+			} lemmas...`,
+		);
+	}
+
+	console.log(`Loaded ${lemmas.length} lemmas!`);
+}

apps/vdb-backend/tsconfig.json

@@ -2,14 +2,17 @@
 	"compilerOptions": {
 		"module": "nodenext",
 		"moduleResolution": "nodenext",
-        "target": "ES2022",
+		"lib": ["es5","es6"],
+		"target": "esnext",
 		"strict": true,
+		"esModuleInterop": true,
+		"strictPropertyInitialization": false,
 		"noUncheckedIndexedAccess": true,
 		"sourceMap": true,
 		"outDir": "dist",
 		"declaration": true,
+		"experimentalDecorators": true,
+		"emitDecoratorMetadata": true
 	},
-    "include": [
-        "src"
-    ]
+	"include": ["src"]
 }

apps/vdn-static/package.json

@@ -11,6 +11,7 @@
 	"dependencies": {
 		"@tailwindcss/vite": "^4.1.6",
 		"@types/node": "^22.15.17",
+		"axios": "^1.11.0",
 		"@vueuse/components": "^13.3.0",
 		"@vueuse/core": "^13.3.0",
 		"bulma": "^1.0.4",

apps/vdn-static/src/App.vue

@@ -53,6 +53,9 @@ const closeBurger = (): void => {
 						@click="closeBurger()"
 						>Resources</RouterLink
 					>
+					<RouterLink class="navbar-item" to="/kotoba">
+						Kotoba
+					</RouterLink>
 					<LocalePicker />
 				</div>
 			</div>

apps/vdn-static/src/components/pages/KotobaPage.vue

@@ -0,0 +1,17 @@
+<template>
+	<div>
+		<section class="section">
+			<h1 class="title">Tropos-agnostic search</h1>
+		</section>
+
+		<section class="section container">
+			<input type="text" name="text" id="aaa">
+		</section>
+	</div>
+</template>
+
+<!--  -->
+
+<script setup lang="ts">
+
+</script>

apps/vdn-static/src/routes/index.ts

@@ -3,10 +3,12 @@ import type { RouteRecordRaw } from "vue-router";
 
 import HomePage from "@/components/pages/HomePage.vue";
 import ResourcesPage from "@/components/pages/ResourcesPage.vue";
+import KotobaPage from "@/components/pages/KotobaPage.vue";
 
 const routes: RouteRecordRaw[] = [
 	{ path: "/", name: "Home", component: HomePage },
 	{ path: "/resources", name: "Resources", component: ResourcesPage },
+	{ path: "/kotoba", name: "Kotoba", component: KotobaPage },
 	// {
 	//   path: '/:pathMatch(.*)*',  // Vue Router 4 catch-all for 404s
 	//   name: 'NotFound',

libs/common/package.json

@@ -21,5 +21,8 @@
 	"devDependencies": {
 		"rimraf": "^6.0.1",
 		"typescript": "~5.8.3"
+	},
+	"dependencies": {
+		"class-transformer": "^0.5.1"
 	}
 }

libs/common/src/dto.ts

@@ -0,0 +1,51 @@
+export interface LemmaDto {
+	lemma_name: string;	
+	word_forms: WordFormDto[];	
+	examples: ExampleDto[];	
+	definitions: DefinitionDto[];	
+	comments: CommentDto[];	
+	media: MediaDto[];	
+	
+	parts_of_speech: PartOfSpeechDto[];
+}
+
+export interface LectDto {
+	name: string;	
+	word_forms: WordFormDto[];
+}
+
+export interface WordFormDto {
+	word_form_id: number;	
+	word_form: string;	
+	lemma: LemmaDto;	
+	lect: LectDto;
+}
+
+export interface ExampleDto {
+	example_id: number;	
+	example_text: string;	
+	lemma: LemmaDto;
+}
+
+export interface MediaDto {
+	media_id: number;	
+	media_url: string;	
+	lemma: LemmaDto;
+}
+
+export interface DefinitionDto {
+	definition_id: number;	
+	definition_text: string;	
+	lemma: LemmaDto;
+}
+
+export interface CommentDto {
+	comment_id: number;	
+	comment_text: string;	
+	lemma: LemmaDto;
+}
+
+export interface PartOfSpeechDto {
+	long_form: string;	
+	short_form: string;
+}

package.json

@@ -2,5 +2,16 @@
 	"devDependencies": {
 		"turbo": "^2.5.4"
 	},
-	"packageManager": "pnpm@10.11.0"
+	"packageManager": "pnpm@10.11.0",
+	"scripts": {
+		"build:vdb-backend": "npm run build -w apps/vdb-backend",
+		"build:vdn-static": "npm run build -w apps/vdn-static",
+		"run:dev:vdb-backend": "npm run dev -w apps/vdb-backend",
+		"run:dev:vdn-static": "npm run dev -w apps/vdn-static",
+		"run:dev:all": "run-p run:dev:vdb-backend run:dev:vdn-static"
+	},
+	"workspaces": [
+		"apps/vdb-backend",
+		"apps/vdn-static"
+	]
 }

pnpm-workspace.yaml

@@ -4,4 +4,6 @@ packages:
 onlyBuiltDependencies:
   - '@parcel/watcher'
   - '@tailwindcss/oxide'
+  - better-sqlite3
   - esbuild
+  - sqlite3