wip: add db classes; can't load SQLite driver.

.gitignore

@@ -23,4 +23,7 @@ dist-ssr
 *.sln
 *.sw?
 
+# Secrets
+*.secret*
 .turbo
+*.sqlite

README.md

@@ -48,18 +48,14 @@ This project uses Turborepo for task management/caching. Install Turborepo globa
 ### Frontend (Viossa.net)
 1. Ensure you're in the root directory of the project (`ViossaDotNet`)
 1. Move into the app's directory: `cd apps/vdn-static`
-1. Now, to run the site, use `turbo dev`. This will set up watchers to build all libraries used by the frontend, as well as hot-refreshing the site as changes are made to it.
+1. To run the site, use `turbo dev`. This will set up watchers to build all libraries used by the frontend, as well as hot-refreshing the site as changes are made to it.
 1. To view the website running locally, visit http://localhost:1224/ in your browser!
 
 ### Backend (Viossa DB)
 1. Ensure you're in the root directory of the project (`ViossaDotNet`)
-2. Move into the app's directory: `cd apps/vdb-backend`
+1. Move into the app's directory: `cd apps/vdb-backend`
-3. Now, to run the site, use `turbo start`. This will build all of the app's dependencies and then start the application.
+1. To run the site, use `turbo start`. This will build all of the app's dependencies and then start the application.
-   1. **NOTE:** Backend apps are not watched/hot-refreshed like frontend apps! If you make changes, you must kill the app by spamming Ctrl+C in the terminal it is running in, before rerunning it with the changes applied.
+   1. **NOTE:** Backend apps are not watched/hot-refreshed like frontend apps! If you make changes, you must kill the app and re-run it to apply changes.
-4. To view a sample response from the backend API, visit http://localhost:1225/sample in your browser!
+1. To view a sample response from the backend API, visit http://localhost:1225/sample in your browser!
-
-## The Content
-
-**What will be in the site?**
 
 [Visit the GitHub Issues page for this repository.](https://github.com/ViossaDiskordServer/ViossaDotNet/issues)

apps/vdb-backend/package.json

@@ -19,11 +19,19 @@
 	"license": "ISC",
 	"packageManager": "pnpm@10.11.0",
 	"dependencies": {
+		"@feathersjs/feathers": "^5.0.6",
+		"@google-cloud/local-auth": "^2.1.0",
 		"@repo/common": "workspace:*",
-		"express": "^5.1.0"
+		"express": "^5.1.0",
+		"google-auth-library": "^10.3.0",
+		"googleapis": "^159.0.0",
+		"reflect-metadata": "^0.2.2",
+		"sqlite3": "^5.1.6",
+		"typeorm": "0.3.26"
 	},
 	"devDependencies": {
 		"@types/express": "^5.0.3",
+		"@types/node": "^22.5.1",
 		"tsx": "^4.19.4"
 	}
 }

apps/vdb-backend/res/credentials.template.json

@@ -0,0 +1,11 @@
+{
+  "installed": {
+    "client_id": "",
+    "project_id": "",
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_secret": "",
+    "redirect_uris": ["http://localhost"]
+  }
+}

apps/vdb-backend/res/token.template.json

@@ -0,0 +1,6 @@
+{
+	"type": "authorized_user",
+	"client_id": "",
+	"client_secret": "",
+	"refresh_token": ""
+}

apps/vdb-backend/src/auth.ts

@@ -0,0 +1,61 @@
+
+import { promises as fs } from "fs";
+import * as path from "path";
+import { authenticate } from "@google-cloud/local-auth";
+import { google } from "googleapis";
+import { OAuth2Client } from 'google-auth-library';
+
+
+//setup google auth
+const SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly"];
+const TOKEN_PATH = path.join(process.cwd(), "res/token.secret.json");
+const CREDENTIALS_PATH = path.join(process.cwd(), "res/credentials.secret.json");
+
+
+/**
+ * Load or request or authorization to call APIs.
+*/
+
+export async function authorize() {
+  let client: any = await loadSavedCredentialsIfExist();
+  if (client) {
+    return client;
+  }
+  client = await authenticate({
+    scopes: SCOPES,
+    keyfilePath: CREDENTIALS_PATH,
+  });
+  if (client.credentials) {
+    await saveCredentials(client);
+  }
+  return client;
+}
+
+async function loadSavedCredentialsIfExist() {
+  try {
+    const content = await fs.readFile(TOKEN_PATH);
+    const credentials = JSON.parse(content.toString());
+    return google.auth.fromJSON(credentials);
+  } catch (err) {
+    return null;
+  }
+}
+
+/**
+ * Serializes credentials to a file compatible with GoogleAuth.fromJSON.
+ *
+ * @param {OAuth2Client} client
+ * @return {Promise<void>}
+ */
+async function saveCredentials(client: OAuth2Client) {
+  const content = await fs.readFile(CREDENTIALS_PATH);
+  const keys = JSON.parse(content.toString());
+  const key = keys.installed || keys.web;
+  const payload = JSON.stringify({
+    type: "authorized_user",
+    client_id: key.client_id,
+    client_secret: key.client_secret,
+    refresh_token: client.credentials.refresh_token,
+  });
+  await fs.writeFile(TOKEN_PATH, payload);
+}

apps/vdb-backend/src/config/dbconfig.ts

@@ -0,0 +1,15 @@
+import "reflect-metadata"
+import { DataSource } from "typeorm"
+import {Lemma, WordForm, Media, Example, Definition, Comment, PartOfSpeech, Lect} from "../db/dbmodel.js"
+
+const persistent_path = process.env.VI_DB_PERSISTENT_PATH || "./res";
+
+export const appDataSource = new DataSource({
+  type: "sqlite",
+  database:`${persistent_path}/dev.sqlite`,
+  synchronize: true,
+  logging: false,
+  entities: [Lemma, WordForm, Example, Media, Definition, Comment, PartOfSpeech, Lect],
+  migrations: [],
+  subscribers: [],
+})

apps/vdb-backend/src/db/dbmodel.ts

@@ -0,0 +1,124 @@
+import {
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  PrimaryColumn,
+  BaseEntity,
+  ManyToOne,
+  OneToMany,
+  OneToOne,
+  ManyToMany,
+  JoinTable,
+  ColumnType
+} from "typeorm";
+
+import "reflect-metadata"
+
+@Entity()
+export class Lemma extends BaseEntity {
+  @PrimaryColumn({type: 'string'})
+  lemma_name: string;
+
+  @OneToMany(() => WordForm, (word_form) => word_form.lemma)
+  word_forms: WordForm[];
+
+  @OneToMany(() => Example, (example) => example.lemma)
+  examples: Example[];
+
+  @OneToMany(() => Definition, (definition) => definition.lemma)
+  definitions: Definition[];
+
+  @OneToMany(() => Comment, (comment) => comment.lemma)
+  comments: Comment[];
+
+  @OneToMany(() => Media, (media) => media.lemma)
+  media: Media[];
+
+  @ManyToMany(() => PartOfSpeech)
+  @JoinTable()
+  parts_of_speech: PartOfSpeech[];
+
+}
+
+@Entity()
+export class Lect extends BaseEntity {
+  @PrimaryColumn({type: 'string'})
+  name: string;
+
+  @OneToMany(() => WordForm, (w) => w.lect)
+  word_forms: WordForm[];
+
+}
+
+@Entity()
+export class WordForm extends BaseEntity {
+  @PrimaryColumn({type: 'number'})
+  word_form_id: number
+  
+  @Column({type: 'string'})
+  word_form: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.word_forms)
+  lemma: Lemma;
+
+  @ManyToOne(() => Lect, (lect) => lect.word_forms)
+  lect: Lect;
+}
+
+@Entity()
+export class Example extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  example_id: number;
+
+  @Column({ nullable: false, type: 'string' })
+  example_text: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.examples)
+  lemma: Lemma; 
+}
+
+@Entity()
+export class Media extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  media_id: number;
+
+  @Column({ nullable: false, type: 'string'})
+  media_url: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.media)
+  lemma: Lemma;
+}
+
+@Entity()
+export class Definition extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  definition_id: number;
+
+  @Column({ nullable: false, type: 'string' })
+  definition_text: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.definitions)
+  lemma: Lemma;
+}
+
+@Entity()
+export class Comment extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  comment_id: number;
+
+  @Column({ nullable: false, type: 'string' })
+  comment_text: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.comments)
+  lemma: Lemma;
+}
+
+@Entity()
+export class PartOfSpeech extends BaseEntity {
+  @PrimaryColumn({type: 'string'})
+  long_form: string;
+
+  @Column({ nullable: false, unique: true, type: 'string' })
+  short_form: string;
+}
+

apps/vdb-backend/src/index.ts

@@ -1,15 +1,185 @@
 import { SAMPLE } from "@repo/common/sample";
 import express from "express";
+import { google, sheets_v4 } from "googleapis";
+import { OAuth2Client } from 'google-auth-library';
+import { appDataSource } from "./config/dbconfig.js";
+import { authorize } from "./auth.js"
+import "reflect-metadata";
+import {
+	Lemma,
+	WordForm,
+	Example,
+	Media,
+	Definition,
+	Comment,
+	PartOfSpeech,
+	Lect,
+} from "./db/dbmodel.js";
+import { error } from "console";
 
-const PORT = 1225;
+const RELOAD_SHEET_ON_START = false;
 
-const app = express();
 
-app.get("/sample", (_req, res) => {
+
+appDataSource
+	.initialize()
+	.then(async () => {
+		initExpress();
+
+		if (RELOAD_SHEET_ON_START) {
+			const lect_repository = appDataSource.getRepository(Lect);
+			const word_form_repository = appDataSource.getRepository(WordForm);
+			const lemma_repository = appDataSource.getRepository(Lemma);
+
+			await word_form_repository.clear();
+			await lemma_repository.clear();
+			await lect_repository.clear();
+
+			authorize().then(loadSheet).catch(console.error);
+		}
+	})
+	.catch((error) => console.log(error));
+
+
+function initExpress() {
+	const app = express();
+	const PORT = 1225;
+
+	const lect_repository = appDataSource.getRepository(Lect);
+	const word_form_repository = appDataSource.getRepository(WordForm);
+	const lemma_repository = appDataSource.getRepository(Lemma);
+
+	app.get("/", (req, res) => {
+		res.render("search", { title: "Suha", message: "Bratsa" });
+	});
+
+	app.get("/sample", (_req, res) => {
 		res.status(200).send(SAMPLE);
-});
+	});
 
-app.listen(PORT, () => {
+	app.get("/search", async (req, res) => {
+
+		const search_term = req.query.search_term?.toString();
+
+		if(!search_term) {
+			return;
+		}
+
+		const lemmas: Lemma[] = (
+			await Lemma.find({
+				relations: {
+					word_forms: { lect: true },
+				},
+			})
+		).filter((e) => {
+			for (const wf of e.word_forms) {
+				if (wf.word_form.includes(search_term)) {
+					return true;
+				}
+			}
+		});
+
+		const lects = await Lect.find();
+		return res.render("search_results", {
+			terms: lemmas.length,
+			results: lemmas,
+			lects: lects,
+		});
+	});
+
+	app.set("views", "./res/views");
+	app.set("view engine", "pug");
+
+	app.listen(PORT, () => {
 		console.log(`Backend started @ http://localhost:${PORT} !`);
 		console.log(SAMPLE);
-});
+	});
+}
+
+/**
+ * @param {google.auth.OAuth2Client} auth The authenticated Google OAuth client.
+ */
+async function loadSheet(auth: OAuth2Client) {
+  const options: any = { version: "v4", auth };
+  const sheets = google.sheets(options);
+  const res = await sheets.spreadsheets.values.get({
+    spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
+    range: "al_ko_mit_govor",
+  });
+
+  const rows = res.data.values;
+  
+  const keys_res = await sheets.spreadsheets.values.get({
+    spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
+    range: "klucz",
+  });
+
+  const keys = keys_res.data.values;
+  
+  if (!rows || rows.length === 0) {
+    console.error("No data found.");
+    return;
+  }
+
+  if (!keys || keys.length === 0) {
+    console.error("No lemma keys found.");
+    return;
+  }
+
+  const lects = rows.shift();
+
+  if (keys.length != rows.length) {
+    console.error("Lemma count doesn't match number of rows.");
+    return;
+  }
+
+  if (!lects){
+	console.error("No lects found");
+	return;
+  }
+
+  for (const lect of lects) {
+    let l = new Lect();
+    l.name = lect;
+    l.save();
+    console.log(l);
+  }
+
+  const nikolect = lects.indexOf("Nikomiko");
+
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i];
+
+    const lemma_key = keys[i]?.[0];
+    const lemma = new Lemma();
+
+    if (lemma_key == null || lemma_key.length == 0) {
+      //assign a random UUID to the lemma as punishment for our failures
+      lemma.lemma_name = crypto.randomUUID();
+      console.log(`womp womp, missing lemma name, calling it ${lemma.lemma_name}`)
+    } else {
+      lemma.lemma_name = lemma_key;
+    }
+
+    console.log(lemma);
+    await lemma.save();
+
+    for(let i=0; i<rows.length; i++){
+      const cell = row?.[i];
+      if (cell === null || cell === undefined || (typeof cell === "string" && cell.length === 0)) {
+        continue;
+      }
+
+      for (let word_form of cell.split(";")) {
+        const f = new WordForm();
+        f.word_form = word_form;
+        f.lemma = lemma;
+        f.lect = lects[i];
+        //console.log(f);
+        await f.save();
+      }
+
+    }
+
+  }
+}

apps/vdb-backend/tsconfig.json

@@ -2,14 +2,17 @@
 	"compilerOptions": {
 		"module": "nodenext",
 		"moduleResolution": "nodenext",
-        "target": "ES2022",
+		"lib": ["es5","es6"],
+		"target": "esnext",
 		"strict": true,
+		"esModuleInterop": true,
+		"strictPropertyInitialization": false,
 		"noUncheckedIndexedAccess": true,
 		"sourceMap": true,
 		"outDir": "dist",
 		"declaration": true,
+		"experimentalDecorators": true,
+		"emitDecoratorMetadata": true
 	},
-    "include": [
+	"include": ["src"]
-        "src"
-    ]
 }