wip: add db classes; can't load SQLite driver.

apps/vdb-backend/src/auth.ts

@@ -0,0 +1,61 @@
+
+import { promises as fs } from "fs";
+import * as path from "path";
+import { authenticate } from "@google-cloud/local-auth";
+import { google } from "googleapis";
+import { OAuth2Client } from 'google-auth-library';
+
+
+//setup google auth
+const SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly"];
+const TOKEN_PATH = path.join(process.cwd(), "res/token.secret.json");
+const CREDENTIALS_PATH = path.join(process.cwd(), "res/credentials.secret.json");
+
+
+/**
+ * Load or request or authorization to call APIs.
+*/
+
+export async function authorize() {
+  let client: any = await loadSavedCredentialsIfExist();
+  if (client) {
+    return client;
+  }
+  client = await authenticate({
+    scopes: SCOPES,
+    keyfilePath: CREDENTIALS_PATH,
+  });
+  if (client.credentials) {
+    await saveCredentials(client);
+  }
+  return client;
+}
+
+async function loadSavedCredentialsIfExist() {
+  try {
+    const content = await fs.readFile(TOKEN_PATH);
+    const credentials = JSON.parse(content.toString());
+    return google.auth.fromJSON(credentials);
+  } catch (err) {
+    return null;
+  }
+}
+
+/**
+ * Serializes credentials to a file compatible with GoogleAuth.fromJSON.
+ *
+ * @param {OAuth2Client} client
+ * @return {Promise<void>}
+ */
+async function saveCredentials(client: OAuth2Client) {
+  const content = await fs.readFile(CREDENTIALS_PATH);
+  const keys = JSON.parse(content.toString());
+  const key = keys.installed || keys.web;
+  const payload = JSON.stringify({
+    type: "authorized_user",
+    client_id: key.client_id,
+    client_secret: key.client_secret,
+    refresh_token: client.credentials.refresh_token,
+  });
+  await fs.writeFile(TOKEN_PATH, payload);
+}

apps/vdb-backend/src/config/dbconfig.ts

@@ -0,0 +1,15 @@
+import "reflect-metadata"
+import { DataSource } from "typeorm"
+import {Lemma, WordForm, Media, Example, Definition, Comment, PartOfSpeech, Lect} from "../db/dbmodel.js"
+
+const persistent_path = process.env.VI_DB_PERSISTENT_PATH || "./res";
+
+export const appDataSource = new DataSource({
+  type: "sqlite",
+  database:`${persistent_path}/dev.sqlite`,
+  synchronize: true,
+  logging: false,
+  entities: [Lemma, WordForm, Example, Media, Definition, Comment, PartOfSpeech, Lect],
+  migrations: [],
+  subscribers: [],
+})

apps/vdb-backend/src/db/dbmodel.ts

@@ -0,0 +1,124 @@
+import "reflect-metadata"
+import {
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  PrimaryColumn,
+  BaseEntity,
+  ManyToOne,
+  OneToMany,
+  OneToOne,
+  ManyToMany,
+  JoinTable,
+  ColumnType
+} from "typeorm";
+
+
+@Entity()
+export class Lemma extends BaseEntity {
+  @PrimaryColumn({type: 'text'})
+  lemma_name: string;
+
+  @OneToMany(() => WordForm, (word_form) => word_form.lemma)
+  word_forms: WordForm[];
+
+  @OneToMany(() => Example, (example) => example.lemma)
+  examples: Example[];
+
+  @OneToMany(() => Definition, (definition) => definition.lemma)
+  definitions: Definition[];
+
+  @OneToMany(() => Comment, (comment) => comment.lemma)
+  comments: Comment[];
+
+  @OneToMany(() => Media, (media) => media.lemma)
+  media: Media[];
+
+  @ManyToMany(() => PartOfSpeech)
+  @JoinTable()
+  parts_of_speech: PartOfSpeech[];
+
+}
+
+@Entity()
+export class Lect extends BaseEntity {
+  @PrimaryColumn({type: 'text'})
+  name: string;
+
+  @OneToMany(() => WordForm, (w) => w.lect)
+  word_forms: WordForm[];
+
+}
+
+@Entity()
+export class WordForm extends BaseEntity {
+  @PrimaryColumn({type: 'integer'})
+  word_form_id: number
+  
+  @Column({type: 'text'})
+  word_form: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.word_forms)
+  lemma: Lemma;
+
+  @ManyToOne(() => Lect, (lect) => lect.word_forms)
+  lect: Lect;
+}
+
+@Entity()
+export class Example extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  example_id: number;
+
+  @Column({ nullable: false, type: 'text' })
+  example_text: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.examples)
+  lemma: Lemma; 
+}
+
+@Entity()
+export class Media extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  media_id: number;
+
+  @Column({ nullable: false, type: 'text'})
+  media_url: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.media)
+  lemma: Lemma;
+}
+
+@Entity()
+export class Definition extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  definition_id: number;
+
+  @Column({ nullable: false, type: 'text' })
+  definition_text: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.definitions)
+  lemma: Lemma;
+}
+
+@Entity()
+export class Comment extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  comment_id: number;
+
+  @Column({ nullable: false, type: 'text' })
+  comment_text: string;
+
+  @ManyToOne(() => Lemma, (lemma) => lemma.comments)
+  lemma: Lemma;
+}
+
+@Entity()
+export class PartOfSpeech extends BaseEntity {
+  @PrimaryColumn({type: 'text'})
+  long_form: string;
+
+  @Column({ nullable: false, unique: true, type: 'text' })
+  short_form: string;
+}
+

apps/vdb-backend/src/index.ts

@@ -1,15 +1,184 @@
+import "reflect-metadata";
 import { SAMPLE } from "@repo/common/sample";
 import express from "express";
+import { google, sheets_v4 } from "googleapis";
+import { OAuth2Client } from 'google-auth-library';
+import { appDataSource } from "./config/dbconfig.js";
+import { authorize } from "./auth.js"
+import {
+	Lemma,
+	WordForm,
+	Example,
+	Media,
+	Definition,
+	Comment,
+	PartOfSpeech,
+	Lect,
+} from "./db/dbmodel.js";
 
-const PORT = 1225;
+const RELOAD_SHEET_ON_START = false;
 
-const app = express();
 
-app.get("/sample", (_req, res) => {
-	res.status(200).send(SAMPLE);
-});
 
-app.listen(PORT, () => {
-	console.log(`Backend started @ http://localhost:${PORT} !`);
-	console.log(SAMPLE);
-});
+appDataSource
+	.initialize()
+	.then(async () => {
+		initExpress();
+
+		if (RELOAD_SHEET_ON_START) {
+			const lect_repository = appDataSource.getRepository(Lect);
+			const word_form_repository = appDataSource.getRepository(WordForm);
+			const lemma_repository = appDataSource.getRepository(Lemma);
+
+			await word_form_repository.clear();
+			await lemma_repository.clear();
+			await lect_repository.clear();
+
+			authorize().then(loadSheet).catch(console.error);
+		}
+	})
+	.catch((error) => console.log(error));
+
+
+function initExpress() {
+	const app = express();
+	const PORT = 1225;
+
+	const lect_repository = appDataSource.getRepository(Lect);
+	const word_form_repository = appDataSource.getRepository(WordForm);
+	const lemma_repository = appDataSource.getRepository(Lemma);
+
+	app.get("/", (req, res) => {
+		res.render("search", { title: "Suha", message: "Bratsa" });
+	});
+
+	app.get("/sample", (_req, res) => {
+		res.status(200).send(SAMPLE);
+	});
+
+	app.get("/search", async (req, res) => {
+
+		const search_term = req.query.search_term?.toString();
+
+		if(!search_term) {
+			return;
+		}
+
+		const lemmas: Lemma[] = (
+			await Lemma.find({
+				relations: {
+					word_forms: { lect: true },
+				},
+			})
+		).filter((e) => {
+			for (const wf of e.word_forms) {
+				if (wf.word_form.includes(search_term)) {
+					return true;
+				}
+			}
+		});
+
+		const lects = await Lect.find();
+		return res.render("search_results", {
+			terms: lemmas.length,
+			results: lemmas,
+			lects: lects,
+		});
+	});
+
+	app.set("views", "./res/views");
+	app.set("view engine", "pug");
+
+	app.listen(PORT, () => {
+		console.log(`Backend started @ http://localhost:${PORT} !`);
+		console.log(SAMPLE);
+	});
+}
+
+/**
+ * @param {google.auth.OAuth2Client} auth The authenticated Google OAuth client.
+ */
+async function loadSheet(auth: OAuth2Client) {
+  const options: any = { version: "v4", auth };
+  const sheets = google.sheets(options);
+  const res = await sheets.spreadsheets.values.get({
+    spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
+    range: "al_ko_mit_govor",
+  });
+
+  const rows = res.data.values;
+  
+  const keys_res = await sheets.spreadsheets.values.get({
+    spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
+    range: "klucz",
+  });
+
+  const keys = keys_res.data.values;
+  
+  if (!rows || rows.length === 0) {
+    console.error("No data found.");
+    return;
+  }
+
+  if (!keys || keys.length === 0) {
+    console.error("No lemma keys found.");
+    return;
+  }
+
+  const lects = rows.shift();
+
+  if (keys.length != rows.length) {
+    console.error("Lemma count doesn't match number of rows.");
+    return;
+  }
+
+  if (!lects){
+	console.error("No lects found");
+	return;
+  }
+
+  for (const lect of lects) {
+    let l = new Lect();
+    l.name = lect;
+    l.save();
+    console.log(l);
+  }
+
+  const nikolect = lects.indexOf("Nikomiko");
+
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i];
+
+    const lemma_key = keys[i]?.[0];
+    const lemma = new Lemma();
+
+    if (lemma_key == null || lemma_key.length == 0) {
+      //assign a random UUID to the lemma as punishment for our failures
+      lemma.lemma_name = crypto.randomUUID();
+      console.log(`womp womp, missing lemma name, calling it ${lemma.lemma_name}`)
+    } else {
+      lemma.lemma_name = lemma_key;
+    }
+
+    console.log(lemma);
+    await lemma.save();
+
+    for(let i=0; i<rows.length; i++){
+      const cell = row?.[i];
+      if (cell === null || cell === undefined || (typeof cell === "string" && cell.length === 0)) {
+        continue;
+      }
+
+      for (let word_form of cell.split(";")) {
+        const f = new WordForm();
+        f.word_form = word_form;
+        f.lemma = lemma;
+        f.lect = lects[i];
+        //console.log(f);
+        await f.save();
+      }
+
+    }
+
+  }
+}