vinet/viossa.net
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: degoogle (#37)

Browse source 
* degoogle

* make the source file path a variable
This commit is contained in:
Sheldon Cooper 2026-01-29 19:45:33 -08:00 • committed by GitHub
parent 3940146f3a
commit f3f3ce939e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 101 additions and 183 deletions
Download patch file Download diff file Expand all files Collapse all files

4
apps/vdb-backend/package.json
View file

@ -22,6 +22,7 @@
"@feathersjs/feathers": "^5.0.6",
"@google-cloud/local-auth": "^3.0.1",
"@repo/common": "workspace:*",
"csv-parser": "^3.2.0",
"express": "^5.1.0",
"google-auth-library": "^9.15.1",
"googleapis": "^149.0.0",
@ -35,6 +36,7 @@
"@total-typescript/ts-reset": "^0.6.1",
"@types/express": "^5.0.3",
"@types/node": "^22.5.1",
"tsx": "^4.19.4"
"tsx": "^4.19.4",
"turbo": "^2.8.0"
}
}

11
apps/vdb-backend/res/credentials.template.json
View file

@ -1,11 +0,0 @@
{
"installed": {
"client_id": "",
"project_id": "",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_secret": "",
"redirect_uris": ["http://localhost"]
}
}

6
apps/vdb-backend/res/token.template.json
View file

@ -1,6 +0,0 @@
{
"type": "authorized_user",
"client_id": "",
"client_secret": "",
"refresh_token": ""
}

114
apps/vdb-backend/src/auth.ts
View file

@ -1,114 +0,0 @@
import { promises as fs } from "fs";
import * as path from "path";
import { authenticate } from "@google-cloud/local-auth";
import { google } from "googleapis";
import { JWTInput, OAuth2Client } from "google-auth-library";
import {
GoogleAuth,
JSONClient,
} from "google-auth-library/build/src/auth/googleauth.js";
import z from "zod";
//setup google auth
const SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly"];
const TOKEN_PATH = path.join(process.cwd(), "res/token.secret.json");
const CREDENTIALS_PATH = path.join(
process.cwd(),
"res/credentials.secret.json",
);
/**
* Load or request or authorization to call APIs.
*/
export async function authorize(): Promise<
GoogleAuth<JSONClient> | OAuth2Client
> {
const savedClient = await loadSavedCredentialsIfExist();
if (savedClient) {
return savedClient;
}
const oauthClient = await authenticate({
scopes: SCOPES,
keyfilePath: CREDENTIALS_PATH,
});
if (oauthClient.credentials) {
await saveCredentials(oauthClient);
}
return oauthClient;
}
const jwtInputZod = z.object({
type: z.string(),
client_id: z.string(),
client_secret: z.string(),
refresh_token: z.string(),
}) satisfies z.ZodType<JWTInput>;
async function loadSavedCredentialsIfExist(): Promise<GoogleAuth<JSONClient> | null> {
try {
const content = await fs.readFile(TOKEN_PATH);
const jwtInputRaw = JSON.parse(content.toString());
const jwtInputRes = jwtInputZod.safeParse(jwtInputRaw);
if (!jwtInputRes.success) {
throw new Error("Malformed JWT/token credentials");
}
const jwtInput = jwtInputRes.data;
const jsonClient = google.auth.fromJSON(jwtInput);
const auth = new GoogleAuth({ authClient: jsonClient });
return auth;
} catch (err) {
return null;
}
}
const credentialsZod = z.object({
installed: z.object({
client_id: z.string(),
project_id: z.string(),
auth_uri: z.string(),
token_uri: z.string(),
auth_provider_x509_cert_url: z.string(),
client_secret: z.string(),
redirect_uris: z.array(z.string()),
}),
web: z.optional(
z.object({
client_id: z.string(),
project_id: z.string(),
auth_uri: z.string(),
token_uri: z.string(),
auth_provider_x509_cert_url: z.string(),
client_secret: z.string(),
redirect_uris: z.array(z.string()),
}),
),
});
/**
* Serializes credentials to a file compatible with GoogleAuth.fromJSON.
*/
async function saveCredentials(client: OAuth2Client): Promise<void> {
const content = await fs.readFile(CREDENTIALS_PATH);
const keysRaw = JSON.parse(content.toString());
const keysRes = credentialsZod.safeParse(keysRaw);
if (!keysRes.success) {
throw new Error("Malformed credentials/keys");
}
const keys = keysRes.data;
const key = keys.installed ?? keys.web;
const payload = JSON.stringify({
type: "authorized_user",
client_id: key.client_id,
client_secret: key.client_secret,
refresh_token: client.credentials.refresh_token,
});
await fs.writeFile(TOKEN_PATH, payload);
}

2
apps/vdb-backend/src/db/dbmodel.ts
View file

@ -7,10 +7,8 @@ import {
BaseEntity,
ManyToOne,
OneToMany,
OneToOne,
ManyToMany,
JoinTable,
ColumnType,
} from "typeorm";
@Entity()

67
apps/vdb-backend/src/index.ts
View file

@ -1,21 +1,14 @@
import "reflect-metadata";
import fetch from "node-fetch";
import { SAMPLE } from "@repo/common/sample";
import express from "express";
import { google, sheets_v4 } from "googleapis";
import { OAuth2Client } from "google-auth-library";
import { appDataSource } from "./config/dbconfig.js";
import { authorize } from "./auth.js";
import crypto from "crypto"
import fs from 'fs';
;import { appDataSource } from "./config/dbconfig.js";
import { Lemma, WordForm, Lect } from "./db/dbmodel.js";
import {
GoogleAuth,
JSONClient,
} from "google-auth-library/build/src/auth/googleauth.js";
import "@total-typescript/ts-reset";
const RELOAD_SHEET_ON_START = false;
global.fetch = fetch as any;
const SOURCE_FILE = 'res/sample.tsv'
appDataSource
.initialize()
@ -23,15 +16,7 @@ appDataSource
initExpress();
if (RELOAD_SHEET_ON_START) {
const lect_repository = appDataSource.getRepository(Lect);
const word_form_repository = appDataSource.getRepository(WordForm);
const lemma_repository = appDataSource.getRepository(Lemma);
await word_form_repository.clear();
await lemma_repository.clear();
await lect_repository.clear();
authorize().then(loadSheet).catch(console.error);
await loadSheet();
}
})
.catch((error) => console.log(error));
@ -97,42 +82,28 @@ function initExpress() {
});
}
//todo: redo this without using google sheets; instead, use a local CSV or like.
/**
* @param auth The authenticated Google OAuth client.
*/
async function loadSheet(auth: OAuth2Client | GoogleAuth<JSONClient>) {
async function loadSheet() {
const lect_repository = appDataSource.getRepository(Lect);
const word_form_repository = appDataSource.getRepository(WordForm);
const lemma_repository = appDataSource.getRepository(Lemma);
const options: sheets_v4.Options = { version: "v4", auth };
const sheets = google.sheets(options);
const res = await sheets.spreadsheets.values.get({
spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
range: "al_ko_mit_govor",
});
const rows = res.data.values;
await word_form_repository.clear();
await lemma_repository.clear();
await lect_repository.clear();
const keys_res = await sheets.spreadsheets.values.get({
spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY",
range: "klucz",
});
const keys = keys_res.data.values;
const rawData: string = fs.readFileSync(SOURCE_FILE, 'utf8');
const rows: string[] = rawData.split('\n');
if (!rows || rows.length === 0) {
console.error("No data found.");
return;
}
if (!keys || keys.length === 0) {
console.error("No lemma keys found.");
return;
}
const lects = rows.shift()?.split('\t');
const lects = rows.shift();
const keys = rows.map(row=>row.split('\t')[0]?.split(';')[0]);
console.log(keys);
if (keys.length != rows.length) {
console.error("Lemma count doesn't match number of rows.");
@ -154,9 +125,10 @@ async function loadSheet(auth: OAuth2Client | GoogleAuth<JSONClient>) {
const nikolect = lects.indexOf("Nikomiko");
const lemmas = Array<Lemma>();
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
const row = rows[i]?.split('\t');
const lemma_key = keys[i]?.[0];
// todo
const lemma_key = null;
const lemma = new Lemma();
if (lemma_key == null || lemma_key.length == 0) {
@ -172,7 +144,7 @@ async function loadSheet(auth: OAuth2Client | GoogleAuth<JSONClient>) {
lemmas.push(lemma);
lemma.word_forms = Array<WordForm>();
for (let i = 0; i < rows.length; i++) {
for (let i = 0; i < row.length; i++) {
const cell = row?.[i];
if (
cell === null ||
@ -186,7 +158,6 @@ async function loadSheet(auth: OAuth2Client | GoogleAuth<JSONClient>) {
const f = new WordForm();
f.word_form = word_form;
f.lect = lects[i];
//console.log(f);
lemma.word_forms.push(f);
}
}