From f3f3ce939e13205d3d8d95818c1577e373277bef Mon Sep 17 00:00:00 2001 From: Sheldon Cooper Date: Thu, 29 Jan 2026 19:45:33 -0800 Subject: [PATCH] feat: degoogle (#37) * degoogle * make the source file path a variable --- .gitignore | 1 + apps/vdb-backend/package.json | 4 +- .../vdb-backend/res/credentials.template.json | 11 -- apps/vdb-backend/res/token.template.json | 6 - apps/vdb-backend/src/auth.ts | 114 ------------------ apps/vdb-backend/src/db/dbmodel.ts | 2 - apps/vdb-backend/src/index.ts | 67 +++------- .../src/components/pages/KotobaPage.vue | 2 +- pnpm-lock.yaml | 77 ++++++++++++ 9 files changed, 101 insertions(+), 183 deletions(-) delete mode 100644 apps/vdb-backend/res/credentials.template.json delete mode 100644 apps/vdb-backend/res/token.template.json delete mode 100644 apps/vdb-backend/src/auth.ts diff --git a/.gitignore b/.gitignore index b57dc93..25984dd 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ dist-ssr *.secret* .turbo *.sqlite +*.tsv diff --git a/apps/vdb-backend/package.json b/apps/vdb-backend/package.json index 3759c5a..2d59cd6 100644 --- a/apps/vdb-backend/package.json +++ b/apps/vdb-backend/package.json @@ -22,6 +22,7 @@ "@feathersjs/feathers": "^5.0.6", "@google-cloud/local-auth": "^3.0.1", "@repo/common": "workspace:*", + "csv-parser": "^3.2.0", "express": "^5.1.0", "google-auth-library": "^9.15.1", "googleapis": "^149.0.0", @@ -35,6 +36,7 @@ "@total-typescript/ts-reset": "^0.6.1", "@types/express": "^5.0.3", "@types/node": "^22.5.1", - "tsx": "^4.19.4" + "tsx": "^4.19.4", + "turbo": "^2.8.0" } } diff --git a/apps/vdb-backend/res/credentials.template.json b/apps/vdb-backend/res/credentials.template.json deleted file mode 100644 index e3ac6d0..0000000 --- a/apps/vdb-backend/res/credentials.template.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "installed": { - "client_id": "", - "project_id": "", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_secret": "", - "redirect_uris": ["http://localhost"] - } -} diff --git a/apps/vdb-backend/res/token.template.json b/apps/vdb-backend/res/token.template.json deleted file mode 100644 index 36fdd60..0000000 --- a/apps/vdb-backend/res/token.template.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": "authorized_user", - "client_id": "", - "client_secret": "", - "refresh_token": "" -} diff --git a/apps/vdb-backend/src/auth.ts b/apps/vdb-backend/src/auth.ts deleted file mode 100644 index 43422a9..0000000 --- a/apps/vdb-backend/src/auth.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { promises as fs } from "fs"; -import * as path from "path"; -import { authenticate } from "@google-cloud/local-auth"; -import { google } from "googleapis"; -import { JWTInput, OAuth2Client } from "google-auth-library"; -import { - GoogleAuth, - JSONClient, -} from "google-auth-library/build/src/auth/googleauth.js"; -import z from "zod"; - -//setup google auth -const SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly"]; -const TOKEN_PATH = path.join(process.cwd(), "res/token.secret.json"); -const CREDENTIALS_PATH = path.join( - process.cwd(), - "res/credentials.secret.json", -); - -/** - * Load or request or authorization to call APIs. - */ -export async function authorize(): Promise< - GoogleAuth | OAuth2Client -> { - const savedClient = await loadSavedCredentialsIfExist(); - if (savedClient) { - return savedClient; - } - - const oauthClient = await authenticate({ - scopes: SCOPES, - keyfilePath: CREDENTIALS_PATH, - }); - - if (oauthClient.credentials) { - await saveCredentials(oauthClient); - } - - return oauthClient; -} - -const jwtInputZod = z.object({ - type: z.string(), - client_id: z.string(), - client_secret: z.string(), - refresh_token: z.string(), -}) satisfies z.ZodType; - -async function loadSavedCredentialsIfExist(): Promise | null> { - try { - const content = await fs.readFile(TOKEN_PATH); - const jwtInputRaw = JSON.parse(content.toString()); - - const jwtInputRes = jwtInputZod.safeParse(jwtInputRaw); - if (!jwtInputRes.success) { - throw new Error("Malformed JWT/token credentials"); - } - - const jwtInput = jwtInputRes.data; - const jsonClient = google.auth.fromJSON(jwtInput); - const auth = new GoogleAuth({ authClient: jsonClient }); - return auth; - } catch (err) { - return null; - } -} - -const credentialsZod = z.object({ - installed: z.object({ - client_id: z.string(), - project_id: z.string(), - auth_uri: z.string(), - token_uri: z.string(), - auth_provider_x509_cert_url: z.string(), - client_secret: z.string(), - redirect_uris: z.array(z.string()), - }), - web: z.optional( - z.object({ - client_id: z.string(), - project_id: z.string(), - auth_uri: z.string(), - token_uri: z.string(), - auth_provider_x509_cert_url: z.string(), - client_secret: z.string(), - redirect_uris: z.array(z.string()), - }), - ), -}); - -/** - * Serializes credentials to a file compatible with GoogleAuth.fromJSON. - */ -async function saveCredentials(client: OAuth2Client): Promise { - const content = await fs.readFile(CREDENTIALS_PATH); - const keysRaw = JSON.parse(content.toString()); - - const keysRes = credentialsZod.safeParse(keysRaw); - if (!keysRes.success) { - throw new Error("Malformed credentials/keys"); - } - - const keys = keysRes.data; - const key = keys.installed ?? keys.web; - const payload = JSON.stringify({ - type: "authorized_user", - client_id: key.client_id, - client_secret: key.client_secret, - refresh_token: client.credentials.refresh_token, - }); - - await fs.writeFile(TOKEN_PATH, payload); -} diff --git a/apps/vdb-backend/src/db/dbmodel.ts b/apps/vdb-backend/src/db/dbmodel.ts index 18143e4..953bea9 100644 --- a/apps/vdb-backend/src/db/dbmodel.ts +++ b/apps/vdb-backend/src/db/dbmodel.ts @@ -7,10 +7,8 @@ import { BaseEntity, ManyToOne, OneToMany, - OneToOne, ManyToMany, JoinTable, - ColumnType, } from "typeorm"; @Entity() diff --git a/apps/vdb-backend/src/index.ts b/apps/vdb-backend/src/index.ts index 0f76771..732530b 100644 --- a/apps/vdb-backend/src/index.ts +++ b/apps/vdb-backend/src/index.ts @@ -1,21 +1,14 @@ import "reflect-metadata"; -import fetch from "node-fetch"; import { SAMPLE } from "@repo/common/sample"; import express from "express"; -import { google, sheets_v4 } from "googleapis"; -import { OAuth2Client } from "google-auth-library"; -import { appDataSource } from "./config/dbconfig.js"; -import { authorize } from "./auth.js"; +import crypto from "crypto" +import fs from 'fs'; +;import { appDataSource } from "./config/dbconfig.js"; import { Lemma, WordForm, Lect } from "./db/dbmodel.js"; -import { - GoogleAuth, - JSONClient, -} from "google-auth-library/build/src/auth/googleauth.js"; import "@total-typescript/ts-reset"; const RELOAD_SHEET_ON_START = false; - -global.fetch = fetch as any; +const SOURCE_FILE = 'res/sample.tsv' appDataSource .initialize() @@ -23,15 +16,7 @@ appDataSource initExpress(); if (RELOAD_SHEET_ON_START) { - const lect_repository = appDataSource.getRepository(Lect); - const word_form_repository = appDataSource.getRepository(WordForm); - const lemma_repository = appDataSource.getRepository(Lemma); - - await word_form_repository.clear(); - await lemma_repository.clear(); - await lect_repository.clear(); - - authorize().then(loadSheet).catch(console.error); + await loadSheet(); } }) .catch((error) => console.log(error)); @@ -97,42 +82,28 @@ function initExpress() { }); } - -//todo: redo this without using google sheets; instead, use a local CSV or like. -/** - * @param auth The authenticated Google OAuth client. - */ -async function loadSheet(auth: OAuth2Client | GoogleAuth) { +async function loadSheet() { const lect_repository = appDataSource.getRepository(Lect); const word_form_repository = appDataSource.getRepository(WordForm); const lemma_repository = appDataSource.getRepository(Lemma); - const options: sheets_v4.Options = { version: "v4", auth }; - const sheets = google.sheets(options); - const res = await sheets.spreadsheets.values.get({ - spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY", - range: "al_ko_mit_govor", - }); - const rows = res.data.values; + await word_form_repository.clear(); + await lemma_repository.clear(); + await lect_repository.clear(); - const keys_res = await sheets.spreadsheets.values.get({ - spreadsheetId: "1-YkCeynx_-KYdubvt14augSPo37_20YgUv_f-i8HVwY", - range: "klucz", - }); - - const keys = keys_res.data.values; + + const rawData: string = fs.readFileSync(SOURCE_FILE, 'utf8'); + const rows: string[] = rawData.split('\n'); if (!rows || rows.length === 0) { console.error("No data found."); return; } - if (!keys || keys.length === 0) { - console.error("No lemma keys found."); - return; - } + const lects = rows.shift()?.split('\t'); - const lects = rows.shift(); + const keys = rows.map(row=>row.split('\t')[0]?.split(';')[0]); + console.log(keys); if (keys.length != rows.length) { console.error("Lemma count doesn't match number of rows."); @@ -154,9 +125,10 @@ async function loadSheet(auth: OAuth2Client | GoogleAuth) { const nikolect = lects.indexOf("Nikomiko"); const lemmas = Array(); for (let i = 0; i < rows.length; i++) { - const row = rows[i]; + const row = rows[i]?.split('\t'); - const lemma_key = keys[i]?.[0]; + // todo + const lemma_key = null; const lemma = new Lemma(); if (lemma_key == null || lemma_key.length == 0) { @@ -172,7 +144,7 @@ async function loadSheet(auth: OAuth2Client | GoogleAuth) { lemmas.push(lemma); lemma.word_forms = Array(); - for (let i = 0; i < rows.length; i++) { + for (let i = 0; i < row.length; i++) { const cell = row?.[i]; if ( cell === null || @@ -186,7 +158,6 @@ async function loadSheet(auth: OAuth2Client | GoogleAuth) { const f = new WordForm(); f.word_form = word_form; f.lect = lects[i]; - //console.log(f); lemma.word_forms.push(f); } } diff --git a/apps/vdn-static/src/components/pages/KotobaPage.vue b/apps/vdn-static/src/components/pages/KotobaPage.vue index 701555b..0b8268b 100644 --- a/apps/vdn-static/src/components/pages/KotobaPage.vue +++ b/apps/vdn-static/src/components/pages/KotobaPage.vue @@ -6,7 +6,7 @@

- ⚠under construction :3⚠ + ⚠ Under construction :3 ⚠

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6126f3a..a71f2a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@repo/common': specifier: workspace:* version: link:../../libs/common + csv-parser: + specifier: ^3.2.0 + version: 3.2.0 express: specifier: ^5.1.0 version: 5.1.0 @@ -60,6 +63,9 @@ importers: tsx: specifier: ^4.19.4 version: 4.20.2 + turbo: + specifier: ^2.8.0 + version: 2.8.0 apps/vdn-static: dependencies: @@ -1192,6 +1198,11 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + csv-parser@3.2.0: + resolution: {integrity: sha512-fgKbp+AJbn1h2dcAHKIdKNSSjfp43BZZykXsCjzALjKy80VXQNHPFJ6T9Afwdzoj24aMkq8GwDS7KGcDPpejrA==} + engines: {node: '>= 10'} + hasBin: true + data-uri-to-buffer@4.0.1: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} @@ -2450,10 +2461,12 @@ packages: tar@6.2.1: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exhorbitant rates) by contacting i@izs.me tar@7.4.3: resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} engines: {node: '>=18'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exhorbitant rates) by contacting i@izs.me tinyglobby@0.2.14: resolution: {integrity: sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==} @@ -2496,35 +2509,69 @@ packages: cpu: [x64] os: [darwin] + turbo-darwin-64@2.8.0: + resolution: {integrity: sha512-N7f4PYqz25yk8c5kituk09bJ89tE4wPPqKXgYccT6nbEQnGnrdvlyCHLyqViNObTgjjrddqjb1hmDkv7VcxE0g==} + cpu: [x64] + os: [darwin] + turbo-darwin-arm64@2.5.4: resolution: {integrity: sha512-2+Nx6LAyuXw2MdXb7pxqle3MYignLvS7OwtsP9SgtSBaMlnNlxl9BovzqdYAgkUW3AsYiQMJ/wBRb7d+xemM5A==} cpu: [arm64] os: [darwin] + turbo-darwin-arm64@2.8.0: + resolution: {integrity: sha512-eVzejaP5fn51gmJAPW68U6mSjFaAZ26rPiE36mMdk+tMC4XBGmJHT/fIgrhcrXMvINCl27RF8VmguRe+MBlSuQ==} + cpu: [arm64] + os: [darwin] + turbo-linux-64@2.5.4: resolution: {integrity: sha512-5May2kjWbc8w4XxswGAl74GZ5eM4Gr6IiroqdLhXeXyfvWEdm2mFYCSWOzz0/z5cAgqyGidF1jt1qzUR8hTmOA==} cpu: [x64] os: [linux] + turbo-linux-64@2.8.0: + resolution: {integrity: sha512-ILR45zviYae3icf4cmUISdj8X17ybNcMh3Ms4cRdJF5sS50qDDTv8qeWqO/lPeHsu6r43gVWDofbDZYVuXYL7Q==} + cpu: [x64] + os: [linux] + turbo-linux-arm64@2.5.4: resolution: {integrity: sha512-/2yqFaS3TbfxV3P5yG2JUI79P7OUQKOUvAnx4MV9Bdz6jqHsHwc9WZPpO4QseQm+NvmgY6ICORnoVPODxGUiJg==} cpu: [arm64] os: [linux] + turbo-linux-arm64@2.8.0: + resolution: {integrity: sha512-z9pUa8ENFuHmadPfjEYMRWlXO82t1F/XBDx2XTg+cWWRZHf85FnEB6D4ForJn/GoKEEvwdPhFLzvvhOssom2ug==} + cpu: [arm64] + os: [linux] + turbo-windows-64@2.5.4: resolution: {integrity: sha512-EQUO4SmaCDhO6zYohxIjJpOKRN3wlfU7jMAj3CgcyTPvQR/UFLEKAYHqJOnJtymbQmiiM/ihX6c6W6Uq0yC7mA==} cpu: [x64] os: [win32] + turbo-windows-64@2.8.0: + resolution: {integrity: sha512-J6juRSRjmSErEqJCv7nVIq2DgZ2NHXqyeV8NQTFSyIvrThKiWe7FDOO6oYpuR06+C1NW82aoN4qQt4/gYvz25w==} + cpu: [x64] + os: [win32] + turbo-windows-arm64@2.5.4: resolution: {integrity: sha512-oQ8RrK1VS8lrxkLriotFq+PiF7iiGgkZtfLKF4DDKsmdbPo0O9R2mQxm7jHLuXraRCuIQDWMIw6dpcr7Iykf4A==} cpu: [arm64] os: [win32] + turbo-windows-arm64@2.8.0: + resolution: {integrity: sha512-qarBZvCu6uka35739TS+y/3CBU3zScrVAfohAkKHG+So+93Wn+5tKArs8HrO2fuTaGou8fMIeTV7V5NgzCVkSQ==} + cpu: [arm64] + os: [win32] + turbo@2.5.4: resolution: {integrity: sha512-kc8ZibdRcuWUG1pbYSBFWqmIjynlD8Lp7IB6U3vIzvOv9VG+6Sp8bzyeBWE3Oi8XV5KsQrznyRTBPvrf99E4mA==} hasBin: true + turbo@2.8.0: + resolution: {integrity: sha512-hYbxnLEdvJF+DLALS+Ia+PbfNtn0sDP0hH2u9AFoskSUDmcVHSrtwHpzdX94MrRJKo9D9tYxY3MyP20gnlrWyA==} + hasBin: true + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -2710,6 +2757,7 @@ packages: vue-i18n@11.1.5: resolution: {integrity: sha512-XCwuaEA5AF97g1frvH/EI1zI9uo1XKTf2/OCFgts7NvUWRsjlgeHPrkJV+a3gpzai2pC4quZ4AnOHFO8QK9hsg==} engines: {node: '>= 16'} + deprecated: This version is NOT deprecated. Previous deprecation was a mistake. peerDependencies: vue: ^3.0.0 @@ -3793,6 +3841,8 @@ snapshots: csstype@3.1.3: {} + csv-parser@3.2.0: {} + data-uri-to-buffer@4.0.1: {} dayjs@1.11.13: {} @@ -5240,21 +5290,39 @@ snapshots: turbo-darwin-64@2.5.4: optional: true + turbo-darwin-64@2.8.0: + optional: true + turbo-darwin-arm64@2.5.4: optional: true + turbo-darwin-arm64@2.8.0: + optional: true + turbo-linux-64@2.5.4: optional: true + turbo-linux-64@2.8.0: + optional: true + turbo-linux-arm64@2.5.4: optional: true + turbo-linux-arm64@2.8.0: + optional: true + turbo-windows-64@2.5.4: optional: true + turbo-windows-64@2.8.0: + optional: true + turbo-windows-arm64@2.5.4: optional: true + turbo-windows-arm64@2.8.0: + optional: true + turbo@2.5.4: optionalDependencies: turbo-darwin-64: 2.5.4 @@ -5264,6 +5332,15 @@ snapshots: turbo-windows-64: 2.5.4 turbo-windows-arm64: 2.5.4 + turbo@2.8.0: + optionalDependencies: + turbo-darwin-64: 2.8.0 + turbo-darwin-arm64: 2.8.0 + turbo-linux-64: 2.8.0 + turbo-linux-arm64: 2.8.0 + turbo-windows-64: 2.8.0 + turbo-windows-arm64: 2.8.0 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1