update schema gen

+ support for PocketBase v0.22.23
+ support for Deno 2
+ now depends on pocketbase npm package
This commit is contained in:
michal-kapala 2024-11-06 16:17:06 +01:00
parent 8692f25e71
commit ce03fd9991
9 changed files with 130 additions and 86 deletions

View File

@ -29,7 +29,7 @@ PocketBase types are:
# Configuration # Configuration
Install the latest [Deno runtime](https://deno.land/) to run the scripts. Install the latest [Deno runtime](https://deno.com/) to run the scripts.
In the root directory create `.env` file with the following environment In the root directory create `.env` file with the following environment
variables: variables:

20
csv.ts
View File

@ -1,12 +1,9 @@
// @deno-types="https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.d.mts" import PocketBase from 'pocketbase';
import PocketBase, {
Collection,
SchemaField,
} from "https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.mjs";
import "https://deno.land/std@0.178.0/dotenv/load.ts"; import "https://deno.land/std@0.178.0/dotenv/load.ts";
import { parse } from "https://deno.land/std@0.175.0/flags/mod.ts"; import { parse } from "https://deno.land/std@0.175.0/flags/mod.ts";
import { parseData, readCsv } from "./utils/csv.ts"; import { parseData, readCsv } from "./utils/csv.ts";
import { createSchema } from "./utils/pocketbase.ts"; import { createSchema } from "./utils/pocketbase.ts";
import { Collection } from "./types/pocketbase.ts";
/** /**
* Structures and populates a new collection from a CSV file. * Structures and populates a new collection from a CSV file.
@ -53,6 +50,10 @@ async function importCsv() {
// read the file // read the file
const data = await readCsv(options.input, options); const data = await readCsv(options.input, options);
if (data === null) {
Deno.exit(-1);
return
}
// sanitize the file name for collection name // sanitize the file name for collection name
const collectName = options.input.replace(".csv", ""); const collectName = options.input.replace(".csv", "");
@ -63,25 +64,24 @@ async function importCsv() {
const _authResponse = await pb.admins.authWithPassword(adminName, adminPass); const _authResponse = await pb.admins.authWithPassword(adminName, adminPass);
// collection schema object // collection schema object
const schema: SchemaField[] = createSchema(data, options.id, "csv"); const schema = createSchema(data, options.id, "csv");
const creationDate = new Date().toISOString(); const creationDate = new Date().toISOString();
// the new collection // the new collection
const collection = new Collection({ const collection: Collection = {
name: collectName, name: collectName,
type: "base", type: "base",
system: false, system: false,
schema, schema,
indexes: [],
listRule: null, listRule: null,
viewRule: null, viewRule: null,
createRule: null, createRule: null,
updateRule: null, updateRule: null,
deleteRule: null, deleteRule: null,
options: {}, options: {},
created: creationDate, };
updated: creationDate,
});
// show the submitted collection // show the submitted collection
console.log(collection); console.log(collection);

View File

@ -1,5 +1,8 @@
{ {
"tasks": { "tasks": {
"dev": "deno run --watch main.ts" "dev": "deno run --watch main.ts"
},
"imports": {
"pocketbase": "npm:pocketbase@^0.21.5"
} }
} }

21
deno.lock generated
View File

@ -1,5 +1,13 @@
{ {
"version": "2", "version": "4",
"specifiers": {
"npm:pocketbase@~0.21.5": "0.21.5"
},
"npm": {
"pocketbase@0.21.5": {
"integrity": "sha512-bnI/uinnQps+ElSlzxkc4yvwuSFfKcoszDtXH/4QT2FhGq2mJVUvDlxn+rjRXVntUjPfmMG5LEPZ1eGqV6ssog=="
}
},
"remote": { "remote": {
"https://deno.land/std@0.164.0/_util/asserts.ts": "d0844e9b62510f89ce1f9878b046f6a57bf88f208a10304aab50efcb48365272", "https://deno.land/std@0.164.0/_util/asserts.ts": "d0844e9b62510f89ce1f9878b046f6a57bf88f208a10304aab50efcb48365272",
"https://deno.land/std@0.164.0/bytes/bytes_list.ts": "aba5e2369e77d426b10af1de0dcc4531acecec27f9b9056f4f7bfbf8ac147ab4", "https://deno.land/std@0.164.0/bytes/bytes_list.ts": "aba5e2369e77d426b10af1de0dcc4531acecec27f9b9056f4f7bfbf8ac147ab4",
@ -8,7 +16,6 @@
"https://deno.land/std@0.164.0/fmt/colors.ts": "9e36a716611dcd2e4865adea9c4bec916b5c60caad4cdcdc630d4974e6bb8bd4", "https://deno.land/std@0.164.0/fmt/colors.ts": "9e36a716611dcd2e4865adea9c4bec916b5c60caad4cdcdc630d4974e6bb8bd4",
"https://deno.land/std@0.164.0/fs/exists.ts": "6a447912e49eb79cc640adacfbf4b0baf8e17ede6d5bed057062ce33c4fa0d68", "https://deno.land/std@0.164.0/fs/exists.ts": "6a447912e49eb79cc640adacfbf4b0baf8e17ede6d5bed057062ce33c4fa0d68",
"https://deno.land/std@0.164.0/io/buffer.ts": "245f1762a949082ddc0a6e9b15589d0be2d29c150266decd04320b8a8318f9f6", "https://deno.land/std@0.164.0/io/buffer.ts": "245f1762a949082ddc0a6e9b15589d0be2d29c150266decd04320b8a8318f9f6",
"https://deno.land/std@0.164.0/io/types.d.ts": "107e1e64834c5ba917c783f446b407d33432c5d612c4b3430df64fc2b4ecf091",
"https://deno.land/std@0.164.0/log/handlers.ts": "61ab932822ba268ad42b25de6f0014892fe469df7627b14245ad32db6fd8b54d", "https://deno.land/std@0.164.0/log/handlers.ts": "61ab932822ba268ad42b25de6f0014892fe469df7627b14245ad32db6fd8b54d",
"https://deno.land/std@0.164.0/log/levels.ts": "82c965b90f763b5313e7595d4ba78d5095a13646d18430ebaf547526131604d1", "https://deno.land/std@0.164.0/log/levels.ts": "82c965b90f763b5313e7595d4ba78d5095a13646d18430ebaf547526131604d1",
"https://deno.land/std@0.164.0/log/logger.ts": "b545159727b023825ee6814d9178fb2a7472f8d95d704c253b771b95c658cf8c", "https://deno.land/std@0.164.0/log/logger.ts": "b545159727b023825ee6814d9178fb2a7472f8d95d704c253b771b95c658cf8c",
@ -21,10 +28,12 @@
"https://deno.land/std@0.178.0/dotenv/load.ts": "0636983549b98f29ab75c9a22a42d9723f0a389ece5498fe971e7bb2556a12e2", "https://deno.land/std@0.178.0/dotenv/load.ts": "0636983549b98f29ab75c9a22a42d9723f0a389ece5498fe971e7bb2556a12e2",
"https://deno.land/std@0.178.0/dotenv/mod.ts": "8dcbc8a40b896a0bf094582aaeadbfc76d3528872faf2efc0302beb1d2f6afd0", "https://deno.land/std@0.178.0/dotenv/mod.ts": "8dcbc8a40b896a0bf094582aaeadbfc76d3528872faf2efc0302beb1d2f6afd0",
"https://deno.land/x/csv@v0.8.0/deps.ts": "597e3d0c81eca1c519ce20f0a7ed573d31cfbb5625c5013763550d717ba69dfa", "https://deno.land/x/csv@v0.8.0/deps.ts": "597e3d0c81eca1c519ce20f0a7ed573d31cfbb5625c5013763550d717ba69dfa",
"https://deno.land/x/csv@v0.8.0/mod.ts": "2a13285c8716257aa5d6b5021d45c74dcb275eaaa46776106ddf78b071cabd88",
"https://deno.land/x/csv@v0.8.0/reader.ts": "ad6543223f8d1c17c78b18dbaa464c3d81f7a0970b08aa5fbde9eb48e2a680d1", "https://deno.land/x/csv@v0.8.0/reader.ts": "ad6543223f8d1c17c78b18dbaa464c3d81f7a0970b08aa5fbde9eb48e2a680d1",
"https://deno.land/x/csv@v0.8.0/utils.ts": "7f2467acb031244c150774b9d6caa5e8c40f9857031a947c6ad20765c5d7bd20", "https://deno.land/x/csv@v0.8.0/utils.ts": "7f2467acb031244c150774b9d6caa5e8c40f9857031a947c6ad20765c5d7bd20"
"https://deno.land/x/csv@v0.8.0/writer.ts": "18a46b755cc215695862d3a896b3a7b24b33cb7ee626cbebeb7138193cb7edcf", },
"https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.mjs": "bcc669b0d3844523cd8121a75c2ead5e14f1dc9426bcf8904621fc26a75b44c0" "workspace": {
"dependencies": [
"npm:pocketbase@~0.21.5"
]
} }
} }

20
json.ts
View File

@ -1,12 +1,9 @@
// @deno-types="https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.d.mts" import PocketBase from 'pocketbase';
import PocketBase, {
Collection,
SchemaField,
} from "https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.mjs";
import "https://deno.land/std@0.178.0/dotenv/load.ts"; import "https://deno.land/std@0.178.0/dotenv/load.ts";
import { parse } from "https://deno.land/std@0.175.0/flags/mod.ts"; import { parse } from "https://deno.land/std@0.175.0/flags/mod.ts";
import { readJson, resolveConflicts } from "./utils/json.ts"; import { readJson, resolveConflicts } from "./utils/json.ts";
import { createSchema } from "./utils/pocketbase.ts"; import { createSchema } from "./utils/pocketbase.ts";
import { Collection } from './types/pocketbase.ts'
/** /**
* Structures and populates a new collection from a JSON file. * Structures and populates a new collection from a JSON file.
@ -51,32 +48,29 @@ async function importJson() {
const _authResponse = await pb.admins.authWithPassword(adminName, adminPass); const _authResponse = await pb.admins.authWithPassword(adminName, adminPass);
// collection schema object // collection schema object
const schema: SchemaField[] = createSchema(data, options.id, "json"); const schema = createSchema(data, options.id, "json");
const creationDate = new Date().toISOString();
// the new collection // the new collection
const collection = new Collection({ const collection: Collection = {
name: collectName, name: collectName,
type: "base", type: "base",
system: false, system: false,
schema, schema,
indexes: [],
listRule: null, listRule: null,
viewRule: null, viewRule: null,
createRule: null, createRule: null,
updateRule: null, updateRule: null,
deleteRule: null, deleteRule: null,
options: {}, options: {},
created: creationDate, };
updated: creationDate,
});
// show the submitted collection // show the submitted collection
console.log(collection); console.log(collection);
// create the new collection // create the new collection
// import will fail if a collection with the same name exists // import will fail if a collection with the same name exists
await pb.collections.import([collection]); await pb.collections.import([collection], false);
console.log( console.log(
`%c[Import] Collection '${collectName}' created!`, `%c[Import] Collection '${collectName}' created!`,

View File

@ -33,3 +33,33 @@ export const POCKETBASE_SYSFIELD = [
"created", "created",
"updated", "updated",
]; ];
export type Options = {
[key: string]: any;
};
export type SchemaField = {
id?: string;
name: string;
type: PocketbaseType;
required: boolean;
system: boolean;
presentable: boolean;
unique: boolean;
options: Options;
};
export type Collection = {
id?: string;
name: string;
type: string;
system: boolean;
schema: SchemaField[];
indexes: string[];
listRule: string | null;
viewRule: string |null;
createRule: string |null;
updateRule: string |null;
deleteRule: string |null;
options: Options;
};

View File

@ -8,13 +8,12 @@ import {
ParserOptions, ParserOptions,
RawCsvRow, RawCsvRow,
} from "../types/csv.ts"; } from "../types/csv.ts";
// @deno-types="https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.d.mts"
import { SchemaField } from "https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.mjs";
import { import {
POCKETBASE_SYSFIELD, POCKETBASE_SYSFIELD,
POCKETBASE_TYPE, POCKETBASE_TYPE,
PocketbaseRowSchema, PocketbaseRowSchema,
PocketbaseType, PocketbaseType,
SchemaField
} from "../types/pocketbase.ts"; } from "../types/pocketbase.ts";
import { createSchemaField, generateRowSchema } from "./pocketbase.ts"; import { createSchemaField, generateRowSchema } from "./pocketbase.ts";
import { isBool, isDate, isEmail, isJson, isNumber, isUrl } from "./regex.ts"; import { isBool, isDate, isEmail, isJson, isNumber, isUrl } from "./regex.ts";
@ -56,7 +55,7 @@ export async function readCsv(filename: string, options: CsvOptions) {
async function parseCsv( async function parseCsv(
filename: string | null, filename: string | null,
csvOptions: ParserOptions, csvOptions: ParserOptions,
): Promise<RawCsvRow[] | null> { ) {
const data: RawCsvRow[] = []; const data: RawCsvRow[] = [];
try { try {
@ -85,7 +84,7 @@ async function parseCsv(
* @param value Raw string value * @param value Raw string value
* @returns * @returns
*/ */
export function parseBool(value: string): boolean { export function parseBool(value: string) {
return ["true", "1"].includes(value); return ["true", "1"].includes(value);
} }
@ -95,7 +94,7 @@ export function parseBool(value: string): boolean {
* @param prop - Column name * @param prop - Column name
* @returns `SchemaField` * @returns `SchemaField`
*/ */
export function addSchemaField(data: RawCsvRow[], prop: string): SchemaField { export function addSchemaField(data: RawCsvRow[], prop: string) {
// The new column is prefixed with underscore if it conflicts with a system field // The new column is prefixed with underscore if it conflicts with a system field
const targetProp = POCKETBASE_SYSFIELD.includes(prop.toLowerCase()) const targetProp = POCKETBASE_SYSFIELD.includes(prop.toLowerCase())
? `_${prop}` ? `_${prop}`
@ -139,7 +138,7 @@ export function addSchemaField(data: RawCsvRow[], prop: string): SchemaField {
export function parseData( export function parseData(
data: RawCsvRow[], data: RawCsvRow[],
schema: SchemaField[], schema: SchemaField[],
): ParsedRow[] { ) {
const rows: ParsedRow[] = []; const rows: ParsedRow[] = [];
// create a row schema for the collection // create a row schema for the collection
@ -159,7 +158,7 @@ export function parseData(
* @param schema - Row type template * @param schema - Row type template
* @returns * @returns
*/ */
function parseRow(rawRow: RawCsvRow, schema: PocketbaseRowSchema): ParsedRow { function parseRow(rawRow: RawCsvRow, schema: PocketbaseRowSchema) {
let parsedRow: ParsedRow = {}; let parsedRow: ParsedRow = {};
const keys = Object.keys(rawRow); const keys = Object.keys(rawRow);

View File

@ -1,5 +1,3 @@
// @deno-types="https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.d.mts"
import { SchemaField } from "https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.mjs";
import { RawJsonRow } from "../types/json.ts"; import { RawJsonRow } from "../types/json.ts";
import { POCKETBASE_SYSFIELD } from "../types/pocketbase.ts"; import { POCKETBASE_SYSFIELD } from "../types/pocketbase.ts";
import { createSchemaField } from "./pocketbase.ts"; import { createSchemaField } from "./pocketbase.ts";
@ -59,7 +57,7 @@ async function parseJson(filename: string) {
* @param prop Column name. * @param prop Column name.
* @returns `SchemaField` * @returns `SchemaField`
*/ */
export function addSchemaField(data: RawJsonRow[], prop: string): SchemaField { export function addSchemaField(data: RawJsonRow[], prop: string) {
// The new column is prefixed with underscore if it conflicts with a system field // The new column is prefixed with underscore if it conflicts with a system field
const targetProp = POCKETBASE_SYSFIELD.includes(prop.toLowerCase()) const targetProp = POCKETBASE_SYSFIELD.includes(prop.toLowerCase())
? `_${prop}` ? `_${prop}`
@ -111,7 +109,7 @@ export function addSchemaField(data: RawJsonRow[], prop: string): SchemaField {
* @param data Data rows. * @param data Data rows.
* @returns * @returns
*/ */
export function resolveConflicts(data: RawJsonRow[]): RawJsonRow[] { export function resolveConflicts(data: RawJsonRow[]) {
const rows: RawJsonRow[] = []; const rows: RawJsonRow[] = [];
for (const r of data) { for (const r of data) {

View File

@ -1,11 +1,10 @@
// @deno-types="https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.d.mts"
import { SchemaField } from "https://unpkg.com/pocketbase@0.12.0/dist/pocketbase.es.mjs";
import { RawCsvRow } from "../types/csv.ts"; import { RawCsvRow } from "../types/csv.ts";
import { RawJsonRow } from "../types/json.ts"; import { RawJsonRow } from "../types/json.ts";
import { import {
POCKETBASE_TYPE, POCKETBASE_TYPE,
PocketbaseRowSchema, PocketbaseRowSchema,
PocketbaseType, PocketbaseType,
SchemaField
} from "../types/pocketbase.ts"; } from "../types/pocketbase.ts";
import { addSchemaField as addCsvSchemaField } from "./csv.ts"; import { addSchemaField as addCsvSchemaField } from "./csv.ts";
import { addSchemaField as addJsonSchemaField } from "./json.ts"; import { addSchemaField as addJsonSchemaField } from "./json.ts";
@ -29,6 +28,7 @@ export function getSchemaType(
"color: red", "color: red",
); );
Deno.exit(-1); Deno.exit(-1);
return "text"
} }
switch (schemaField.type) { switch (schemaField.type) {
@ -59,6 +59,7 @@ export function getSchemaType(
"color: red", "color: red",
); );
Deno.exit(-2); Deno.exit(-2);
return "text"
} }
} }
@ -74,84 +75,94 @@ export function createSchemaField(
): SchemaField { ): SchemaField {
switch (type) { switch (type) {
case POCKETBASE_TYPE.BOOL: case POCKETBASE_TYPE.BOOL:
return new SchemaField({ return {
name, name,
type, type,
system: false, system: false,
required: false, required: false,
presentable: false,
unique: false, unique: false,
options: {}, options: {},
}); };
case POCKETBASE_TYPE.NUMBER: case POCKETBASE_TYPE.NUMBER:
return new SchemaField({ return {
name, name,
type, type,
system: false, system: false,
required: false, required: false,
presentable: false,
unique: false, unique: false,
options: { options: {
min: null, min: null,
max: null, max: null,
noDecimal: false,
}, },
}); };
case POCKETBASE_TYPE.PLAIN_TEXT: case POCKETBASE_TYPE.PLAIN_TEXT:
return new SchemaField({ return {
name, name,
type, type,
system: false, system: false,
required: false, required: false,
presentable: false,
unique: false, unique: false,
options: { options: {
min: null, min: null,
max: null, max: null,
pattern: "", pattern: "",
}, },
}); };
case POCKETBASE_TYPE.EMAIL: case POCKETBASE_TYPE.EMAIL:
return new SchemaField({ return {
name,
type,
system: false,
required: false,
unique: false,
options: {
min: null,
max: null,
},
});
case POCKETBASE_TYPE.JSON:
return new SchemaField({
name,
type,
system: false,
required: false,
unique: false,
options: {},
});
case POCKETBASE_TYPE.DATETIME:
return new SchemaField({
name,
type,
system: false,
required: false,
unique: false,
options: {
min: null,
max: null,
},
});
case POCKETBASE_TYPE.URL:
return new SchemaField({
name, name,
type, type,
system: false, system: false,
required: false, required: false,
presentable: false,
unique: false, unique: false,
options: { options: {
exceptDomains: null, exceptDomains: null,
onlyDomains: null, onlyDomains: null,
}, },
}); };
case POCKETBASE_TYPE.JSON:
return {
name,
type,
system: false,
required: false,
presentable: false,
unique: false,
options: {
maxSize: 2000000
},
};
case POCKETBASE_TYPE.DATETIME:
return {
name,
type,
system: false,
required: false,
presentable: false,
unique: false,
options: {
min: "",
max: "",
},
};
case POCKETBASE_TYPE.URL:
return {
name,
type,
system: false,
required: false,
presentable: false,
unique: false,
options: {
exceptDomains: null,
onlyDomains: null,
},
};
} }
} }
@ -182,7 +193,7 @@ export function createSchema(
data: { [key: string]: any }, data: { [key: string]: any },
stringifyId: boolean, stringifyId: boolean,
inputFormat: "csv" | "json", inputFormat: "csv" | "json",
): SchemaField[] { ) {
const schema: SchemaField[] = []; const schema: SchemaField[] = [];
// Seeks patterns in up to 1k records to avoid poor performance on large datasets // Seeks patterns in up to 1k records to avoid poor performance on large datasets