phares3757/pocketbase-import
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use batch api (#1)

Browse Source 
+ bulk record creation support for JSON and CSV
This commit is contained in:
michal-kapala
2024-11-27 17:34:56 +01:00
parent df56216609
commit b5d11cfc28
3 changed files with 40 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
csv.ts
View File

@ -55,7 +55,7 @@ async function importCsv() {
return return
} }
// sanitize the file name for collection name // sanitize the file name for collection name
const collectName = options.input.replace(".csv", ""); const collectionName = options.input.replace(".csv", "");
// connect to pocketbase // connect to pocketbase
const pb = new PocketBase(pbUrl); const pb = new PocketBase(pbUrl);
@ -70,7 +70,7 @@ async function importCsv() {
// the new collection // the new collection
const collection: Collection = { const collection: Collection = {
name: collectName, name: collectionName,
type: "base", type: "base",
system: false, system: false,
fields, fields,
@ -90,7 +90,7 @@ async function importCsv() {
await pb.collections.import([collection]); await pb.collections.import([collection]);
console.log( console.log(
`%c[Import] Collection '${collectName}' created!`, `%c[Import] Collection '${collectionName}' created!`,
"color: green", "color: green",
); );
@ -99,26 +99,25 @@ async function importCsv() {
console.log(`[Import] Importing ${rows.length} rows...`); console.log(`[Import] Importing ${rows.length} rows...`);
// number of successfully inserted rows const batch = pb.createBatch();
let insertCount = 0; for (let rowCount = 0; rowCount < rows.length; rowCount++)
batch.collection(collectionName).create(rows[rowCount])
for (insertCount; insertCount < rows.length; insertCount++) {
try { try {
await pb.collection(collectName).create(rows[insertCount], { const result = await batch.send();
"$autoCancel": false, let createdCount = 0;
}); for (const reqRes of result) {
} catch (e) { if (reqRes.status === 200)
// breaks on first error createdCount++;
console.error(e);
break;
} }
} const color = createdCount === data.length ? "green" : "orange";
const color = insertCount === data.length ? "green" : "orange";
console.log( console.log(
`%c[Import] Imported rows: ${insertCount}/${data.length}`, `%c[Import] Imported rows: ${createdCount}/${data.length}`,
`color: ${color}`, `color: ${color}`,
); );
} catch(err) {
console.error(err);
}
} }
importCsv(); importCsv();

33
json.ts
View File

@ -39,7 +39,7 @@ async function importJson() {
const data = await readJson(options.input); const data = await readJson(options.input);
// sanitize the file name for collection name // sanitize the file name for collection name
const collectName = options.input.replace(".json", ""); const collectionName = options.input.replace(".json", "");
// connect to pocketbase // connect to pocketbase
const pb = new PocketBase(pbUrl); const pb = new PocketBase(pbUrl);
@ -52,7 +52,7 @@ async function importJson() {
// the new collection // the new collection
const collection: Collection = { const collection: Collection = {
name: collectName, name: collectionName,
type: "base", type: "base",
system: false, system: false,
fields, fields,
@ -72,7 +72,7 @@ async function importJson() {
await pb.collections.import([collection], false); await pb.collections.import([collection], false);
console.log( console.log(
`%c[Import] Collection '${collectName}' created!`, `%c[Import] Collection '${collectionName}' created!`,
"color: green", "color: green",
); );
@ -81,26 +81,25 @@ async function importJson() {
console.log(`[Import] Importing ${rows.length} rows...`); console.log(`[Import] Importing ${rows.length} rows...`);
// number of successfully inserted rows const batch = pb.createBatch();
let insertCount = 0; for (let rowCount = 0; rowCount < rows.length; rowCount++)
batch.collection(collectionName).create(rows[rowCount])
for (insertCount; insertCount < rows.length; insertCount++) {
try { try {
await pb.collection(collectName).create(rows[insertCount], { const result = await batch.send();
"$autoCancel": false, let createdCount = 0;
}); for (const reqRes of result) {
} catch (e) { if (reqRes.status === 200)
// breaks on first error createdCount++;
console.error(e);
break;
} }
} const color = createdCount === data.length ? "green" : "orange";
const color = insertCount === data.length ? "green" : "orange";
console.log( console.log(
`%c[Import] Imported rows: ${insertCount}/${data.length}`, `%c[Import] Imported rows: ${createdCount}/${data.length}`,
`color: ${color}`, `color: ${color}`,
); );
} catch(err) {
console.error(err);
}
} }
importJson(); importJson();

1
utils/csv.ts
View File

@ -143,7 +143,6 @@ export function parseData(
// create a row schema for the collection // create a row schema for the collection
const rowSchema = generateRowSchema(schema); const rowSchema = generateRowSchema(schema);
console.log("RowSchema", rowSchema);
data.forEach((rawRow) => { data.forEach((rawRow) => {
rows.push(parseRow(rawRow, rowSchema)); rows.push(parseRow(rawRow, rowSchema));