add batch request chunking (#1)

+ multiple batch requests of size configurable via max_batch

json.ts

@@ -16,7 +16,7 @@ async function importJson() {
 
   // parse CLI args
   const options = parse(Deno.args, {
-    string: ["input"],
+    string: ["input", "max_batch"],
     boolean: ["id"],
     default: {
       /**
@@ -27,6 +27,10 @@ async function importJson() {
        * Flag to always set `_id` column type to Plain text (detected by default).
        */
       id: false,
+      /**
+       * Default max batch request size (configurable in PB dashboard).
+       */
+      max_batch: "50"
     },
   });
 
@@ -35,6 +39,14 @@ async function importJson() {
     Deno.exit(-1);
   }
 
+  let BATCH_SIZE = 50;
+  try {
+    BATCH_SIZE = parseInt(options.max_batch)
+  } catch (err) {
+    console.error("%cOptionError: invalid 'max_batch' value, should be an integer", "color: red");
+    Deno.exit(-1);
+  }
+
   // read the file
   const data = await readJson(options.input);
 
@@ -65,7 +77,7 @@ async function importJson() {
   };
 
   // show the submitted collection
-  console.log(collection);
+  console.log("Collection", collection);
 
   // create the new collection
   // import will fail if a collection with the same name exists
@@ -81,25 +93,46 @@ async function importJson() {
 
   console.log(`[Import] Importing ${rows.length} rows...`);
 
-  const batch = pb.createBatch();
-  for (let rowCount = 0; rowCount < rows.length; rowCount++)
-    batch.collection(collectionName).create(rows[rowCount])
-  
-  try {
-    const result = await batch.send();
-    let createdCount = 0;
-    for (const reqRes of result) {
-      if (reqRes.status === 200)
-        createdCount++;
+  const chunks = Math.floor(rows.length / BATCH_SIZE);
+  const batches = chunks * BATCH_SIZE < rows.length ? chunks + 1 : chunks;
+  let createdCount = 0;
+  let chunk = 0;
+  while (chunk < batches) {
+    // create new request
+    console.log(`[Import] Batch request #${chunk+1}`);
+    const batch = pb.createBatch();
+    let chunkSize = chunk === batches - 1 ? rows.length % BATCH_SIZE : BATCH_SIZE;
+    if (chunkSize === 0)
+      chunkSize = BATCH_SIZE;
+    for (let rowCount = 0; rowCount < chunkSize; rowCount++)
+      batch.collection(collectionName).create(rows[chunk * BATCH_SIZE + rowCount])
+    // send the chunk
+    try {
+      const result = await batch.send();
+      // TODO: this should become a debug-level log
+      //console.log("Array<BatchRequestResult>", result);
+      let chunkCreatedCount = 0;
+      for (const reqRes of result) {
+        if (reqRes.status === 200)
+          chunkCreatedCount++;
+      }
+      const color = chunkCreatedCount === chunkSize ? "green" : "orange";
+      console.log(
+        `%c[Import] Batch request #${chunk+1} - imported rows: ${chunkCreatedCount}/${chunkSize}`,
+        `color: ${color}`,
+      );
+      createdCount += chunkCreatedCount;
+    } catch(err) {
+      console.error(err);
     }
-    const color = createdCount === data.length ? "green" : "orange";
-    console.log(
-      `%c[Import] Imported rows: ${createdCount}/${data.length}`,
-      `color: ${color}`,
-    );
-  } catch(err) {
-    console.error(err);
+    chunk++;
   }
+
+  const color = createdCount === data.length ? "green" : "orange";
+  console.log(
+    `%c[Import] Imported rows: ${createdCount}/${data.length}`,
+    `color: ${color}`,
+  );
 }
 
 importJson();