Box SDK with node.js
I have an application which when scaled up seems to hang on node.js
The application takes a list of files from a csv and then checks if the folder exists and creates if required.
Then it uploads the file to the folder and sets collaboration on the file.
With 5 or even 30 files there seems to be no problems but when this is raised to 500 for the production test it reads the CSV but then seems to hang and does nothing else.
Below is the code used for this process.
Does anyone know why this is causing a problem ?
As there are no errors thrown and I cannot see what box is receiving its hard to tell if its the client (node) side or box where the issue is occuring
var stream = fs.createReadStream(inputFilePath);
csv
.fromStream(stream, {headers : true})
.on("data", function(data){
// return new Promise((resolve,reject)=>{
try {
console.log('Row: ' + i);
i = i + 1;
//res.send(data);
// Set folder values
var nFolderId = '';
// Does Folder Exist - Test on Box
foundfolder(data.EMAIL,folderId,client).then((nFolderId) => {
console.log('Found ?: ' + nFolderId);
if ( nFolderId == '0' ) {
client.folders.create(folderId, data.EMAIL).then( folder => {
console.log('New Folder :' + folder.id) ;
var stream = fs.createReadStream(data.FILEFROM);
client.files.uploadFile(folder.id, data.FILETO, stream, function(err, items) {
if (err) {
console.log("Failed Write :" + data.FILETO);
} else {
console.log("Success Writen :" + data.FILETO);
client.collaborations.createWithUserEmail(data.EMAIL, folder.id, client.collaborationRoles.VIEWER, function(err, items) {
if (err) {
console.log("Failed Collab :" + data.EMAIL);
} else {
console.log("Success Collab :" + data.EMAIL);
}
})
}
});
});
} else {
console.log( 'Existing Folder:' + nFolderId);
var stream = fs.createReadStream(data.FILEFROM);
client.files.uploadFile(nFolderId, data.FILETO, stream, function(err, items) {
if (err) {
console.log("Failed Write :" + data.FILETO);
} else {
console.log("Success Writen :" + data.FILETO);
foundcollab(data.EMAIL,folderId,client).then((nFolderId) => {
console.log('Found ?: ' + nFolderId);
if ( nFolderId !== '0' ) {
client.collaborations.delete(data.EMAIL, folder.id, client.collaborationRoles.VIEWER, function(err, items) {
if (err) {
console.log("Failed Collab :" + data.EMAIL);
} else {
console.log("Success Collab :" + data.EMAIL);
}
})
}
})
client.collaborations.createWithUserEmail(data.EMAIL, nFolderId, client.collaborationRoles.VIEWER, function(err, items) {
if (err) {
console.log("Failed Collab :" + data.EMAIL);
} else {
console.log("Success Collab :" + data.EMAIL);
}
});
}
});
}
})
// } // end promise
}
catch(err) {
console.log('err found' );
}
})
.on('end',function(){
//some final operation
});
The folder check is carried out by this code
function foundfolder(fname,folderId,client) {
return new Promise(function(resolve,reject) {
console.log(fname);
var nFolderId = '0';
var foundfolder = 'False';
client.folders.getItems(folderId, null, function(err, items) {
if (err) {
reject(err);
} else {
items.entries.forEach(function(element) {
if ( element.name == fname ) {
nFolderId = element.id;
};
}, this);
resolve(nFolderId);
}
})
})
Thanks for any pointers.
-
Hi — I'm Matt, one of the members of the Box SDK Team! Our team builds and maintains the Box Node SDK, and I can try and give you some general pointers to help you figure out what's going wrong in your code.
First, you seem to be mixing different patterns for handling async work — that actually isn't necessary with the Node SDK! All the SDK methods return Promises, so you could use those in order to simplify your logic a bit and make the overall flow easier to follow. That might help make things a bit clearer and allow you to more easily trace execution of your code to find any potential issues.
Next, I can spot a few issues that might be tripping you up — hopefully I can point you in the right direction!
- Depending on how quickly the code is reading CSV rows, you may be running into rate limits from the API — the code is potentially making 4 API calls per row in the CSV, which means a total of 2,000 API calls. The Box API documentation indicates that the current limits are 10 calls per second and specifically 4 uploads per second; if you're exceeding those the Box Node SDK will automatically attempt to retry after waiting for a period of time, which could cause perceptible delays.
- As written, the `foundfolder()` method will only work for folders with less than 100 items in them. If you're dealing with more items than that you may want to either increase the limit option to a higher value or ensure you're paging through the result set to get all folder items rather than just the first 100.
- The code is calling `client.collaborations.delete()` with incorrect arguments — it just takes a Collaboration ID (see https://github.com/box/box-node-sdk/blob/master/docs/collaborations.md#remove-a-collaboration for more information)
- In the Existing Folder case, the code is effectively calling `foundcollab()` and `client.collaborations.createWithUserEmail()` in parallel, when it looks like they should maybe be sequential. You may want to check to ensure that the asynchronous order of the logic is correct (which may be easier if you convert everything to using Promises throughout)
I hope that helps!
Please sign in to leave a comment.
Comments
1 comment