Node.js MongoDB Driver API

Source: lib/bulk/unordered.js

  1. "use strict";
  3. var common = require('./common')
  4. 	, utils = require('../utils')
  5.   , toError = require('../utils').toError
  6. 	, handleCallback = require('../utils').handleCallback
  7.   , shallowClone = utils.shallowClone
  8.   , BulkWriteResult = common.BulkWriteResult
  9.   , ObjectID = require('mongodb-core').BSON.ObjectID
  10. 	, BSON = require('mongodb-core').BSON
  11.   , Define = require('../metadata')
  12.   , Batch = common.Batch
  13.   , mergeBatchResults = common.mergeBatchResults;
  15. var bson = new BSON([BSON.Binary, BSON.Code, BSON.DBRef, BSON.Decimal128,
  16. 	BSON.Double, BSON.Int32, BSON.Long, BSON.Map, BSON.MaxKey, BSON.MinKey,
  17. 	BSON.ObjectId, BSON.BSONRegExp, BSON.Symbol, BSON.Timestamp]);
  19. /**
  20.  * Create a FindOperatorsUnordered instance (INTERNAL TYPE, do not instantiate directly)
  21.  * @class
  22.  * @property {number} length Get the number of operations in the bulk.
  23.  * @return {FindOperatorsUnordered} a FindOperatorsUnordered instance.
  24.  */
  25. var FindOperatorsUnordered = function(self) {
  26.   this.s = self.s;
  27. }
  29. /**
  30.  * Add a single update document to the bulk operation
  31.  *
  32.  * @method
  33.  * @param {object} doc update operations
  34.  * @throws {MongoError}
  35.  * @return {UnorderedBulkOperation}
  36.  */
  37. FindOperatorsUnordered.prototype.update = function(updateDocument) {
  38.   // Perform upsert
  39.   var upsert = typeof this.s.currentOp.upsert == 'boolean' ? this.s.currentOp.upsert : false;
  41.   // Establish the update command
  42.   var document = {
  43.       q: this.s.currentOp.selector
  44.     , u: updateDocument
  45.     , multi: true
  46.     , upsert: upsert
  47.   }
  49.   // Clear out current Op
  50.   this.s.currentOp = null;
  51.   // Add the update document to the list
  52.   return addToOperationsList(this, common.UPDATE, document);
  53. }
  55. /**
  56.  * Add a single update one document to the bulk operation
  57.  *
  58.  * @method
  59.  * @param {object} doc update operations
  60.  * @throws {MongoError}
  61.  * @return {UnorderedBulkOperation}
  62.  */
  63. FindOperatorsUnordered.prototype.updateOne = function(updateDocument) {
  64.   // Perform upsert
  65.   var upsert = typeof this.s.currentOp.upsert == 'boolean' ? this.s.currentOp.upsert : false;
  67.   // Establish the update command
  68.   var document = {
  69.       q: this.s.currentOp.selector
  70.     , u: updateDocument
  71.     , multi: false
  72.     , upsert: upsert
  73.   }
  75.   // Clear out current Op
  76.   this.s.currentOp = null;
  77.   // Add the update document to the list
  78.   return addToOperationsList(this, common.UPDATE, document);
  79. }
  81. /**
  82.  * Add a replace one operation to the bulk operation
  83.  *
  84.  * @method
  85.  * @param {object} doc the new document to replace the existing one with
  86.  * @throws {MongoError}
  87.  * @return {UnorderedBulkOperation}
  88.  */
  89. FindOperatorsUnordered.prototype.replaceOne = function(updateDocument) {
  90.   this.updateOne(updateDocument);
  91. }
  93. /**
  94.  * Upsert modifier for update bulk operation
  95.  *
  96.  * @method
  97.  * @throws {MongoError}
  98.  * @return {FindOperatorsUnordered}
  99.  */
  100. FindOperatorsUnordered.prototype.upsert = function() {
  101.   this.s.currentOp.upsert = true;
  102.   return this;
  103. }
  105. /**
  106.  * Add a remove one operation to the bulk operation
  107.  *
  108.  * @method
  109.  * @throws {MongoError}
  110.  * @return {UnorderedBulkOperation}
  111.  */
  112. FindOperatorsUnordered.prototype.removeOne = function() {
  113.   // Establish the update command
  114.   var document = {
  115.       q: this.s.currentOp.selector
  116.     , limit: 1
  117.   }
  119.   // Clear out current Op
  120.   this.s.currentOp = null;
  121.   // Add the remove document to the list
  122.   return addToOperationsList(this, common.REMOVE, document);
  123. }
  125. /**
  126.  * Add a remove operation to the bulk operation
  127.  *
  128.  * @method
  129.  * @throws {MongoError}
  130.  * @return {UnorderedBulkOperation}
  131.  */
  132. FindOperatorsUnordered.prototype.remove = function() {
  133.   // Establish the update command
  134.   var document = {
  135.       q: this.s.currentOp.selector
  136.     , limit: 0
  137.   }
  139.   // Clear out current Op
  140.   this.s.currentOp = null;
  141.   // Add the remove document to the list
  142.   return addToOperationsList(this, common.REMOVE, document);
  143. }
  145. //
  146. // Add to the operations list
  147. //
  148. var addToOperationsList = function(_self, docType, document) {
  149.   // Get the bsonSize
  150.   var bsonSize = bson.calculateObjectSize(document, {
  151. 		checkKeys: false,
  152. 	});
  153.   // Throw error if the doc is bigger than the max BSON size
  154.   if(bsonSize >= _self.s.maxBatchSizeBytes) throw toError("document is larger than the maximum size " + _self.s.maxBatchSizeBytes);
  155.   // Holds the current batch
  156.   _self.s.currentBatch = null;
  157.   // Get the right type of batch
  158.   if(docType == common.INSERT) {
  159.     _self.s.currentBatch = _self.s.currentInsertBatch;
  160.   } else if(docType == common.UPDATE) {
  161.     _self.s.currentBatch = _self.s.currentUpdateBatch;
  162.   } else if(docType == common.REMOVE) {
  163.     _self.s.currentBatch = _self.s.currentRemoveBatch;
  164.   }
  166.   // Create a new batch object if we don't have a current one
  167.   if(_self.s.currentBatch == null) _self.s.currentBatch = new Batch(docType, _self.s.currentIndex);
  169.   // Check if we need to create a new batch
  170.   if(((_self.s.currentBatch.size + 1) >= _self.s.maxWriteBatchSize)
  171.     || ((_self.s.currentBatch.sizeBytes + bsonSize) >= _self.s.maxBatchSizeBytes)
  172.     || (_self.s.currentBatch.batchType != docType)) {
  173.     // Save the batch to the execution stack
  174.     _self.s.batches.push(_self.s.currentBatch);
  176.     // Create a new batch
  177.     _self.s.currentBatch = new Batch(docType, _self.s.currentIndex);
  178.   }
  180.   // We have an array of documents
  181.   if(Array.isArray(document)) {
  182.     throw toError("operation passed in cannot be an Array");
  183.   } else {
  184.     _self.s.currentBatch.operations.push(document);
  185.     _self.s.currentBatch.originalIndexes.push(_self.s.currentIndex);
  186.     _self.s.currentIndex = _self.s.currentIndex + 1;
  187.   }
  189.   // Save back the current Batch to the right type
  190.   if(docType == common.INSERT) {
  191.     _self.s.currentInsertBatch = _self.s.currentBatch;
  192.     _self.s.bulkResult.insertedIds.push({index: _self.s.bulkResult.insertedIds.length, _id: document._id});
  193.   } else if(docType == common.UPDATE) {
  194.     _self.s.currentUpdateBatch = _self.s.currentBatch;
  195.   } else if(docType == common.REMOVE) {
  196.     _self.s.currentRemoveBatch = _self.s.currentBatch;
  197.   }
  199.   // Update current batch size
  200.   _self.s.currentBatch.size = _self.s.currentBatch.size + 1;
  201.   _self.s.currentBatch.sizeBytes = _self.s.currentBatch.sizeBytes + bsonSize;
  203.   // Return self
  204.   return _self;
  205. }
  207. /**
  208.  * Create a new UnorderedBulkOperation instance (INTERNAL TYPE, do not instantiate directly)
  209.  * @class
  210.  * @property {number} length Get the number of operations in the bulk.
  211.  * @return {UnorderedBulkOperation} a UnorderedBulkOperation instance.
  212.  */
  213. var UnorderedBulkOperation = function(topology, collection, options) {
  214. 	options = options == null ? {} : options;
  216. 	// Get the namespace for the write operations
  217.   var namespace = collection.collectionName;
  218.   // Used to mark operation as executed
  219.   var executed = false;
  221. 	// Current item
  222.   // var currentBatch = null;
  223. 	var currentOp = null;
  225. 	// Handle to the bson serializer, used to calculate running sizes
  226. 	var bson = topology.bson;
  228.   // Set max byte size
  229.   var maxBatchSizeBytes = topology.isMasterDoc && topology.isMasterDoc.maxBsonObjectSize
  230.     ? topology.isMasterDoc.maxBsonObjectSize : (1024*1025*16);
  231.   var maxWriteBatchSize = topology.isMasterDoc && topology.isMasterDoc.maxWriteBatchSize
  232.     ? topology.isMasterDoc.maxWriteBatchSize : 1000;
  234.   // Get the write concern
  235.   var writeConcern = common.writeConcern(shallowClone(options), collection, options);
  237.   // Get the promiseLibrary
  238.   var promiseLibrary = options.promiseLibrary;
  240.   // No promise library selected fall back
  241.   if(!promiseLibrary) {
  242.     promiseLibrary = typeof global.Promise == 'function' ?
  243.       global.Promise : require('es6-promise').Promise;
  244.   }
  246.   // Final results
  247.   var bulkResult = {
  248.       ok: 1
  249.     , writeErrors: []
  250.     , writeConcernErrors: []
  251.     , insertedIds: []
  252.     , nInserted: 0
  253.     , nUpserted: 0
  254.     , nMatched: 0
  255.     , nModified: 0
  256.     , nRemoved: 0
  257.     , upserted: []
  258.   };
  260.   // Internal state
  261.   this.s = {
  262.     // Final result
  263.       bulkResult: bulkResult
  264.     // Current batch state
  265.     , currentInsertBatch: null
  266.     , currentUpdateBatch: null
  267.     , currentRemoveBatch: null
  268.     , currentBatch: null
  269.     , currentIndex: 0
  270.     , batches: []
  271.     // Write concern
  272.     , writeConcern: writeConcern
  273.     // Max batch size options
  274.     , maxBatchSizeBytes: maxBatchSizeBytes
  275.     , maxWriteBatchSize: maxWriteBatchSize
  276.     // Namespace
  277.     , namespace: namespace
  278.     // BSON
  279.     , bson: bson
  280.     // Topology
  281.     , topology: topology
  282.     // Options
  283.     , options: options
  284.     // Current operation
  285.     , currentOp: currentOp
  286.     // Executed
  287.     , executed: executed
  288.     // Collection
  289.     , collection: collection
  290.     // Promise Library
  291.     , promiseLibrary: promiseLibrary
  292.     // Bypass validation
  293.     , bypassDocumentValidation: typeof options.bypassDocumentValidation == 'boolean' ? options.bypassDocumentValidation : false
  294.   }
  295. }
  297. var define = UnorderedBulkOperation.define = new Define('UnorderedBulkOperation', UnorderedBulkOperation, false);
  299. /**
  300.  * Add a single insert document to the bulk operation
  301.  *
  302.  * @param {object} doc the document to insert
  303.  * @throws {MongoError}
  304.  * @return {UnorderedBulkOperation}
  305.  */
  306. UnorderedBulkOperation.prototype.insert = function(document) {
  307.   if(this.s.collection.s.db.options.forceServerObjectId !== true && document._id == null) document._id = new ObjectID();
  308.   return addToOperationsList(this, common.INSERT, document);
  309. }
  311. /**
  312.  * Initiate a find operation for an update/updateOne/remove/removeOne/replaceOne
  313.  *
  314.  * @method
  315.  * @param {object} selector The selector for the bulk operation.
  316.  * @throws {MongoError}
  317.  * @return {FindOperatorsUnordered}
  318.  */
  319. UnorderedBulkOperation.prototype.find = function(selector) {
  320.   if (!selector) {
  321.     throw toError("Bulk find operation must specify a selector");
  322.   }
  324.   // Save a current selector
  325.   this.s.currentOp = {
  326.     selector: selector
  327.   }
  329.   return new FindOperatorsUnordered(this);
  330. }
  332. Object.defineProperty(UnorderedBulkOperation.prototype, 'length', {
  333.   enumerable: true,
  334.   get: function() {
  335.     return this.s.currentIndex;
  336.   }
  337. });
  339. UnorderedBulkOperation.prototype.raw = function(op) {
  340.   var key = Object.keys(op)[0];
  342.   // Set up the force server object id
  343.   var forceServerObjectId = typeof this.s.options.forceServerObjectId == 'boolean'
  344.     ? this.s.options.forceServerObjectId : this.s.collection.s.db.options.forceServerObjectId;
  346.   // Update operations
  347.   if((op.updateOne && op.updateOne.q)
  348.     || (op.updateMany && op.updateMany.q)
  349.     || (op.replaceOne && op.replaceOne.q)) {
  350.     op[key].multi = op.updateOne || op.replaceOne ? false : true;
  351.     return addToOperationsList(this, common.UPDATE, op[key]);
  352.   }
  354.   // Crud spec update format
  355.   if(op.updateOne || op.updateMany || op.replaceOne) {
  356.     var multi = op.updateOne || op.replaceOne ? false : true;
  357.     var operation = {q: op[key].filter, u: op[key].update || op[key].replacement, multi: multi}
  358.     if(op[key].upsert) operation.upsert = true;
  359.     return addToOperationsList(this, common.UPDATE, operation);
  360.   }
  362.   // Remove operations
  363.   if(op.removeOne || op.removeMany || (op.deleteOne && op.deleteOne.q) || op.deleteMany && op.deleteMany.q) {
  364.     op[key].limit = op.removeOne ? 1 : 0;
  365.     return addToOperationsList(this, common.REMOVE, op[key]);
  366.   }
  368.   // Crud spec delete operations, less efficient
  369.   if(op.deleteOne || op.deleteMany) {
  370.     var limit = op.deleteOne ? 1 : 0;
  371.     operation = {q: op[key].filter, limit: limit}
  372.     return addToOperationsList(this, common.REMOVE, operation);
  373.   }
  375.   // Insert operations
  376.   if(op.insertOne && op.insertOne.document == null) {
  377.     if(forceServerObjectId !== true && op.insertOne._id == null) op.insertOne._id = new ObjectID();
  378.     return addToOperationsList(this, common.INSERT, op.insertOne);
  379.   } else if(op.insertOne && op.insertOne.document) {
  380.     if(forceServerObjectId !== true && op.insertOne.document._id == null) op.insertOne.document._id = new ObjectID();
  381.     return addToOperationsList(this, common.INSERT, op.insertOne.document);
  382.   }
  384.   if(op.insertMany) {
  385.     for(var i = 0; i < op.insertMany.length; i++) {
  386.       if(forceServerObjectId !== true && op.insertMany[i]._id == null) op.insertMany[i]._id = new ObjectID();
  387.       addToOperationsList(this, common.INSERT, op.insertMany[i]);
  388.     }
  390.     return;
  391.   }
  393.   // No valid type of operation
  394.   throw toError("bulkWrite only supports insertOne, insertMany, updateOne, updateMany, removeOne, removeMany, deleteOne, deleteMany");
  395. }
  397. //
  398. // Execute the command
  399. var executeBatch = function(self, batch, callback) {
  400.   var finalOptions = {ordered: false}
  401.   if(self.s.writeConcern != null) {
  402.     finalOptions.writeConcern = self.s.writeConcern;
  403.   }
  405.   var resultHandler = function(err, result) {
  406. 		// Error is a driver related error not a bulk op error, terminate
  407. 		if(err && err.driver || err && err.message) {
  408. 			return handleCallback(callback, err);
  409. 		}
  411.     // If we have and error
  412.     if(err) err.ok = 0;
  413.     handleCallback(callback, null, mergeBatchResults(false, batch, self.s.bulkResult, err, result));
  414.   }
  416. 	// Set an operationIf if provided
  417. 	if(self.operationId) {
  418. 		resultHandler.operationId = self.operationId;
  419. 	}
  421. 	// Serialize functions
  422. 	if(self.s.options.serializeFunctions) {
  423. 		finalOptions.serializeFunctions = true
  424. 	}
  426.   // Is the bypassDocumentValidation options specific
  427.   if(self.s.bypassDocumentValidation == true) {
  428.     finalOptions.bypassDocumentValidation = true;
  429.   }
  431.   try {
  432.     if(batch.batchType == common.INSERT) {
  433.       self.s.topology.insert(self.s.collection.namespace, batch.operations, finalOptions, resultHandler);
  434.     } else if(batch.batchType == common.UPDATE) {
  435.       self.s.topology.update(self.s.collection.namespace, batch.operations, finalOptions, resultHandler);
  436.     } else if(batch.batchType == common.REMOVE) {
  437.       self.s.topology.remove(self.s.collection.namespace, batch.operations, finalOptions, resultHandler);
  438.     }
  439.   } catch(err) {
  440.     // Force top level error
  441.     err.ok = 0;
  442.     // Merge top level error and return
  443.     handleCallback(callback, null, mergeBatchResults(false, batch, self.s.bulkResult, err, null));
  444.   }
  445. }
  447. //
  448. // Execute all the commands
  449. var executeBatches = function(self, callback) {
  450.   var numberOfCommandsToExecute = self.s.batches.length;
  451.   // Execute over all the batches
  452.   for(var i = 0; i < self.s.batches.length; i++) {
  453.     executeBatch(self, self.s.batches[i], function(err) {
  454. 			// Driver layer error capture it
  455. 			if(err) error = err;
  456. 			// Count down the number of commands left to execute
  457.       numberOfCommandsToExecute = numberOfCommandsToExecute - 1;
  459.       // Execute
  460.       if(numberOfCommandsToExecute == 0) {
  461. 				// Driver level error
  462. 				if(error) return handleCallback(callback, error);
  463. 				// Treat write errors
  464.         var error = self.s.bulkResult.writeErrors.length > 0 ? toError(self.s.bulkResult.writeErrors[0]) : null;
  465.         handleCallback(callback, error, new BulkWriteResult(self.s.bulkResult));
  466.       }
  467.     });
  468.   }
  469. }
  471. /**
  472.  * The callback format for results
  473.  * @callback UnorderedBulkOperation~resultCallback
  474.  * @param {MongoError} error An error instance representing the error during the execution.
  475.  * @param {BulkWriteResult} result The bulk write result.
  476.  */
  478. /**
  479.  * Execute the ordered bulk operation
  480.  *
  481.  * @method
  482.  * @param {object} [options=null] Optional settings.
  483.  * @param {(number|string)} [options.w=null] The write concern.
  484.  * @param {number} [options.wtimeout=null] The write concern timeout.
  485.  * @param {boolean} [options.j=false] Specify a journal write concern.
  486.  * @param {boolean} [options.fsync=false] Specify a file sync write concern.
  487.  * @param {UnorderedBulkOperation~resultCallback} [callback] The result callback
  488.  * @throws {MongoError}
  489.  * @return {Promise} returns Promise if no callback passed
  490.  */
  491. UnorderedBulkOperation.prototype.execute = function(_writeConcern, callback) {
  492.   var self = this;
  493.   if (this.s.executed) {
  494.     var executedError = toError('batch cannot be re-executed');
  495.     return (typeof callback === 'function') ?
  496.       callback(executedError, null) : this.s.promiseLibrary.reject(executedError);
  497.   }
  499.   if (typeof _writeConcern === 'function') {
  500.     callback = _writeConcern;
  501.   } else if (_writeConcern && typeof _writeConcern === 'object') {
  502.     this.s.writeConcern = _writeConcern;
  503.   }
  505.   // If we have current batch
  506.   if (this.s.currentInsertBatch) this.s.batches.push(this.s.currentInsertBatch);
  507.   if (this.s.currentUpdateBatch) this.s.batches.push(this.s.currentUpdateBatch);
  508.   if (this.s.currentRemoveBatch) this.s.batches.push(this.s.currentRemoveBatch);
  510.   // If we have no operations in the bulk raise an error
  511.   if (this.s.batches.length == 0) {
  512.     var emptyBatchError = toError('Invalid Operation, no operations specified');
  513.     return (typeof callback === 'function') ?
  514.       callback(emptyBatchError, null) : this.s.promiseLibrary.reject(emptyBatchError);
  515.   }
  517.   // Execute using callback
  518.   if (typeof callback === 'function') return executeBatches(this, callback);
  520.   // Return a Promise
  521.   return new this.s.promiseLibrary(function(resolve, reject) {
  522.     executeBatches(self, function(err, r) {
  523.       if (err) return reject(err);
  524.       resolve(r);
  525.     });
  526.   });
  527. }
  529. define.classMethod('execute', {callback: true, promise:false});
  531. /**
  532.  * Returns an unordered batch object
  533.  * @ignore
  534.  */
  535. var initializeUnorderedBulkOp = function(topology, collection, options) {
  536. 	return new UnorderedBulkOperation(topology, collection, options);
  537. }
  539. initializeUnorderedBulkOp.UnorderedBulkOperation = UnorderedBulkOperation;
  540. module.exports = initializeUnorderedBulkOp;
  541. module.exports.Bulk = UnorderedBulkOperation;