From 7fc1849729602f8c2333aa9eef6b61a09f12d3d3 Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 17:13:56 -0700 Subject: [PATCH 01/11] Move Parse Server logic into Parse Server and out of MongoAdapter --- src/Adapters/Storage/Mongo/MongoTransform.js | 64 ------------------ src/RestQuery.js | 68 ++++++++++++++++++-- 2 files changed, 64 insertions(+), 68 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index 33aa666b09..ca04840314 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -868,66 +868,6 @@ function untransformObject(schema, className, mongoObject, isNestedObject = fals } } -function transformSelect(selectObject, key ,objects) { - var values = []; - for (var result of objects) { - values.push(result[key]); - } - delete selectObject['$select']; - if (Array.isArray(selectObject['$in'])) { - selectObject['$in'] = selectObject['$in'].concat(values); - } else { - selectObject['$in'] = values; - } -} - -function transformDontSelect(dontSelectObject, key, objects) { - var values = []; - for (var result of objects) { - values.push(result[key]); - } - delete dontSelectObject['$dontSelect']; - if (Array.isArray(dontSelectObject['$nin'])) { - dontSelectObject['$nin'] = dontSelectObject['$nin'].concat(values); - } else { - dontSelectObject['$nin'] = values; - } -} - -function transformInQuery(inQueryObject, className, results) { - var values = []; - for (var result of results) { - values.push({ - __type: 'Pointer', - className: className, - objectId: result.objectId - }); - } - delete inQueryObject['$inQuery']; - if (Array.isArray(inQueryObject['$in'])) { - inQueryObject['$in'] = inQueryObject['$in'].concat(values); - } else { - inQueryObject['$in'] = values; - } -} - -function transformNotInQuery(notInQueryObject, className, results) { - var values = []; - for (var result of results) { - values.push({ - __type: 'Pointer', - className: className, - objectId: result.objectId - }); - } - delete notInQueryObject['$notInQuery']; - if (Array.isArray(notInQueryObject['$nin'])) { - notInQueryObject['$nin'] = notInQueryObject['$nin'].concat(values); - } else { - notInQueryObject['$nin'] = values; - } -} - var DateCoder = { JSONToDatabase(json) { return new Date(json.iso); @@ -1021,9 +961,5 @@ module.exports = { parseObjectToMongoObjectForCreate, transformUpdate, transformWhere, - transformSelect, - transformDontSelect, - transformInQuery, - transformNotInQuery, untransformObject }; diff --git a/src/RestQuery.js b/src/RestQuery.js index 34324f5e0e..a1d48fa7e8 100644 --- a/src/RestQuery.js +++ b/src/RestQuery.js @@ -188,6 +188,23 @@ RestQuery.prototype.validateClientClassCreation = function() { } }; +function transformInQuery(inQueryObject, className, results) { + var values = []; + for (var result of results) { + values.push({ + __type: 'Pointer', + className: className, + objectId: result.objectId + }); + } + delete inQueryObject['$inQuery']; + if (Array.isArray(inQueryObject['$in'])) { + inQueryObject['$in'] = inQueryObject['$in'].concat(values); + } else { + inQueryObject['$in'] = values; + } +} + // Replaces a $inQuery clause by running the subquery, if there is an // $inQuery clause. // The $inQuery clause turns into an $in with values that are just @@ -213,12 +230,29 @@ RestQuery.prototype.replaceInQuery = function() { this.config, this.auth, inQueryValue.className, inQueryValue.where, additionalOptions); return subquery.execute().then((response) => { - this.config.database.transform.transformInQuery(inQueryObject, subquery.className, response.results); + transformInQuery(inQueryObject, subquery.className, response.results); // Recurse to repeat return this.replaceInQuery(); }); }; +function transformNotInQuery(notInQueryObject, className, results) { + var values = []; + for (var result of results) { + values.push({ + __type: 'Pointer', + className: className, + objectId: result.objectId + }); + } + delete notInQueryObject['$notInQuery']; + if (Array.isArray(notInQueryObject['$nin'])) { + notInQueryObject['$nin'] = notInQueryObject['$nin'].concat(values); + } else { + notInQueryObject['$nin'] = values; + } +} + // Replaces a $notInQuery clause by running the subquery, if there is an // $notInQuery clause. // The $notInQuery clause turns into a $nin with values that are just @@ -244,12 +278,25 @@ RestQuery.prototype.replaceNotInQuery = function() { this.config, this.auth, notInQueryValue.className, notInQueryValue.where, additionalOptions); return subquery.execute().then((response) => { - this.config.database.transform.transformNotInQuery(notInQueryObject, subquery.className, response.results); + transformNotInQuery(notInQueryObject, subquery.className, response.results); // Recurse to repeat return this.replaceNotInQuery(); }); }; +const transformSelect = (selectObject, key ,objects) => { + var values = []; + for (var result of objects) { + values.push(result[key]); + } + delete selectObject['$select']; + if (Array.isArray(selectObject['$in'])) { + selectObject['$in'] = selectObject['$in'].concat(values); + } else { + selectObject['$in'] = values; + } +} + // Replaces a $select clause by running the subquery, if there is a // $select clause. // The $select clause turns into an $in with values selected out of @@ -281,12 +328,25 @@ RestQuery.prototype.replaceSelect = function() { this.config, this.auth, selectValue.query.className, selectValue.query.where, additionalOptions); return subquery.execute().then((response) => { - this.config.database.transform.transformSelect(selectObject, selectValue.key, response.results); + transformSelect(selectObject, selectValue.key, response.results); // Keep replacing $select clauses return this.replaceSelect(); }) }; +const transformDontSelect = (dontSelectObject, key, objects) => { + var values = []; + for (var result of objects) { + values.push(result[key]); + } + delete dontSelectObject['$dontSelect']; + if (Array.isArray(dontSelectObject['$nin'])) { + dontSelectObject['$nin'] = dontSelectObject['$nin'].concat(values); + } else { + dontSelectObject['$nin'] = values; + } +} + // Replaces a $dontSelect clause by running the subquery, if there is a // $dontSelect clause. // The $dontSelect clause turns into an $nin with values selected out of @@ -316,7 +376,7 @@ RestQuery.prototype.replaceDontSelect = function() { this.config, this.auth, dontSelectValue.query.className, dontSelectValue.query.where, additionalOptions); return subquery.execute().then((response) => { - this.config.database.transform.transformDontSelect(dontSelectObject, dontSelectValue.key, response.results); + transformDontSelect(dontSelectObject, dontSelectValue.key, response.results); // Keep replacing $dontSelect clauses return this.replaceDontSelect(); }) From 9a04a9372b57282e6f04aa0aaa46d23dbae60622 Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 17:36:40 -0700 Subject: [PATCH 02/11] Move untransforming up one level --- src/Controllers/DatabaseController.js | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 522a2f1443..6ca2b44811 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -146,12 +146,8 @@ DatabaseController.prototype.validateObject = function(className, object, query, }); }; -// Like transform.untransformObject but you need to provide a className. // Filters out any data that shouldn't be on this REST-formatted object. -DatabaseController.prototype.untransformObject = function( - schema, isMaster, aclGroup, className, mongoObject) { - var object = this.transform.untransformObject(schema, className, mongoObject); - +const filterSensitiveData = (isMaster, aclGroup, className, object) => { if (className !== '_User') { return object; } @@ -706,11 +702,10 @@ DatabaseController.prototype.find = function(className, query, { return collection.count(mongoWhere, mongoOptions); } else { return collection.find(mongoWhere, mongoOptions) - .then(mongoResults => { - return mongoResults.map(result => { - return this.untransformObject(schemaController, isMaster, aclGroup, className, result); - }); - }); + .then(mongoResults => mongoResults.map(mongoObject => { + var parseObject = this.transform.untransformObject(schemaController, className, mongoObject); + return filterSensitiveData(isMaster, aclGroup, className, parseObject); + })); } }); }); From 614f45a7cda3f9d05e63ebff92332a73b5510138 Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 19:51:06 -0700 Subject: [PATCH 03/11] Make find() in MongoStorageAdapter --- spec/MongoTransform.spec.js | 20 +++++------ src/Adapters/Storage/Mongo/MongoCollection.js | 36 ++++++++++--------- .../Storage/Mongo/MongoStorageAdapter.js | 8 +++++ src/Adapters/Storage/Mongo/MongoTransform.js | 12 +++---- src/Controllers/DatabaseController.js | 7 ++-- 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/spec/MongoTransform.spec.js b/spec/MongoTransform.spec.js index 905b7647c1..ca142134f0 100644 --- a/spec/MongoTransform.spec.js +++ b/spec/MongoTransform.spec.js @@ -121,10 +121,10 @@ describe('transformWhere', () => { }); }); -describe('untransformObject', () => { +describe('mongoObjectToParseObject', () => { it('built-in timestamps', (done) => { var input = {createdAt: new Date(), updatedAt: new Date()}; - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(typeof output.createdAt).toEqual('string'); expect(typeof output.updatedAt).toEqual('string'); done(); @@ -132,7 +132,7 @@ describe('untransformObject', () => { it('pointer', (done) => { var input = {_p_userPointer: '_User$123'}; - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(typeof output.userPointer).toEqual('object'); expect(output.userPointer).toEqual( {__type: 'Pointer', className: '_User', objectId: '123'} @@ -142,14 +142,14 @@ describe('untransformObject', () => { it('null pointer', (done) => { var input = {_p_userPointer: null}; - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(output.userPointer).toBeUndefined(); done(); }); it('file', (done) => { var input = {picture: 'pic.jpg'}; - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(typeof output.picture).toEqual('object'); expect(output.picture).toEqual({__type: 'File', name: 'pic.jpg'}); done(); @@ -157,7 +157,7 @@ describe('untransformObject', () => { it('geopoint', (done) => { var input = {location: [180, -180]}; - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(typeof output.location).toEqual('object'); expect(output.location).toEqual( {__type: 'GeoPoint', longitude: 180, latitude: -180} @@ -167,7 +167,7 @@ describe('untransformObject', () => { it('nested array', (done) => { var input = {arr: [{_testKey: 'testValue' }]}; - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(Array.isArray(output.arr)).toEqual(true); expect(output.arr).toEqual([{ _testKey: 'testValue'}]); done(); @@ -185,7 +185,7 @@ describe('untransformObject', () => { }, regularKey: "some data", }]} - let output = transform.untransformObject(dummySchema, null, input); + let output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(dd(output, input)).toEqual(undefined); done(); }); @@ -253,7 +253,7 @@ describe('transform schema key changes', () => { _rperm: ["*"], _wperm: ["Kevin"] }; - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(typeof output.ACL).toEqual('object'); expect(output._rperm).toBeUndefined(); expect(output._wperm).toBeUndefined(); @@ -267,7 +267,7 @@ describe('transform schema key changes', () => { long: mongodb.Long.fromNumber(Number.MAX_SAFE_INTEGER), double: new mongodb.Double(Number.MAX_VALUE) } - var output = transform.untransformObject(dummySchema, null, input); + var output = transform.mongoObjectToParseObject(dummySchema, null, input); expect(output.long).toBe(Number.MAX_SAFE_INTEGER); expect(output.double).toBe(Number.MAX_VALUE); done(); diff --git a/src/Adapters/Storage/Mongo/MongoCollection.js b/src/Adapters/Storage/Mongo/MongoCollection.js index bf41582b19..e309281884 100644 --- a/src/Adapters/Storage/Mongo/MongoCollection.js +++ b/src/Adapters/Storage/Mongo/MongoCollection.js @@ -1,5 +1,6 @@ let mongodb = require('mongodb'); let Collection = mongodb.Collection; +import * as transform from './MongoTransform'; export default class MongoCollection { _mongoCollection:Collection; @@ -13,25 +14,28 @@ export default class MongoCollection { // none, then build the geoindex. // This could be improved a lot but it's not clear if that's a good // idea. Or even if this behavior is a good idea. + + // Depends on the className and schemaController because mongoObjectToParseObject does. + // TODO: break this dependency find(query, { skip, limit, sort } = {}) { return this._rawFind(query, { skip, limit, sort }) - .catch(error => { - // Check for "no geoindex" error - if (error.code != 17007 && !error.message.match(/unable to find index for .geoNear/)) { - throw error; - } - // Figure out what key needs an index - let key = error.message.match(/field=([A-Za-z_0-9]+) /)[1]; - if (!key) { - throw error; - } + .catch(error => { + // Check for "no geoindex" error + if (error.code != 17007 && !error.message.match(/unable to find index for .geoNear/)) { + throw error; + } + // Figure out what key needs an index + let key = error.message.match(/field=([A-Za-z_0-9]+) /)[1]; + if (!key) { + throw error; + } - var index = {}; - index[key] = '2d'; - return this._mongoCollection.createIndex(index) - // Retry, but just once. - .then(() => this._rawFind(query, { skip, limit, sort })); - }); + var index = {}; + index[key] = '2d'; + return this._mongoCollection.createIndex(index) + // Retry, but just once. + .then(() => this._rawFind(query, { skip, limit, sort })); + }) } _rawFind(query, { skip, limit, sort } = {}) { diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index e194dfe00f..b4e1f84eb1 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -197,6 +197,14 @@ export class MongoStorageAdapter { }); } + // Executes a find. Accepts: className, query in Parse format, and { skip, limit, sort }. + // Accepts the schemaController for legacy reasons. + find(className, query, { skip, limit, sort }, schemaController) { + return this.adaptiveCollection(className) + .then(collection => collection.find(query, { skip, limit, sort })) + .then(objects => objects.map(object => transform.mongoObjectToParseObject(schemaController, className, object))); + } + get transform() { return transform; } diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index ca04840314..6478fdeabf 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -731,7 +731,7 @@ const specialKeysForUntransform = [ // Converts from a mongo-format object to a REST-format object. // Does not strip out anything based on a lack of authentication. -function untransformObject(schema, className, mongoObject, isNestedObject = false) { +function mongoObjectToParseObject(schema, className, mongoObject, isNestedObject = false) { switch(typeof mongoObject) { case 'string': case 'number': @@ -740,14 +740,14 @@ function untransformObject(schema, className, mongoObject, isNestedObject = fals case 'undefined': case 'symbol': case 'function': - throw 'bad value in untransformObject'; + throw 'bad value in mongoObjectToParseObject'; case 'object': if (mongoObject === null) { return null; } if (mongoObject instanceof Array) { return mongoObject.map(arrayEntry => { - return untransformObject(schema, className, arrayEntry, true); + return mongoObjectToParseObject(schema, className, arrayEntry, true); }); } @@ -770,7 +770,7 @@ function untransformObject(schema, className, mongoObject, isNestedObject = fals var restObject = untransformACL(mongoObject); for (var key in mongoObject) { if (isNestedObject && _.includes(specialKeysForUntransform, key)) { - restObject[key] = untransformObject(schema, className, mongoObject[key], true); + restObject[key] = mongoObjectToParseObject(schema, className, mongoObject[key], true); continue; } switch(key) { @@ -854,7 +854,7 @@ function untransformObject(schema, className, mongoObject, isNestedObject = fals break; } } - restObject[key] = untransformObject(schema, className, mongoObject[key], true); + restObject[key] = mongoObjectToParseObject(schema, className, mongoObject[key], true); } } @@ -961,5 +961,5 @@ module.exports = { parseObjectToMongoObjectForCreate, transformUpdate, transformWhere, - untransformObject + mongoObjectToParseObject, }; diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 6ca2b44811..67240a9128 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -701,11 +701,8 @@ DatabaseController.prototype.find = function(className, query, { delete mongoOptions.limit; return collection.count(mongoWhere, mongoOptions); } else { - return collection.find(mongoWhere, mongoOptions) - .then(mongoResults => mongoResults.map(mongoObject => { - var parseObject = this.transform.untransformObject(schemaController, className, mongoObject); - return filterSensitiveData(isMaster, aclGroup, className, parseObject); - })); + return this.adapter.find(className, mongoWhere, mongoOptions, schemaController) + .then(objects => objects.map(object => filterSensitiveData(isMaster, aclGroup, className, object))); } }); }); From 0d275cc5fa3e7af78061b90d38196f801e317c13 Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 20:12:44 -0700 Subject: [PATCH 04/11] Put nested object untransforming into it's own function --- src/Adapters/Storage/Mongo/MongoTransform.js | 148 +++++++++++++++++-- 1 file changed, 135 insertions(+), 13 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index 6478fdeabf..3c92fef3be 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -729,9 +729,7 @@ const specialKeysForUntransform = [ '_expiresAt', ]; -// Converts from a mongo-format object to a REST-format object. -// Does not strip out anything based on a lack of authentication. -function mongoObjectToParseObject(schema, className, mongoObject, isNestedObject = false) { +const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => { switch(typeof mongoObject) { case 'string': case 'number': @@ -747,7 +745,7 @@ function mongoObjectToParseObject(schema, className, mongoObject, isNestedObject } if (mongoObject instanceof Array) { return mongoObject.map(arrayEntry => { - return mongoObjectToParseObject(schema, className, arrayEntry, true); + return nestedMongoObjectToNestedParseObject(schema, className, arrayEntry); }); } @@ -769,8 +767,8 @@ function mongoObjectToParseObject(schema, className, mongoObject, isNestedObject var restObject = untransformACL(mongoObject); for (var key in mongoObject) { - if (isNestedObject && _.includes(specialKeysForUntransform, key)) { - restObject[key] = mongoObjectToParseObject(schema, className, mongoObject[key], true); + if (_.includes(specialKeysForUntransform, key)) { + restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); continue; } switch(key) { @@ -840,8 +838,6 @@ function mongoObjectToParseObject(schema, className, mongoObject, isNestedObject objectId: objData[1] }; break; - } else if (!isNestedObject && key[0] == '_' && key != '__type') { - throw ('bad key in untransform: ' + key); } else { var expectedType = schema.getExpectedType(className, key); var value = mongoObject[key]; @@ -854,15 +850,141 @@ function mongoObjectToParseObject(schema, className, mongoObject, isNestedObject break; } } - restObject[key] = mongoObjectToParseObject(schema, className, mongoObject[key], true); + restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); } } + return restObject; + default: + throw 'unknown js type'; + } +} - if (!isNestedObject) { - let relationFields = schema.getRelationFields(className); - Object.assign(restObject, relationFields); +// Converts from a mongo-format object to a REST-format object. +// Does not strip out anything based on a lack of authentication. +const mongoObjectToParseObject = (schema, className, mongoObject) => { + switch(typeof mongoObject) { + case 'string': + case 'number': + case 'boolean': + return mongoObject; + case 'undefined': + case 'symbol': + case 'function': + throw 'bad value in mongoObjectToParseObject'; + case 'object': + if (mongoObject === null) { + return null; } - return restObject; + if (mongoObject instanceof Array) { + return mongoObject.map(arrayEntry => { + return nestedMongoObjectToNestedParseObject(schema, className, arrayEntry); + }); + } + + if (mongoObject instanceof Date) { + return Parse._encode(mongoObject); + } + + if (mongoObject instanceof mongodb.Long) { + return mongoObject.toNumber(); + } + + if (mongoObject instanceof mongodb.Double) { + return mongoObject.value; + } + + if (BytesCoder.isValidDatabaseObject(mongoObject)) { + return BytesCoder.databaseToJSON(mongoObject); + } + + var restObject = untransformACL(mongoObject); + for (var key in mongoObject) { + switch(key) { + case '_id': + restObject['objectId'] = '' + mongoObject[key]; + break; + case '_hashed_password': + restObject['password'] = mongoObject[key]; + break; + case '_acl': + case '_email_verify_token': + case '_perishable_token': + case '_tombstone': + break; + case '_session_token': + restObject['sessionToken'] = mongoObject[key]; + break; + case 'updatedAt': + case '_updated_at': + restObject['updatedAt'] = Parse._encode(new Date(mongoObject[key])).iso; + break; + case 'createdAt': + case '_created_at': + restObject['createdAt'] = Parse._encode(new Date(mongoObject[key])).iso; + break; + case 'expiresAt': + case '_expiresAt': + restObject['expiresAt'] = Parse._encode(new Date(mongoObject[key])); + break; + default: + // Check other auth data keys + var authDataMatch = key.match(/^_auth_data_([a-zA-Z0-9_]+)$/); + if (authDataMatch) { + var provider = authDataMatch[1]; + restObject['authData'] = restObject['authData'] || {}; + restObject['authData'][provider] = mongoObject[key]; + break; + } + + if (key.indexOf('_p_') == 0) { + var newKey = key.substring(3); + var expected; + if (schema && schema.getExpectedType) { + expected = schema.getExpectedType(className, newKey); + } + if (!expected) { + log.info('transform.js', + 'Found a pointer column not in the schema, dropping it.', + className, newKey); + break; + } + if (expected && expected.type !== 'Pointer') { + log.info('transform.js', 'Found a pointer in a non-pointer column, dropping it.', className, key); + break; + } + if (mongoObject[key] === null) { + break; + } + var objData = mongoObject[key].split('$'); + var newClass = (expected ? expected.targetClass : objData[0]); + if (objData[0] !== newClass) { + throw 'pointer to incorrect className'; + } + restObject[newKey] = { + __type: 'Pointer', + className: objData[0], + objectId: objData[1] + }; + break; + } else if (key[0] == '_' && key != '__type') { + throw ('bad key in untransform: ' + key); + } else { + var expectedType = schema.getExpectedType(className, key); + var value = mongoObject[key]; + if (expectedType && expectedType.type === 'File' && FileCoder.isValidDatabaseObject(value)) { + restObject[key] = FileCoder.databaseToJSON(value); + break; + } + if (expectedType && expectedType.type === 'GeoPoint' && GeoPointCoder.isValidDatabaseObject(value)) { + restObject[key] = GeoPointCoder.databaseToJSON(value); + break; + } + } + restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); + } + } + + return { ...restObject, ...schema.getRelationFields(className) }; default: throw 'unknown js type'; } From f4bc033bcba3d433176a7f9f645b6860e072092d Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 20:43:32 -0700 Subject: [PATCH 05/11] Simplfy nested untransform --- src/Adapters/Storage/Mongo/MongoTransform.js | 143 ++++++------------- 1 file changed, 46 insertions(+), 97 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index 3c92fef3be..ea97ebb7cb 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -713,22 +713,6 @@ function transformUpdateOperator({ } } -const specialKeysForUntransform = [ - '_id', - '_hashed_password', - '_acl', - '_email_verify_token', - '_perishable_token', - '_tombstone', - '_session_token', - 'updatedAt', - '_updated_at', - 'createdAt', - '_created_at', - 'expiresAt', - '_expiresAt', -]; - const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => { switch(typeof mongoObject) { case 'string': @@ -744,9 +728,7 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => return null; } if (mongoObject instanceof Array) { - return mongoObject.map(arrayEntry => { - return nestedMongoObjectToNestedParseObject(schema, className, arrayEntry); - }); + return mongoObject.map(arrayEntry => nestedMongoObjectToNestedParseObject(schema, className, arrayEntry)); } if (mongoObject instanceof Date) { @@ -767,91 +749,58 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => var restObject = untransformACL(mongoObject); for (var key in mongoObject) { - if (_.includes(specialKeysForUntransform, key)) { - restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); - continue; - } - switch(key) { - case '_id': - restObject['objectId'] = '' + mongoObject[key]; - break; - case '_hashed_password': - restObject['password'] = mongoObject[key]; - break; - case '_acl': - case '_email_verify_token': - case '_perishable_token': - case '_tombstone': - break; - case '_session_token': - restObject['sessionToken'] = mongoObject[key]; - break; - case 'updatedAt': - case '_updated_at': - restObject['updatedAt'] = Parse._encode(new Date(mongoObject[key])).iso; - break; - case 'createdAt': - case '_created_at': - restObject['createdAt'] = Parse._encode(new Date(mongoObject[key])).iso; + // Check auth data keys + var authDataMatch = key.match(/^_auth_data_([a-zA-Z0-9_]+)$/); + if (authDataMatch) { + var provider = authDataMatch[1]; + restObject['authData'] = restObject['authData'] || {}; + restObject['authData'][provider] = mongoObject[key]; break; - case 'expiresAt': - case '_expiresAt': - restObject['expiresAt'] = Parse._encode(new Date(mongoObject[key])); + } + + if (key.indexOf('_p_') == 0) { + var newKey = key.substring(3); + var expected; + if (schema && schema.getExpectedType) { + expected = schema.getExpectedType(className, newKey); + } + if (!expected) { + log.info('transform.js', + 'Found a pointer column not in the schema, dropping it.', + className, newKey); + break; + } + if (expected && expected.type !== 'Pointer') { + log.info('transform.js', 'Found a pointer in a non-pointer column, dropping it.', className, key); + break; + } + if (mongoObject[key] === null) { + break; + } + var objData = mongoObject[key].split('$'); + var newClass = (expected ? expected.targetClass : objData[0]); + if (objData[0] !== newClass) { + throw 'pointer to incorrect className'; + } + restObject[newKey] = { + __type: 'Pointer', + className: objData[0], + objectId: objData[1] + }; break; - default: - // Check other auth data keys - var authDataMatch = key.match(/^_auth_data_([a-zA-Z0-9_]+)$/); - if (authDataMatch) { - var provider = authDataMatch[1]; - restObject['authData'] = restObject['authData'] || {}; - restObject['authData'][provider] = mongoObject[key]; + } else { + var expectedType = schema.getExpectedType(className, key); + var value = mongoObject[key]; + if (expectedType && expectedType.type === 'File' && FileCoder.isValidDatabaseObject(value)) { + restObject[key] = FileCoder.databaseToJSON(value); break; } - - if (key.indexOf('_p_') == 0) { - var newKey = key.substring(3); - var expected; - if (schema && schema.getExpectedType) { - expected = schema.getExpectedType(className, newKey); - } - if (!expected) { - log.info('transform.js', - 'Found a pointer column not in the schema, dropping it.', - className, newKey); - break; - } - if (expected && expected.type !== 'Pointer') { - log.info('transform.js', 'Found a pointer in a non-pointer column, dropping it.', className, key); - break; - } - if (mongoObject[key] === null) { - break; - } - var objData = mongoObject[key].split('$'); - var newClass = (expected ? expected.targetClass : objData[0]); - if (objData[0] !== newClass) { - throw 'pointer to incorrect className'; - } - restObject[newKey] = { - __type: 'Pointer', - className: objData[0], - objectId: objData[1] - }; + if (expectedType && expectedType.type === 'GeoPoint' && GeoPointCoder.isValidDatabaseObject(value)) { + restObject[key] = GeoPointCoder.databaseToJSON(value); break; - } else { - var expectedType = schema.getExpectedType(className, key); - var value = mongoObject[key]; - if (expectedType && expectedType.type === 'File' && FileCoder.isValidDatabaseObject(value)) { - restObject[key] = FileCoder.databaseToJSON(value); - break; - } - if (expectedType && expectedType.type === 'GeoPoint' && GeoPointCoder.isValidDatabaseObject(value)) { - restObject[key] = GeoPointCoder.databaseToJSON(value); - break; - } } - restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); } + restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); } return restObject; default: From 5999a0d5449d3a9f34e26ba02b60934da01f738f Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 20:57:31 -0700 Subject: [PATCH 06/11] Don't mess with inner object keys called _auth_data_* --- spec/ParseAPI.spec.js | 9 +++++++++ src/Adapters/Storage/Mongo/MongoTransform.js | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/ParseAPI.spec.js b/spec/ParseAPI.spec.js index 21603a0956..366ab74f37 100644 --- a/spec/ParseAPI.spec.js +++ b/spec/ParseAPI.spec.js @@ -1120,4 +1120,13 @@ describe('miscellaneous', function() { done(); }) }); + + it('does not change inner object keys names _auth_data_something', done => { + new Parse.Object('O').save({ innerObj: {_auth_data_facebook: 7}}) + .then(object => new Parse.Query('O').get(object.id)) + .then(object => { + expect(object.get('innerObj')).toEqual({_auth_data_facebook: 7}); + done(); + }); + }); }); diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index ea97ebb7cb..4fc1f9195a 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -749,15 +749,6 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => var restObject = untransformACL(mongoObject); for (var key in mongoObject) { - // Check auth data keys - var authDataMatch = key.match(/^_auth_data_([a-zA-Z0-9_]+)$/); - if (authDataMatch) { - var provider = authDataMatch[1]; - restObject['authData'] = restObject['authData'] || {}; - restObject['authData'][provider] = mongoObject[key]; - break; - } - if (key.indexOf('_p_') == 0) { var newKey = key.substring(3); var expected; From 411e0f93cbcee5b723187361903707514f26f15f Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 21:20:53 -0700 Subject: [PATCH 07/11] Prevent untransforming inner object keys named _p_* --- spec/ParseAPI.spec.js | 11 ++++- src/Adapters/Storage/Mongo/MongoTransform.js | 48 ++++---------------- 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/spec/ParseAPI.spec.js b/spec/ParseAPI.spec.js index 366ab74f37..498c84dcba 100644 --- a/spec/ParseAPI.spec.js +++ b/spec/ParseAPI.spec.js @@ -1121,7 +1121,7 @@ describe('miscellaneous', function() { }) }); - it('does not change inner object keys names _auth_data_something', done => { + it('does not change inner object key names _auth_data_something', done => { new Parse.Object('O').save({ innerObj: {_auth_data_facebook: 7}}) .then(object => new Parse.Query('O').get(object.id)) .then(object => { @@ -1129,4 +1129,13 @@ describe('miscellaneous', function() { done(); }); }); + + it('does not change inner object key names _p_somethign', done => { + new Parse.Object('O').save({ innerObj: {_p_data: 7}}) + .then(object => new Parse.Query('O').get(object.id)) + .then(object => { + expect(object.get('innerObj')).toEqual({_p_data: 7}); + done(); + }); + }); }); diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index 4fc1f9195a..29e8a9c324 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -749,47 +749,15 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => var restObject = untransformACL(mongoObject); for (var key in mongoObject) { - if (key.indexOf('_p_') == 0) { - var newKey = key.substring(3); - var expected; - if (schema && schema.getExpectedType) { - expected = schema.getExpectedType(className, newKey); - } - if (!expected) { - log.info('transform.js', - 'Found a pointer column not in the schema, dropping it.', - className, newKey); - break; - } - if (expected && expected.type !== 'Pointer') { - log.info('transform.js', 'Found a pointer in a non-pointer column, dropping it.', className, key); - break; - } - if (mongoObject[key] === null) { - break; - } - var objData = mongoObject[key].split('$'); - var newClass = (expected ? expected.targetClass : objData[0]); - if (objData[0] !== newClass) { - throw 'pointer to incorrect className'; - } - restObject[newKey] = { - __type: 'Pointer', - className: objData[0], - objectId: objData[1] - }; + var expectedType = schema.getExpectedType(className, key); + var value = mongoObject[key]; + if (expectedType && expectedType.type === 'File' && FileCoder.isValidDatabaseObject(value)) { + restObject[key] = FileCoder.databaseToJSON(value); + break; + } + if (expectedType && expectedType.type === 'GeoPoint' && GeoPointCoder.isValidDatabaseObject(value)) { + restObject[key] = GeoPointCoder.databaseToJSON(value); break; - } else { - var expectedType = schema.getExpectedType(className, key); - var value = mongoObject[key]; - if (expectedType && expectedType.type === 'File' && FileCoder.isValidDatabaseObject(value)) { - restObject[key] = FileCoder.databaseToJSON(value); - break; - } - if (expectedType && expectedType.type === 'GeoPoint' && GeoPointCoder.isValidDatabaseObject(value)) { - restObject[key] = GeoPointCoder.databaseToJSON(value); - break; - } } restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); } From 8065d59815e840472744639acd56391a77589fc7 Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 21:43:21 -0700 Subject: [PATCH 08/11] Fix inner keys named _rperm, _wperm --- spec/ParseAPI.spec.js | 9 +++++++++ src/Adapters/Storage/Mongo/MongoTransform.js | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/spec/ParseAPI.spec.js b/spec/ParseAPI.spec.js index 498c84dcba..9573784c55 100644 --- a/spec/ParseAPI.spec.js +++ b/spec/ParseAPI.spec.js @@ -1138,4 +1138,13 @@ describe('miscellaneous', function() { done(); }); }); + + it('does not change inner object key names _rperm, _wperm', done => { + new Parse.Object('O').save({ innerObj: {_rperm: 7, _wperm: 8}}) + .then(object => new Parse.Query('O').get(object.id)) + .then(object => { + expect(object.get('innerObj')).toEqual({_rperm: 7, _wperm: 8}); + done(); + }); + }); }); diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index 29e8a9c324..0441a265b8 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -747,7 +747,7 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => return BytesCoder.databaseToJSON(mongoObject); } - var restObject = untransformACL(mongoObject); + var restObject = {}; for (var key in mongoObject) { var expectedType = schema.getExpectedType(className, key); var value = mongoObject[key]; From 097400d623cd6d12a666583c354ad86cebdd9f1f Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 22:38:18 -0700 Subject: [PATCH 09/11] Fix bugs with inner objects behaving strange when other fields have same name as key in specific circumstances --- spec/ParseAPI.spec.js | 23 ++++++++++++++++++++ src/Adapters/Storage/Mongo/MongoTransform.js | 16 +------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/spec/ParseAPI.spec.js b/spec/ParseAPI.spec.js index 9573784c55..80fe442c1f 100644 --- a/spec/ParseAPI.spec.js +++ b/spec/ParseAPI.spec.js @@ -1147,4 +1147,27 @@ describe('miscellaneous', function() { done(); }); }); + + it('does not change inner objects if the key has the same name as a geopoint field on the class, and the value is an array of length 2, or if the key has the same name as a file field on the class, and the value is a string', done => { + let file = new Parse.File('myfile.txt', { base64: 'eAo=' }); + file.save() + .then(f => { + let obj = new Parse.Object('O'); + obj.set('fileField', f); + obj.set('geoField', new Parse.GeoPoint(0, 0)); + obj.set('innerObj', { + fileField: "data", + geoField: [1,2], + }); + return obj.save(); + }) + .then(object => object.fetch()) + .then(object => { + expect(object.get('innerObj')).toEqual({ + fileField: "data", + geoField: [1,2], + }); + done(); + }); + }); }); diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index 0441a265b8..66ec96f3b1 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -747,21 +747,7 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => return BytesCoder.databaseToJSON(mongoObject); } - var restObject = {}; - for (var key in mongoObject) { - var expectedType = schema.getExpectedType(className, key); - var value = mongoObject[key]; - if (expectedType && expectedType.type === 'File' && FileCoder.isValidDatabaseObject(value)) { - restObject[key] = FileCoder.databaseToJSON(value); - break; - } - if (expectedType && expectedType.type === 'GeoPoint' && GeoPointCoder.isValidDatabaseObject(value)) { - restObject[key] = GeoPointCoder.databaseToJSON(value); - break; - } - restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); - } - return restObject; + return _.mapValues(mongoObject, value => nestedMongoObjectToNestedParseObject(schema, className, value)); default: throw 'unknown js type'; } From 2c4fdad46a804a5498c10018aab1d280c4343c18 Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Sat, 21 May 2016 22:45:38 -0700 Subject: [PATCH 10/11] remove params from untransform nested object --- src/Adapters/Storage/Mongo/MongoTransform.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoTransform.js b/src/Adapters/Storage/Mongo/MongoTransform.js index 66ec96f3b1..3bb008f78a 100644 --- a/src/Adapters/Storage/Mongo/MongoTransform.js +++ b/src/Adapters/Storage/Mongo/MongoTransform.js @@ -713,7 +713,7 @@ function transformUpdateOperator({ } } -const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => { +const nestedMongoObjectToNestedParseObject = mongoObject => { switch(typeof mongoObject) { case 'string': case 'number': @@ -728,7 +728,7 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => return null; } if (mongoObject instanceof Array) { - return mongoObject.map(arrayEntry => nestedMongoObjectToNestedParseObject(schema, className, arrayEntry)); + return mongoObject.map(nestedMongoObjectToNestedParseObject); } if (mongoObject instanceof Date) { @@ -747,7 +747,7 @@ const nestedMongoObjectToNestedParseObject = (schema, className, mongoObject) => return BytesCoder.databaseToJSON(mongoObject); } - return _.mapValues(mongoObject, value => nestedMongoObjectToNestedParseObject(schema, className, value)); + return _.mapValues(mongoObject, nestedMongoObjectToNestedParseObject); default: throw 'unknown js type'; } @@ -770,9 +770,7 @@ const mongoObjectToParseObject = (schema, className, mongoObject) => { return null; } if (mongoObject instanceof Array) { - return mongoObject.map(arrayEntry => { - return nestedMongoObjectToNestedParseObject(schema, className, arrayEntry); - }); + return mongoObject.map(nestedMongoObjectToNestedParseObject); } if (mongoObject instanceof Date) { @@ -874,7 +872,7 @@ const mongoObjectToParseObject = (schema, className, mongoObject) => { break; } } - restObject[key] = nestedMongoObjectToNestedParseObject(schema, className, mongoObject[key]); + restObject[key] = nestedMongoObjectToNestedParseObject(mongoObject[key]); } } From 10d5f7c4188b1d7622d60647db501a4e80ca223a Mon Sep 17 00:00:00 2001 From: Drew Gross Date: Mon, 23 May 2016 11:54:54 -0700 Subject: [PATCH 11/11] Revert changes to find --- src/Adapters/Storage/Mongo/MongoCollection.js | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoCollection.js b/src/Adapters/Storage/Mongo/MongoCollection.js index e309281884..bf41582b19 100644 --- a/src/Adapters/Storage/Mongo/MongoCollection.js +++ b/src/Adapters/Storage/Mongo/MongoCollection.js @@ -1,6 +1,5 @@ let mongodb = require('mongodb'); let Collection = mongodb.Collection; -import * as transform from './MongoTransform'; export default class MongoCollection { _mongoCollection:Collection; @@ -14,28 +13,25 @@ export default class MongoCollection { // none, then build the geoindex. // This could be improved a lot but it's not clear if that's a good // idea. Or even if this behavior is a good idea. - - // Depends on the className and schemaController because mongoObjectToParseObject does. - // TODO: break this dependency find(query, { skip, limit, sort } = {}) { return this._rawFind(query, { skip, limit, sort }) - .catch(error => { - // Check for "no geoindex" error - if (error.code != 17007 && !error.message.match(/unable to find index for .geoNear/)) { - throw error; - } - // Figure out what key needs an index - let key = error.message.match(/field=([A-Za-z_0-9]+) /)[1]; - if (!key) { - throw error; - } + .catch(error => { + // Check for "no geoindex" error + if (error.code != 17007 && !error.message.match(/unable to find index for .geoNear/)) { + throw error; + } + // Figure out what key needs an index + let key = error.message.match(/field=([A-Za-z_0-9]+) /)[1]; + if (!key) { + throw error; + } - var index = {}; - index[key] = '2d'; - return this._mongoCollection.createIndex(index) - // Retry, but just once. - .then(() => this._rawFind(query, { skip, limit, sort })); - }) + var index = {}; + index[key] = '2d'; + return this._mongoCollection.createIndex(index) + // Retry, but just once. + .then(() => this._rawFind(query, { skip, limit, sort })); + }); } _rawFind(query, { skip, limit, sort } = {}) {