diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index a15ba0c2fa..0000000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index de88257b3b..8ea322c438 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ lib/ # cache folder .cache + +# Mac DS_Store files +.DS_Store diff --git a/README.md b/README.md index b0a8d01559..c4a86d1056 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,14 @@ PARSE_SERVER_MAX_UPLOAD_SIZE ``` -##### Configuring S3 Adapter +##### Configuring File Adapters +Parse Server allows developers to choose from several options when hosting files: the `GridStoreAdapter`, which backed by MongoDB; the `S3Adapter`, which is backed by [Amazon S3](https://aws.amazon.com/s3/); or the `GCSAdapter`, which is backed by [Google Cloud Storage](https://cloud.google.com/storage/). -You can use the following environment variable setup the S3 adapter +`GridStoreAdapter` is used by default and requires no setup, but if you're interested in using S3 or GCS, additional configuration information is available below. + +###### Configuring `S3Adapter` + +You can use the following environment variable setup to enable the S3 adapter: ```js S3_ACCESS_KEY @@ -149,6 +154,19 @@ S3_DIRECT_ACCESS ``` +###### Configuring `GCSAdapter` + +You can use the following environment variable setup to enable the GCS adapter: + +```js +GCP_PROJECT_ID +GCP_KEYFILE_PATH +GCS_BUCKET +GCS_BUCKET_PREFIX +GCS_DIRECT_ACCESS + +``` + ## Contributing We really want Parse to be yours, to see it grow and thrive in the open source community. Please see the [Contributing to Parse Server guide](CONTRIBUTING.md). diff --git a/package.json b/package.json index ee0e0b4e5b..89ceb754c0 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "commander": "^2.9.0", "deepcopy": "^0.6.1", "express": "^4.13.4", + "gcloud": "^0.28.0", "mailgun-js": "^0.7.7", "mime": "^1.3.4", "mongodb": "~2.1.0", diff --git a/spec/AdapterLoader.spec.js b/spec/AdapterLoader.spec.js index 69381fc547..56bf0d448d 100644 --- a/spec/AdapterLoader.spec.js +++ b/spec/AdapterLoader.spec.js @@ -3,44 +3,45 @@ var loadAdapter = require("../src/Adapters/AdapterLoader").loadAdapter; var FilesAdapter = require("../src/Adapters/Files/FilesAdapter").default; var ParsePushAdapter = require("../src/Adapters/Push/ParsePushAdapter"); var S3Adapter = require("../src/Adapters/Files/S3Adapter").default; +var GCSAdapter = require("../src/Adapters/Files/GCSAdapter").default; describe("AdapterLoader", ()=>{ - + it("should instantiate an adapter from string in object", (done) => { var adapterPath = require('path').resolve("./spec/MockAdapter"); var adapter = loadAdapter({ adapter: adapterPath, options: { - key: "value", + key: "value", foo: "bar" } }); - + expect(adapter instanceof Object).toBe(true); expect(adapter.options.key).toBe("value"); expect(adapter.options.foo).toBe("bar"); done(); }); - + it("should instantiate an adapter from string", (done) => { var adapterPath = require('path').resolve("./spec/MockAdapter"); var adapter = loadAdapter(adapterPath); - + expect(adapter instanceof Object).toBe(true); done(); }); - + it("should instantiate an adapter from string that is module", (done) => { var adapterPath = require('path').resolve("./src/Adapters/Files/FilesAdapter"); var adapter = loadAdapter({ adapter: adapterPath }); - + expect(adapter instanceof FilesAdapter).toBe(true); done(); }); - + it("should instantiate an adapter from function/Class", (done) => { var adapter = loadAdapter({ adapter: FilesAdapter @@ -48,27 +49,27 @@ describe("AdapterLoader", ()=>{ expect(adapter instanceof FilesAdapter).toBe(true); done(); }); - + it("should instantiate the default adapter from Class", (done) => { var adapter = loadAdapter(null, FilesAdapter); expect(adapter instanceof FilesAdapter).toBe(true); done(); }); - + it("should use the default adapter", (done) => { var defaultAdapter = new FilesAdapter(); var adapter = loadAdapter(null, defaultAdapter); expect(adapter instanceof FilesAdapter).toBe(true); done(); }); - + it("should use the provided adapter", (done) => { var originalAdapter = new FilesAdapter(); var adapter = loadAdapter(originalAdapter); expect(adapter).toBe(originalAdapter); done(); }); - + it("should fail loading an improperly configured adapter", (done) => { var Adapter = function(options) { if (!options.foo) { @@ -79,14 +80,14 @@ describe("AdapterLoader", ()=>{ param: "key", doSomething: function() {} }; - + expect(() => { var adapter = loadAdapter(adapterOptions, Adapter); expect(adapter).toEqual(adapterOptions); }).not.toThrow("foo is required for that adapter"); done(); }); - + it("should load push adapter from options", (done) => { var options = { ios: { @@ -100,7 +101,7 @@ describe("AdapterLoader", ()=>{ }).not.toThrow(); done(); }); - + it("should load S3Adapter from direct passing", (done) => { var s3Adapter = new S3Adapter("key", "secret", "bucket") expect(() => { @@ -109,4 +110,13 @@ describe("AdapterLoader", ()=>{ }).not.toThrow(); done(); }) + + it("should load GCSAdapter from direct passing", (done) => { + var gcsAdapter = new GCSAdapter("projectId", "path/to/keyfile", "bucket") + expect(() => { + var adapter = loadAdapter(gcsAdapter, FilesAdapter); + expect(adapter).toBe(gcsAdapter); + }).not.toThrow(); + done(); + }) }); diff --git a/spec/FilesController.spec.js b/spec/FilesController.spec.js index 2306676e83..3b2108e71e 100644 --- a/spec/FilesController.spec.js +++ b/spec/FilesController.spec.js @@ -1,6 +1,7 @@ var FilesController = require('../src/Controllers/FilesController').FilesController; var GridStoreAdapter = require("../src/Adapters/Files/GridStoreAdapter").GridStoreAdapter; var S3Adapter = require("../src/Adapters/Files/S3Adapter").S3Adapter; +var GCSAdapter = require("../src/Adapters/Files/GCSAdapter").GCSAdapter; var Config = require("../src/Config"); var FCTestFactory = require("./FilesControllerTestFactory"); @@ -8,26 +9,44 @@ var FCTestFactory = require("./FilesControllerTestFactory"); // Small additional tests to improve overall coverage describe("FilesController",()=>{ - + // Test the grid store adapter var gridStoreAdapter = new GridStoreAdapter('mongodb://localhost:27017/parse'); FCTestFactory.testAdapter("GridStoreAdapter", gridStoreAdapter); - + if (process.env.S3_ACCESS_KEY && process.env.S3_SECRET_KEY) { - + // Test the S3 Adapter var s3Adapter = new S3Adapter(process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY, 'parse.server.tests'); - + FCTestFactory.testAdapter("S3Adapter",s3Adapter); - + // Test S3 with direct access var s3DirectAccessAdapter = new S3Adapter(process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY, 'parse.server.tests', { directAccess: true }); - + FCTestFactory.testAdapter("S3AdapterDirect", s3DirectAccessAdapter); - + } else if (!process.env.TRAVIS) { console.log("set S3_ACCESS_KEY and S3_SECRET_KEY to test S3Adapter") } + + if (process.env.GCP_PROJECT_ID && process.env.GCP_KEYFILE_PATH && process.env.GCS_BUCKET) { + + // Test the GCS Adapter + var gcsAdapter = new GCSAdapter(process.env.GCP_PROJECT_ID, process.env.GCP_KEYFILE_PATH, process.env.GCS_BUCKET); + + FCTestFactory.testAdapter("GCSAdapter", gcsAdapter); + + // Test GCS with direct access + var gcsDirectAccessAdapter = new GCSAdapter(process.env.GCP_PROJECT_ID, process.env.GCP_KEYFILE_PATH, process.env.GCS_BUCKET, { + directAccess: true + }); + + FCTestFactory.testAdapter("GCSAdapterDirect", gcsDirectAccessAdapter); + + } else if (!process.env.TRAVIS) { + console.log("set GCP_PROJECT_ID, GCP_KEYFILE_PATH, and GCS_BUCKET to test GCSAdapter") + } }); diff --git a/spec/FilesControllerTestFactory.js b/spec/FilesControllerTestFactory.js index 217a383a16..b467d031f5 100644 --- a/spec/FilesControllerTestFactory.js +++ b/spec/FilesControllerTestFactory.js @@ -1,35 +1,34 @@ - var FilesController = require('../src/Controllers/FilesController').FilesController; var Config = require("../src/Config"); var testAdapter = function(name, adapter) { // Small additional tests to improve overall coverage - + var config = new Config(Parse.applicationId); var filesController = new FilesController(adapter); describe("FilesController with "+name,()=>{ - + it("should properly expand objects", (done) => { - + var result = filesController.expandFilesInObject(config, function(){}); - + expect(result).toBeUndefined(); - + var fullFile = { type: '__type', url: "http://an.url" } - + var anObject = { aFile: fullFile } filesController.expandFilesInObject(config, anObject); expect(anObject.aFile.url).toEqual("http://an.url"); - + done(); - }) - + }) + it("should properly create, read, delete files", (done) => { var filename; filesController.createFile(config, "file.txt", "hello world").then( (result) => { @@ -51,14 +50,14 @@ var testAdapter = function(name, adapter) { console.error(err); done(); }).then((result) => { - + filesController.getFileData(config, filename).then((res) => { fail("the file should be deleted"); done(); }, (err) => { - done(); + done(); }); - + }, (err) => { fail("The adapter should delete the file"); console.error(err); diff --git a/src/Adapters/AdapterLoader.js b/src/Adapters/AdapterLoader.js index a9521f0b03..654948e96c 100644 --- a/src/Adapters/AdapterLoader.js +++ b/src/Adapters/AdapterLoader.js @@ -1,6 +1,5 @@ export function loadAdapter(adapter, defaultAdapter, options) { - - if (!adapter) + if (!adapter) { if (!defaultAdapter) { return options; @@ -20,7 +19,7 @@ export function loadAdapter(adapter, defaultAdapter, options) { if (adapter.default) { adapter = adapter.default; } - + return loadAdapter(adapter, undefined, options); } else if (adapter.module) { return loadAdapter(adapter.module, undefined, adapter.options); @@ -30,7 +29,7 @@ export function loadAdapter(adapter, defaultAdapter, options) { return loadAdapter(adapter.adapter, undefined, adapter.options); } // return the adapter as provided - return adapter; + return adapter; } export default loadAdapter; diff --git a/src/Adapters/Files/GCSAdapter.js b/src/Adapters/Files/GCSAdapter.js new file mode 100644 index 0000000000..8fb34af840 --- /dev/null +++ b/src/Adapters/Files/GCSAdapter.js @@ -0,0 +1,125 @@ +// GCSAdapter +// Store Parse Files in Google Cloud Storage: https://cloud.google.com/storage +import { storage } from 'gcloud'; +import { FilesAdapter } from './FilesAdapter'; +import requiredParameter from '../../requiredParameter'; + +function requiredOrFromEnvironment(env, name) { + let environmentVariable = process.env[env]; + if (!environmentVariable) { + requiredParameter(`GCSAdapter requires an ${name}`); + } + return environmentVariable; +} + +function fromEnvironmentOrDefault(env, defaultValue) { + let environmentVariable = process.env[env]; + if (environmentVariable) { + return environmentVariable; + } + return defaultValue; +} + +export class GCSAdapter extends FilesAdapter { + // GCS Project ID and the name of a corresponding Keyfile are required. + // Unlike the S3 adapter, you must create a new Cloud Storage bucket, as this is not created automatically. + // See https://googlecloudplatform.github.io/gcloud-node/#/docs/master/guides/authentication + // for more details. + constructor( + projectId = requiredOrFromEnvironment('GCP_PROJECT_ID', 'projectId'), + keyFilename = requiredOrFromEnvironment('GCP_KEYFILE_PATH', 'keyfile path'), + bucket = requiredOrFromEnvironment('GCS_BUCKET', 'bucket name'), + { bucketPrefix = fromEnvironmentOrDefault('GCS_BUCKET_PREFIX', ''), + directAccess = fromEnvironmentOrDefault('GCS_DIRECT_ACCESS', false) } = {}) { + super(); + + this._bucket = bucket; + this._bucketPrefix = bucketPrefix; + this._directAccess = directAccess; + + let options = { + projectId: projectId, + keyFilename: keyFilename + }; + + this._gcsClient = new storage(options); + } + + // For a given config object, filename, and data, store a file in GCS. + // Resolves the promise or fails with an error. + createFile(config, filename, data, contentType) { + let params = { + contentType: contentType || 'application/octet-stream' + }; + + return new Promise((resolve, reject) => { + let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename); + // gcloud supports upload(file) not upload(bytes), so we need to stream. + var uploadStream = file.createWriteStream(params); + uploadStream.on('error', (err) => { + return reject(err); + }).on('finish', () => { + // Second call to set public read ACL after object is uploaded. + if (this._directAccess) { + file.makePublic((err, res) => { + if (err !== null) { + return reject(err); + } + resolve(); + }); + } else { + resolve(); + } + }); + uploadStream.write(data); + uploadStream.end(); + }); + } + + // Deletes a file with the given file name. + // Returns a promise that succeeds with the delete response, or fails with an error. + deleteFile(config, filename) { + return new Promise((resolve, reject) => { + let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename); + file.delete((err, res) => { + if(err !== null) { + return reject(err); + } + resolve(res); + }); + }); + } + + // Search for and return a file if found by filename. + // Returns a promise that succeeds with the buffer result from GCS, or fails with an error. + getFileData(config, filename) { + return new Promise((resolve, reject) => { + let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename); + // Check for existence, since gcloud-node seemed to be caching the result + file.exists((err, exists) => { + if (exists) { + file.download((err, data) => { + if (err !== null) { + return reject(err); + } + return resolve(data); + }); + } else { + reject(err); + } + }); + }); + } + + // Generates and returns the location of a file stored in GCS for the given request and filename. + // The location is the direct GCS link if the option is set, + // otherwise we serve the file through parse-server. + getFileLocation(config, filename) { + if (this._directAccess) { + return `https://${this._bucket}.storage.googleapis.com/${this._bucketPrefix + filename}`; + } + return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename)); + } +} + +export default GCSAdapter; diff --git a/src/Controllers/AdaptableController.js b/src/Controllers/AdaptableController.js index ab7d715667..7ff8ce2906 100644 --- a/src/Controllers/AdaptableController.js +++ b/src/Controllers/AdaptableController.js @@ -2,7 +2,7 @@ AdaptableController.js AdaptableController is the base class for all controllers -that support adapter, +that support adapter, The super class takes care of creating the right instance for the adapter based on the parameters passed @@ -28,30 +28,30 @@ export class AdaptableController { this.validateAdapter(adapter); this[_adapter] = adapter; } - + get adapter() { return this[_adapter]; } - + get config() { return new Config(this.appId); } - + expectedAdapterType() { throw new Error("Subclasses should implement expectedAdapterType()"); } - + validateAdapter(adapter) { if (!adapter) { throw new Error(this.constructor.name+" requires an adapter"); } - + let Type = this.expectedAdapterType(); // Allow skipping for testing - if (!Type) { + if (!Type) { return; } - + // Makes sure the prototype matches let mismatches = Object.getOwnPropertyNames(Type.prototype).reduce( (obj, key) => { const adapterType = typeof adapter[key]; @@ -64,7 +64,7 @@ export class AdaptableController { } return obj; }, {}); - + if (Object.keys(mismatches).length > 0) { throw new Error("Adapter prototype don't match expected prototype", adapter, mismatches); } diff --git a/src/Controllers/FilesController.js b/src/Controllers/FilesController.js index 712e326c16..335dfdf2bd 100644 --- a/src/Controllers/FilesController.js +++ b/src/Controllers/FilesController.js @@ -13,11 +13,11 @@ export class FilesController extends AdaptableController { } createFile(config, filename, data, contentType) { - + let extname = path.extname(filename); - + const hasExtension = extname.length > 0; - + if (!hasExtension && contentType && mime.extension(contentType)) { filename = filename + '.' + mime.extension(contentType); } else if (hasExtension && !contentType) { diff --git a/src/index.js b/src/index.js index 6244fc388f..86320b2f69 100644 --- a/src/index.js +++ b/src/index.js @@ -24,6 +24,7 @@ import { FileLoggerAdapter } from './Adapters/Logger/FileLoggerAdapter'; import { FilesController } from './Controllers/FilesController'; import { FilesRouter } from './Routers/FilesRouter'; import { FunctionsRouter } from './Routers/FunctionsRouter'; +import { GCSAdapter } from './Adapters/Files/GCSAdapter'; import { GlobalConfigRouter } from './Routers/GlobalConfigRouter'; import { GridStoreAdapter } from './Adapters/Files/GridStoreAdapter'; import { HooksController } from './Controllers/HooksController'; @@ -259,4 +260,5 @@ function addParseCloud() { module.exports = { ParseServer: ParseServer, S3Adapter: S3Adapter, + GCSAdapter: GCSAdapter };