Merge pull request intel-cloud#287 from noralife/noralife/gcs-plugin

Plugin for Google Cloud Storage
sunbines · Dec 2, 2015 · 308db3c · 308db3c
2 parents ce145b7 + 6ef3101
commit 308db3c
Show file tree

Hide file tree

Showing 25 changed files with 357 additions and 1 deletion.
diff --git a/dev/cosbench-gcs/.classpath b/dev/cosbench-gcs/.classpath
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry exported="true" kind="lib" path="google-api-client-1.19.0.jar"/>
+	<classpathentry exported="true" kind="lib" path="google-api-services-storage-v1-rev18-1.19.0.jar"/>
+	<classpathentry exported="true" kind="lib" path="google-http-client-1.19.0.jar"/>
+	<classpathentry exported="true" kind="lib" path="google-http-client-jackson2-1.19.0.jar"/>
+	<classpathentry exported="true" kind="lib" path="google-oauth-client-1.19.0.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
+	<classpathentry exported="true" kind="lib" path="commons-codec-1.3.jar"/>
+	<classpathentry exported="true" kind="lib" path="commons-logging-1.1.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="guava-jdk5-13.0.jar"/>
+	<classpathentry exported="true" kind="lib" path="hamcrest-core-1.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="httpclient-4.0.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="httpcore-4.0.1.jar"/>
+	<classpathentry exported="true" kind="lib" path="jackson-core-2.1.3.jar"/>
+	<classpathentry exported="true" kind="lib" path="jsr305-1.3.9.jar"/>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>
diff --git a/dev/cosbench-gcs/.project b/dev/cosbench-gcs/.project
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>cosbench-gcs</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.ManifestBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.pde.SchemaBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.pde.PluginNature</nature>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>
diff --git a/dev/cosbench-gcs/META-INF/MANIFEST.MF b/dev/cosbench-gcs/META-INF/MANIFEST.MF
@@ -0,0 +1,37 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: Google Cloud Storage Client Bundle
+Bundle-SymbolicName: cosbench-gcs
+Bundle-Version: 0.4.1.0
+Bundle-Vendor: Intel
+Bundle-RequiredExecutionEnvironment: JavaSE-1.6
+Import-Package: com.intel.cosbench.api.auth,
+ com.intel.cosbench.api.context,
+ com.intel.cosbench.api.storage,
+ com.intel.cosbench.client.http,
+ com.intel.cosbench.config,
+ com.intel.cosbench.log,
+ org.apache.commons.codec;version="[1.3.0,2.0.0)",
+ org.apache.commons.codec.net;version="[1.3.0,2.0.0)",
+ org.apache.http;version="[4.1.4,5.0.0)",
+ org.apache.http.client;version="[4.1.3,5.0.0)",
+ org.apache.http.client.methods;version="[4.1.3,5.0.0)",
+ org.apache.http.conn;version="[4.1.3,5.0.0)",
+ org.apache.http.entity;version="[4.1.4,5.0.0)",
+ org.apache.http.message;version="[4.1.4,5.0.0)",
+ org.apache.http.params;version="[4.1.4,5.0.0)",
+ org.apache.http.util;version="[4.1.4,5.0.0)"
+Bundle-ClassPath: .,
+ google-api-client-1.19.0.jar,
+ google-api-services-storage-v1-rev18-1.19.0.jar,
+ google-http-client-1.19.0.jar,
+ google-http-client-jackson2-1.19.0.jar,
+ google-oauth-client-1.19.0.jar,
+ commons-codec-1.3.jar,
+ commons-logging-1.1.1.jar,
+ guava-jdk5-13.0.jar,
+ hamcrest-core-1.1.jar,
+ httpclient-4.0.1.jar,
+ httpcore-4.0.1.jar,
+ jackson-core-2.1.3.jar,
+ jsr305-1.3.9.jar
diff --git a/dev/cosbench-gcs/META-INF/spring/plugin-context.xml b/dev/cosbench-gcs/META-INF/spring/plugin-context.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:osgi="http://www.springframework.org/schema/osgi"
+	xsi:schemaLocation="
+         http://www.springframework.org/schema/beans
+         http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
+         http://www.springframework.org/schema/osgi
+         http://www.springframework.org/schema/osgi/spring-osgi.xsd">
+
+	<bean name="storageFactory" class="com.intel.cosbench.api.GcsStor.GcsStorageFactory" />
+
+	<osgi:service ref="storageFactory" context-class-loader="service-provider"
+		interface="com.intel.cosbench.api.storage.StorageAPIFactory">
+	</osgi:service>
+
+</beans>
diff --git a/dev/cosbench-gcs/build.properties b/dev/cosbench-gcs/build.properties
@@ -0,0 +1,18 @@
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+               .,\
+               google-api-client-1.19.0.jar,\
+               google-api-services-storage-v1-rev18-1.19.0.jar,\
+               google-http-client-1.19.0.jar,\
+               google-http-client-jackson2-1.19.0.jar,\
+               google-oauth-client-1.19.0.jar,\
+               guava-jdk5-13.0.jar,\
+               hamcrest-core-1.1.jar,\
+               httpclient-4.0.1.jar,\
+               httpcore-4.0.1.jar,\
+               jackson-core-2.1.3.jar,\
+               jsr305-1.3.9.jar,\
+               commons-codec-1.3.jar,\
+               commons-logging-1.1.1.jar
+
diff --git a/dev/cosbench-gcs/commons-codec-1.3.jar b/dev/cosbench-gcs/commons-codec-1.3.jar
diff --git a/dev/cosbench-gcs/commons-logging-1.1.1.jar b/dev/cosbench-gcs/commons-logging-1.1.1.jar
diff --git a/dev/cosbench-gcs/google-api-client-1.19.0.jar b/dev/cosbench-gcs/google-api-client-1.19.0.jar
diff --git a/dev/cosbench-gcs/google-api-services-storage-v1-rev18-1.19.0.jar b/dev/cosbench-gcs/google-api-services-storage-v1-rev18-1.19.0.jar
diff --git a/dev/cosbench-gcs/google-http-client-1.19.0.jar b/dev/cosbench-gcs/google-http-client-1.19.0.jar
diff --git a/dev/cosbench-gcs/google-http-client-jackson2-1.19.0.jar b/dev/cosbench-gcs/google-http-client-jackson2-1.19.0.jar
diff --git a/dev/cosbench-gcs/google-oauth-client-1.19.0.jar b/dev/cosbench-gcs/google-oauth-client-1.19.0.jar
diff --git a/dev/cosbench-gcs/guava-jdk5-13.0.jar b/dev/cosbench-gcs/guava-jdk5-13.0.jar
diff --git a/dev/cosbench-gcs/hamcrest-core-1.1.jar b/dev/cosbench-gcs/hamcrest-core-1.1.jar
diff --git a/dev/cosbench-gcs/httpclient-4.0.1.jar b/dev/cosbench-gcs/httpclient-4.0.1.jar
diff --git a/dev/cosbench-gcs/httpcore-4.0.1.jar b/dev/cosbench-gcs/httpcore-4.0.1.jar
diff --git a/dev/cosbench-gcs/jackson-core-2.1.3.jar b/dev/cosbench-gcs/jackson-core-2.1.3.jar
diff --git a/dev/cosbench-gcs/jsr305-1.3.9.jar b/dev/cosbench-gcs/jsr305-1.3.9.jar
diff --git a/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorage.java b/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorage.java
@@ -0,0 +1,147 @@
+package com.intel.cosbench.api.GcsStor;
+
+import static com.intel.cosbench.client.GcsStor.GcsStorConstants.CONN_TIMEOUT_KEY;
+import static com.intel.cosbench.client.GcsStor.GcsStorConstants.CONN_TIMEOUT_DEFAULT;
+import static com.intel.cosbench.client.GcsStor.GcsStorConstants.JSON_KEY_FILE;
+import static com.intel.cosbench.client.GcsStor.GcsStorConstants.JSON_KEY_FILE_DEFAULT;
+import static com.intel.cosbench.client.GcsStor.GcsStorConstants.PROJECT_ID;
+import static com.intel.cosbench.client.GcsStor.GcsStorConstants.PROJECT_ID_DEFAULT;
+
+import java.io.*;
+
+import com.intel.cosbench.api.storage.*;
+import com.intel.cosbench.api.context.*;
+import com.intel.cosbench.config.Config;
+import com.intel.cosbench.log.Logger;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.InputStreamContent;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.StorageScopes;
+import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.StorageObject;
+
+public class GcsStorage extends NoneStorage {
+	private int timeout;
+	private String jsonKeyFile;
+	private String projectId;
+	private static Storage client;
+
+	private static final JsonFactory JSON_FACTORY = JacksonFactory
+			.getDefaultInstance();
+
+	@Override
+	public void init(Config config, Logger logger) {
+		super.init(config, logger);
+		initParms(config);
+		try {
+			InputStream input = new FileInputStream(jsonKeyFile);
+			GoogleCredential credential = GoogleCredential.fromStream(input);
+			if (credential.createScopedRequired()) {
+				credential = credential.createScoped(StorageScopes.all());
+			}
+			HttpTransport httpTransport = GoogleNetHttpTransport
+					.newTrustedTransport();
+			client = new Storage.Builder(httpTransport, JSON_FACTORY,
+					credential).build();
+			logger.debug("GCS client has been initialized");
+		} catch (Exception e) {
+			throw new StorageException(e);
+		}
+	}
+
+	private void initParms(Config config) {
+		timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT);
+		parms.put(CONN_TIMEOUT_KEY, timeout);
+
+		jsonKeyFile = config.get(JSON_KEY_FILE, JSON_KEY_FILE_DEFAULT);
+		parms.put(JSON_KEY_FILE, jsonKeyFile);
+
+		projectId = config.get(PROJECT_ID, PROJECT_ID_DEFAULT);
+		parms.put(PROJECT_ID, projectId);
+
+		logger.debug("using storage config: {}", parms);
+	}
+
+	@Override
+	public void setAuthContext(AuthContext info) {
+		super.setAuthContext(info);
+	}
+
+	@Override
+	public void dispose() {
+		super.dispose();
+		client = null;
+	}
+
+	@Override
+	public InputStream getObject(String container, String object, Config config) {
+		super.getObject(container, object, config);
+		InputStream stream;
+		try {
+			Storage.Objects.Get getRequest = client.objects().get(container,
+					object);
+			stream = getRequest.executeMediaAsInputStream();
+		} catch (IOException e) {
+			throw new StorageException(e);
+		}
+		return stream;
+	}
+
+	@Override
+	public void createContainer(String container, Config config) {
+		super.createContainer(container, config);
+		Bucket newBucket = new Bucket();
+		newBucket.setName(container);
+		try {
+			Storage.Buckets.Insert bucketInsertRequest = client.buckets()
+					.insert(projectId, newBucket);
+			bucketInsertRequest.execute();
+		} catch (IOException e) {
+			throw new StorageException(e);
+		}
+	}
+
+	@Override
+	public void createObject(String container, String object, InputStream data,
+			long length, Config config) {
+		super.createObject(container, object, data, length, config);
+		InputStreamContent contentStream = new InputStreamContent(
+				"application/octet-stream", data);
+		StorageObject objectMetadata = new StorageObject().setName(object);
+		try {
+			Storage.Objects.Insert insertRequest = client.objects().insert(
+					container, objectMetadata, contentStream);
+			insertRequest.execute();
+		} catch (IOException e) {
+			throw new StorageException(e);
+		}
+
+	}
+
+	@Override
+	public void deleteContainer(String container, Config config) {
+		super.deleteContainer(container, config);
+		try {
+			Storage.Buckets.Delete bucketDeleteRequest = client.buckets()
+					.delete(container);
+			bucketDeleteRequest.execute();
+		} catch (IOException e) {
+			throw new StorageException(e);
+		}
+	}
+
+	@Override
+	public void deleteObject(String container, String object, Config config) {
+		super.deleteObject(container, object, config);
+		try {
+			client.objects().delete(container, object).execute();
+		} catch (Exception e) {
+			throw new StorageException(e);
+		}
+	}
+
+}
diff --git a/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorageFactory.java b/dev/cosbench-gcs/src/com/intel/cosbench/api/GcsStor/GcsStorageFactory.java
@@ -0,0 +1,17 @@
+package com.intel.cosbench.api.GcsStor;
+
+import com.intel.cosbench.api.storage.*;
+
+public class GcsStorageFactory implements StorageAPIFactory {
+
+	@Override
+	public String getStorageName() {
+		return "gcs";
+	}
+
+	@Override
+	public StorageAPI getStorageAPI() {
+		return new GcsStorage();
+	}
+
+}
diff --git a/dev/cosbench-gcs/src/com/intel/cosbench/client/GcsStor/GcsStorConstants.java b/dev/cosbench-gcs/src/com/intel/cosbench/client/GcsStor/GcsStorConstants.java
@@ -0,0 +1,24 @@
+package com.intel.cosbench.client.GcsStor;
+
+public interface GcsStorConstants {
+
+	// --------------------------------------------------------------------------
+	// CONNECTION
+	// --------------------------------------------------------------------------
+
+	String CONN_TIMEOUT_KEY = "timeout";
+	int CONN_TIMEOUT_DEFAULT = 30000;
+
+	// --------------------------------------------------------------------------
+	// PROJECT_ID
+	// --------------------------------------------------------------------------
+	String PROJECT_ID = "projectid";
+	String PROJECT_ID_DEFAULT = "xxxxx-xxxx";
+
+	// --------------------------------------------------------------------------
+	// JSON_KEY_FILE
+	// --------------------------------------------------------------------------
+	String JSON_KEY_FILE = "jsonkeyfile";
+	String JSON_KEY_FILE_DEFAULT = "/path/to/json/key/file";
+
+}
diff --git a/release/conf/.driver/config.ini b/release/conf/.driver/config.ini
@@ -72,6 +72,7 @@ plugins/cosbench-scality@7\:start,\
 plugins/cosbench-swauth@7\:start,\
 plugins/cosbench-httpauth@7:\start,\
 plugins/cosbench-s3@7\:start,\
+plugins/cosbench-gcs@7\:start,\
 plugins/cosbench-librados@7\:start,\
 plugins/cosbench-cdmi-base@7\:start,\
 plugins/cosbench-cdmi-swift@7\:start,\

diff --git a/release/conf/gcs-config-sample.xml b/release/conf/gcs-config-sample.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<workload name="gcs-sample" description="sample benchmark for Google Cloud Storage">
+
+  <!-- Authentication needs service account credentials. -->
+  <!-- Sample credential file: ./release/conf/gcs-service-account-sample.json -->
+  <storage type="gcs" config="projectid=<projectid>;jsonkeyfile=</path/to/json/key.json/>" />
+
+  <workflow>
+
+    <workstage name="init">
+      <work type="init" workers="1" config="cprefix=gcstestqwer;containers=r(1,2)" />
+    </workstage>
+
+    <workstage name="prepare">
+      <work type="prepare" workers="1" config="cprefix=gcstestqwer;containers=r(1,2);objects=r(1,10);sizes=c(64)KB" />
+    </workstage>
+
+    <workstage name="main">
+      <work name="main" workers="8" runtime="30">
+        <operation type="read" ratio="80" config="cprefix=gcstestqwer;containers=u(1,2);objects=u(1,10)" />
+        <operation type="write" ratio="20" config="cprefix=gcstestqwer;containers=u(1,2);objects=u(11,20);sizes=c(64)KB" />
+      </work>
+    </workstage>
+
+    <workstage name="cleanup">
+      <work type="cleanup" workers="1" config="cprefix=gcstestqwer;containers=r(1,2);objects=r(1,20)" />
+    </workstage>
+
+    <workstage name="dispose">
+      <work type="dispose" workers="1" config="cprefix=gcstestqwer;containers=r(1,2)" />
+    </workstage>
+
+  </workflow>
+
+</workload>
diff --git a/release/conf/gcs-service-account-sample.json b/release/conf/gcs-service-account-sample.json
@@ -0,0 +1,12 @@
+// Service account json file can be generated from https://console.developers.google.com/apis/credentials
+{
+  "type": "service_account",
+  "private_key_id": "****************************************",
+  "private_key": "-----BEGIN PRIVATE KEY-----\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n*******************\n-----END PRIVATE KEY-----\n",
+  "client_email": "*****@*****.****",
+  "client_id": "************************",
+  "auth_uri": "https://****.****.***/",
+  "token_uri": "https://****.****.***/",
+  "auth_provider_x509_cert_url": "https://****.****.***/",
+  "client_x509_cert_url": "https://****.****.***/"
+}
diff --git a/release/start-driver.sh b/release/start-driver.sh
@@ -22,7 +22,7 @@
 SERVICE_NAME=driver
 VERSION=`cat VERSION`
 
-OSGI_BUNDLES="cosbench-log_${VERSION} cosbench-tomcat_${VERSION} cosbench-config_${VERSION} cosbench-http_${VERSION} cosbench-cdmi-util_${VERSION} cosbench-core_${VERSION} cosbench-core-web_${VERSION} cosbench-api_${VERSION} cosbench-mock_${VERSION} cosbench-ampli_${VERSION} cosbench-swift_${VERSION} cosbench-keystone_${VERSION} cosbench-httpauth_${VERSION} cosbench-s3_${VERSION} cosbench-librados_${VERSION} cosbench-scality_${VERSION} cosbench-cdmi-swift_${VERSION} cosbench-cdmi-base_${VERSION} cosbench-driver_${VERSION} cosbench-driver-web_${VERSION}"
+OSGI_BUNDLES="cosbench-log_${VERSION} cosbench-tomcat_${VERSION} cosbench-config_${VERSION} cosbench-http_${VERSION} cosbench-cdmi-util_${VERSION} cosbench-core_${VERSION} cosbench-core-web_${VERSION} cosbench-api_${VERSION} cosbench-mock_${VERSION} cosbench-ampli_${VERSION} cosbench-swift_${VERSION} cosbench-keystone_${VERSION} cosbench-httpauth_${VERSION} cosbench-s3_${VERSION} cosbench-gcs_${VERSION} cosbench-librados_${VERSION} cosbench-scality_${VERSION} cosbench-cdmi-swift_${VERSION} cosbench-cdmi-base_${VERSION} cosbench-driver_${VERSION} cosbench-driver-web_${VERSION}"
 
 OSGI_CONSOLE_PORT=18089