Merge pull request #41 from IBMStreams/develop

joergboe · web-flow · commit 104fb9b7766a · 2019-10-11T17:04:57.000+02:00
Develop
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -41,3 +41,6 @@
 * Enhancemement: Parameter contextResourceBase is now independent from parameter context
 * Enhancemement in description for generation of client certitficate which can be used in a brower
 
+## v4.3.0
+* Enhancement: New parameter 'sslAppConfigName' to get the SSL server key/certificate and the client trust material from Streams application configuration with this name
+
diff --git a/com.ibm.streamsx.inetserver/com.ibm.streamsx.inet/namespace-info.spl b/com.ibm.streamsx.inetserver/com.ibm.streamsx.inet/namespace-info.spl
@@ -32,6 +32,7 @@
  * * `keyStore`
  * * `keyStorePassword`
  * * `keyPassword`
+ * * `sslAppConfigName`
  *
  * **Note:** Jetty server instances will fail to start if they are placed on a single host and use the same port number. 
  * 
diff --git a/com.ibm.streamsx.inetserver/impl/java/src/com/ibm/streamsx/inet/rest/engine/ServletEngine.java b/com.ibm.streamsx.inetserver/impl/java/src/com/ibm/streamsx/inet/rest/engine/ServletEngine.java
@@ -4,14 +4,23 @@
 */
 package com.ibm.streamsx.inet.rest.engine;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
 import java.lang.management.ManagementFactory;
 import java.net.URI;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Base64.Decoder;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -49,6 +58,7 @@
 import org.eclipse.jetty.util.thread.ThreadPool;
 
 import com.ibm.streams.operator.OperatorContext;
+import com.ibm.streams.operator.ProcessingElement;
 import com.ibm.streams.operator.StreamingData;
 import com.ibm.streams.operator.management.OperatorManagement;
 import com.ibm.streamsx.inet.rest.ops.Functions;
@@ -87,6 +97,8 @@ public class ServletEngine implements ServletEngineMBean, MBeanRegistration {
 
 	public static final String SSL_TRUSTSTORE_PARAM = "trustStore";
 	public static final String SSL_TRUSTSTORE_PASSWORD_PARAM = "trustStorePassword";
+	
+	public static final String SSL_APP_CONFIG_NAME_PARAM = "sslAppConfigName";
 
 	public static final int IDLE_TIMEOUT = 30000;
 	public static final int STRICT_TRANSPORT_SECURITY_MAX_AGE = 2000;
@@ -168,7 +180,8 @@ public void join() throws InterruptedException {
 		server = new Server(tp);
 		handlers = new ContextHandlerCollection();
 
-		if (operatorContext.getParameterNames().contains(SSL_CERT_ALIAS_PARAM))
+		if (	   (operatorContext.getParameterNames().contains(SSL_CERT_ALIAS_PARAM))
+				|| (operatorContext.getParameterNames().contains(SSL_APP_CONFIG_NAME_PARAM)))
 			setHTTPSConnector(operatorContext, server, portNumber, host);
 		else
 			setHTTPConnector(operatorContext, server, portNumber, host);
@@ -208,47 +221,102 @@ private void setHTTPConnector(OperatorContext operatorContext, Server server, in
 
 	/**
 	 * Setup an HTTPS connector.
+	 * @throws KeyStoreException 
+	 * @throws IOException 
+	 * @throws CertificateException 
+	 * @throws NoSuchAlgorithmException 
 	 */
-	private void setHTTPSConnector(OperatorContext operatorContext, Server server, int httpsPort, String host) {
+	private void setHTTPSConnector(OperatorContext operatorContext, Server server, int httpsPort, String host)
+			throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException {
 		SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
-		//Key Store is required
-		String keyStorePath = operatorContext.getParameterValues(SSL_KEYSTORE_PARAM).get(0);
-		File keyStorePathFile = new File(keyStorePath);
-		if (!keyStorePathFile.isAbsolute())
-			keyStorePathFile = new File(operatorContext.getPE().getApplicationDirectory(), keyStorePath);
-		String keyStorePathToLoad = keyStorePathFile.getAbsolutePath();
-		System.out.println("keyStorePathToLoad=" + keyStorePathToLoad);
-		sslContextFactory.setKeyStorePath(keyStorePathToLoad);
-		//the key store password is optional
-		if (operatorContext.getParameterNames().contains(SSL_KEYSTORE_PASSWORD_PARAM)) {
-			String keyStorePassword = operatorContext.getParameterValues(SSL_KEYSTORE_PASSWORD_PARAM).get(0);
-			System.out.println("keyStorePassword=****");
-			sslContextFactory.setKeyStorePassword(Functions.obfuscate(keyStorePassword));
-		}
-		//Key password is required
-		String keyPassword = operatorContext.getParameterValues(SSL_KEY_PASSWORD_PARAM).get(0);
-		System.out.println("keyPassword=****");
-		sslContextFactory.setKeyManagerPassword(Functions.obfuscate(keyPassword));
-		//Key alias
-		String alias = operatorContext.getParameterValues(SSL_CERT_ALIAS_PARAM).get(0);
-		sslContextFactory.setCertAlias(alias);
-		//Trust Store & password
-		if (operatorContext.getParameterNames().contains(SSL_TRUSTSTORE_PARAM)) {
-			String trustStorePath = operatorContext.getParameterValues(SSL_TRUSTSTORE_PARAM).get(0);
-			File trustStorePathFile = new File(trustStorePath);
-			if (!trustStorePathFile.isAbsolute())
-				trustStorePathFile = new File(operatorContext.getPE().getApplicationDirectory(), trustStorePath);
-			String trustStorePathToLoad = trustStorePathFile.getAbsolutePath();
-			System.out.println("trustStorePathToLoad=" + trustStorePathToLoad);
-			sslContextFactory.setTrustStorePath(trustStorePathToLoad);
-			sslContextFactory.setNeedClientAuth(true);
-			if (operatorContext.getParameterNames().contains(SSL_TRUSTSTORE_PASSWORD_PARAM)) {
-				String trustStorePassword = operatorContext.getParameterValues(SSL_TRUSTSTORE_PASSWORD_PARAM).get(0);
-				System.out.println("trustStorePassword=****");
-				sslContextFactory.setTrustStorePassword(Functions.obfuscate(trustStorePassword));
+
+		//ssl configuration with legacy parameters
+		if (operatorContext.getParameterNames().contains(SSL_CERT_ALIAS_PARAM)) {
+			
+			//Key Store is required
+			String keyStorePath = operatorContext.getParameterValues(SSL_KEYSTORE_PARAM).get(0);
+			File keyStorePathFile = new File(keyStorePath);
+			if (!keyStorePathFile.isAbsolute())
+				keyStorePathFile = new File(operatorContext.getPE().getApplicationDirectory(), keyStorePath);
+			String keyStorePathToLoad = keyStorePathFile.getAbsolutePath();
+			System.out.println("keyStorePathToLoad=" + keyStorePathToLoad);
+			sslContextFactory.setKeyStorePath(keyStorePathToLoad);
+			
+			//the key store password is optional
+			if (operatorContext.getParameterNames().contains(SSL_KEYSTORE_PASSWORD_PARAM)) {
+				String keyStorePassword = operatorContext.getParameterValues(SSL_KEYSTORE_PASSWORD_PARAM).get(0);
+				System.out.println("keyStorePassword=****");
+				sslContextFactory.setKeyStorePassword(Functions.obfuscate(keyStorePassword));
+			}
+			
+			//Key password is required
+			String keyPassword = operatorContext.getParameterValues(SSL_KEY_PASSWORD_PARAM).get(0);
+			System.out.println("keyPassword=****");
+			sslContextFactory.setKeyManagerPassword(Functions.obfuscate(keyPassword));
+			
+			//Key alias
+			String alias = operatorContext.getParameterValues(SSL_CERT_ALIAS_PARAM).get(0);
+			sslContextFactory.setCertAlias(alias);
+			
+			//Trust Store & password if necessary
+			if (operatorContext.getParameterNames().contains(SSL_TRUSTSTORE_PARAM)) {
+				String trustStorePath = operatorContext.getParameterValues(SSL_TRUSTSTORE_PARAM).get(0);
+				File trustStorePathFile = new File(trustStorePath);
+				if (!trustStorePathFile.isAbsolute())
+					trustStorePathFile = new File(operatorContext.getPE().getApplicationDirectory(), trustStorePath);
+				String trustStorePathToLoad = trustStorePathFile.getAbsolutePath();
+				System.out.println("trustStorePathToLoad=" + trustStorePathToLoad);
+				sslContextFactory.setTrustStorePath(trustStorePathToLoad);
+				sslContextFactory.setNeedClientAuth(true);
+				if (operatorContext.getParameterNames().contains(SSL_TRUSTSTORE_PASSWORD_PARAM)) {
+					String trustStorePassword = operatorContext.getParameterValues(SSL_TRUSTSTORE_PASSWORD_PARAM).get(0);
+					System.out.println("trustStorePassword=****");
+					sslContextFactory.setTrustStorePassword(Functions.obfuscate(trustStorePassword));
+				}
+			}
+
+		//Configuration with application configuration
+		} else {
+			
+			ProcessingElement pe = operatorContext.getPE();
+			String certAppConfigName = operatorContext.getParameterValues(SSL_APP_CONFIG_NAME_PARAM).get(0);
+			Map<String, String> certProps = pe.getApplicationConfiguration(certAppConfigName);
+			System.out.println("streams-certs len: " + new Integer(certProps.size()).toString() + " keyset: " + certProps.keySet().toString());
+			if (certProps.isEmpty())
+				throw new IllegalArgumentException("app config with name " + certAppConfigName + " is required");
+			
+			//Key Store and password is required
+			if ( ! certProps.containsKey("server.jks"))
+				throw new IllegalArgumentException("Property server.jks is required in app config " + certAppConfigName);
+			String keyB64Str = certProps.get("server.jks");
+			Decoder decoder = Base64.getDecoder();
+			byte[] keyBytes = decoder.decode(keyB64Str);
+			InputStream keyStream = new ByteArrayInputStream(keyBytes);
+
+			if ( ! certProps.containsKey("server.pass"))
+				throw new IllegalArgumentException("Property server.pass is required in app config " + certAppConfigName);
+			String password = certProps.get("server.pass");
+
+			System.out.println("Load key store and passwd from app config " + certAppConfigName);
+			KeyStore keyStore = KeyStore.getInstance("JKS");
+			keyStore.load(keyStream, password.toCharArray());
+			sslContextFactory.setKeyStore(keyStore);
+			sslContextFactory.setKeyManagerPassword(password);
+			
+			//Set optionally trust material
+			if (certProps.containsKey("cacerts.jks")) {
+				String trustB64Str = certProps.get("cacerts.jks");
+				byte[] trustBytes = decoder.decode(trustB64Str);
+				InputStream trustStream = new ByteArrayInputStream(trustBytes);
+				
+				System.out.println("Load trust store and passwd from app config " + certAppConfigName);
+				KeyStore trustStore = KeyStore.getInstance("JKS");
+				trustStore.load(trustStream, password.toCharArray());
+				sslContextFactory.setTrustStore(trustStore);
+				sslContextFactory.setNeedClientAuth(true);
 			}
 		}
-		
+
 		sslContextFactory.setRenegotiationAllowed(false);
 		sslContextFactory.setIncludeProtocols("TLSv1.2");
 		String[] specs = {"^.*_(MD5|SHA|SHA1)$","^TLS_RSA_.*$","^.*_NULL_.*$","^.*_anon_.*$"};
diff --git a/com.ibm.streamsx.inetserver/impl/java/src/com/ibm/streamsx/inet/rest/ops/ServletOperator.java b/com.ibm.streamsx.inetserver/impl/java/src/com/ibm/streamsx/inet/rest/ops/ServletOperator.java
@@ -100,6 +100,20 @@ public void setTrustStore(String ks) {}
 	@Parameter(optional = true, description = "Password to the trust store.")
 	public void setTrustStorePassword(String ksp) {}
 
+	@Parameter(optional = true, description = "The operator can get the SSL server key/certificate and the client trust "
+			+ "material from Streams application configuration with this name. If this parameter is present, `"
+			+ ServletEngine.SSL_CERT_ALIAS_PARAM +"`, `" + ServletEngine.SSL_KEYSTORE_PARAM + "`, `" + ServletEngine.SSL_KEY_PASSWORD_PARAM
+			+ "`, `" + ServletEngine.SSL_TRUSTSTORE_PARAM + "` and `" + ServletEngine.SSL_TRUSTSTORE_PASSWORD_PARAM + "` are "
+			+ "not allowed. The application configuration must contain the folowing properties:\\n"
+			+ " \\t\\t* server.jks - Base 64 encoded representation of the Java key store with one key-certificate pair to be used as server key and certitficate.\\n"
+			+ " \\t\\t* server.pass -Password for server.jks (and its key) and cacerts.jks.\\n"
+			+ "The app configuration may contain property:\\n"
+			+ " \\t\\t* cacerts.jks - Base 64 encoded representation of the Java trust store with the client trust material.\\n\\n"
+			+ " If the property cacerts.jks is present, the operator requests client certificates."
+			+ " To create the Streams application configuration you may use a comand like:\\n"
+			+ "    streamtool mkappconfig --description 'server cert and trust store' --property \\\"server.jks=$(base64 --wrap=0 etc/keystore.jks)\\\" --property \\\"server.pass=password\\\" --property \\\"cacerts.jks=$(base64 --wrap=0 etc/cacerts.jks)\\\" streams-certs")
+	public void setSslAppConfigName(String ac) {}
+
 	// Creates a metric that the ServletEngine will fill in.
 	private Metric serverPort;
 	@CustomMetric(description="Jetty (HTTP/HTTPS) server port if the operator hosts the server, 0 otherwise", kind=Kind.GAUGE)
@@ -113,7 +127,7 @@ public void setTrustStorePassword(String ksp) {}
 	public Metric getHttps() { return https; }
 	
 	@ContextCheck
-	public static void checkContextParameters(OperatorContextChecker checker) {	
+	public static void checkContextParameters(OperatorContextChecker checker) {
 
 		checker.checkDependentParameters(ServletEngine.SSL_CERT_ALIAS_PARAM, ServletEngine.SSL_KEYSTORE_PARAM, ServletEngine.SSL_KEY_PASSWORD_PARAM);
 		checker.checkDependentParameters(ServletEngine.SSL_KEY_PASSWORD_PARAM, ServletEngine.SSL_CERT_ALIAS_PARAM);
@@ -122,6 +136,10 @@ public static void checkContextParameters(OperatorContextChecker checker) {
 		checker.checkDependentParameters(ServletEngine.SSL_TRUSTSTORE_PARAM, ServletEngine.SSL_CERT_ALIAS_PARAM);
 		checker.checkDependentParameters(ServletEngine.SSL_TRUSTSTORE_PASSWORD_PARAM, ServletEngine.SSL_TRUSTSTORE_PARAM);
 	
+		checker.checkExcludedParameters(ServletEngine.SSL_APP_CONFIG_NAME_PARAM, 
+				ServletEngine.SSL_CERT_ALIAS_PARAM, ServletEngine.SSL_KEYSTORE_PARAM, ServletEngine.SSL_KEY_PASSWORD_PARAM,
+				ServletEngine.SSL_TRUSTSTORE_PARAM, ServletEngine.SSL_TRUSTSTORE_PASSWORD_PARAM);
+		
 		checker.checkDependentParameters(ServletEngine.CONTEXT_RESOURCE_BASE_PARAM, ServletEngine.CONTEXT_PARAM);
 	}
 	
diff --git a/com.ibm.streamsx.inetserver/info.xml b/com.ibm.streamsx.inetserver/info.xml
@@ -14,7 +14,7 @@ This toolkit separates its functionality into a number of namespaces:
 * [namespace:com.ibm.streamsx.inet.wsserver|com.ibm.streamsx.inet.wsserver]: Operators that embed a (jetty) WebSocket server to expose streaming data as WebSocket messages.
 
     </description>
-    <version>4.2.0</version>
+    <version>4.3.0</version>
     <requiredProductVersion>4.0.1.0</requiredProductVersion>
   </identity>
   <dependencies/>
diff --git a/samples/HTTPTupleInjectAndView/com.ibm.streamsx.inet.sample.rest/SimpleInject.spl b/samples/HTTPTupleInjectAndView/com.ibm.streamsx.inet.sample.rest/SimpleInject.spl
@@ -35,6 +35,18 @@ use com.ibm.streamsx.inet.rest::*;
  *     output/bin/standalone
  *     Point browser to [https://localhost:1443] and accept the SEC_ERROR_UNKNOWN_ISSUER error and add an exception.
  * 
+ * This sample is enabled to use a secure connection by default. The server key and self signed certificate are stored 
+ * in key store etc/keystore.jks.
+ * 
+ * To enable client authentication with client certificate add parameter `trustStore: "etc/cacerts.jks` to all three 
+ * rest operators and import the client key/certificate file `etc/client.pfx` into your browser certificate manager.
+ * 
+ * Alternatively the import of the key- and trust-material from an Streams application configuration is supported.
+ * Add parameter `sslAppConfigName: "streams-certs"` to all tree rest operators and remove `certificateAlias`, 
+ * `keyStore`, `keyPassword` and `trustStore`.
+ * Generate the application configuration:
+ * 		streamtool mkappconfig --description 'server cert and trust store' --property "server.jks=$(base64 --wrap=0 etc/keystore.jks)" --property "server.pass=changeit" --property "cacerts.jks=$(base64 --wrap=0 etc/cacerts.jks)" streams-certs
+ * 
  */
 public composite SimpleInject {
 	
@@ -45,7 +57,8 @@ public composite SimpleInject {
 				certificateAlias: "mykey";
 				keyStore: "etc/keystore.jks";
 				keyPassword: "changeit";
-				//context: "sensors";
+				//trustStore: "etc/cacerts.jks";
+				//sslAppConfigName: "streams-certs";
 			config
 				// Ensure the operators are in a single PE to have a single web-server
 				placement: partitionColocation("jetty1443");
@@ -59,6 +72,8 @@ public composite SimpleInject {
 				certificateAlias: "mykey";
 				keyStore: "etc/keystore.jks";
 				keyPassword: "changeit";
+				//trustStore: "etc/cacerts.jks";
+				//sslAppConfigName: "streams-certs";
 				context: "state";
 				contextResourceBase: "opt/statetest";
 			config
@@ -72,6 +87,8 @@ public composite SimpleInject {
 				certificateAlias: "mykey";
 				keyStore: "etc/keystore.jks";
 				keyPassword: "changeit";
+				//trustStore: "etc/cacerts.jks";
+				//sslAppConfigName: "streams-certs";
 				context: "wct";
 				contextResourceBase: "opt/wctest";
 			config
diff --git a/samples/HTTPTupleInjectAndView/etc/cacerts.jks b/samples/HTTPTupleInjectAndView/etc/cacerts.jks
diff --git a/samples/HTTPTupleInjectAndView/etc/client.pfx b/samples/HTTPTupleInjectAndView/etc/client.pfx

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@`
`32`	`32`	* * `keyStore`
`33`	`33`	* * `keyStorePassword`
`34`	`34`	* * `keyPassword`
	`35`	+ * * `sslAppConfigName`
`35`	`36`	`*`
`36`	`37`	`* Note: Jetty server instances will fail to start if they are placed on a single host and use the same port number.`
`37`	`38`	`*`