Merge pull request #4468 from esl/MIM-2360_deprecate_fast_tls

MIM-2360 deprecate fast tls for C2S

Author: chrzaszcz

big_tests/tests/login_SUITE.erl

@@ -38,17 +38,25 @@ all() ->
      {group, login_scram_store_plain},
      {group, login_specific_scram},
      {group, login_scram_tls},
+     {group, fast_tls},
      {group, messages},
      {group, access}
     ].
 
 groups() ->
     [{login, [parallel], all_tests()},
      {login_digest, [sequence], digest_tests()},
+     %% The SCRAM tests below have SCRAM_PLUS tests skipped
+     %% because SCRAM_PLUS is not implemented for just_tls yet
      {login_scram, [parallel], scram_tests()},
      {login_scram_store_plain, [parallel], scram_tests()},
      {login_scram_tls, [parallel], scram_tests()},
      {login_specific_scram, [sequence], configure_specific_scram_test()},
+     %% rerun SCRAM tests with fast_tls including SCRAM_PLUS tests
+     {fast_tls, [{group, login_scram},
+                 {group, login_scram_store_plain},
+                 {group, login_scram_tls},
+                 {group, login_specific_scram}]},
      {messages, [sequence], [messages_story]},
      {access, [], access_tests()}].
 
@@ -119,6 +127,8 @@ end_per_suite(Config) ->
     escalus_fresh:clean(),
     escalus:end_per_suite(Config).
 
+init_per_group(fast_tls, ConfigIn) ->
+    configure_c2s_listener_with_fast_tls(ConfigIn);
 init_per_group(login_digest = GroupName, ConfigIn) ->
     Config = backup_and_set_options(GroupName, ConfigIn),
     case mongoose_helper:supports_sasl_module(cyrsasl_digest) of
@@ -175,6 +185,8 @@ auth_opts(login_scram_store_plain) ->
 auth_opts(_GroupName) ->
     mongoose_helper:auth_opts_with_password_format(scram).
 
+end_per_group(fast_tls, Config) ->
+    restore_c2s(Config);
 end_per_group(login_digest, Config) ->
     mongoose_helper:restore_config(Config),
     escalus:delete_users(Config, escalus:get_users([alice, bob]));
@@ -258,12 +270,16 @@ log_one(Config) ->
         end).
 
 log_one_scram_plus(Config) ->
-    escalus:fresh_story(Config, [{neustradamus, 1}], fun(Neustradamus) ->
-
-        escalus_client:send(Neustradamus, escalus_stanza:chat_to(Neustradamus, <<"Hi!">>)),
-        escalus:assert(is_chat_message, [<<"Hi!">>], escalus_client:wait_for_stanza(Neustradamus))
-
-        end).
+    %% SCRAM PLUS tests are to be run for fast_tls only
+    case proplists:get_value(fast_tls, Config) of
+        true ->
+            escalus:fresh_story(Config, [{neustradamus, 1}], fun(Neustradamus) ->
+                escalus_client:send(Neustradamus, escalus_stanza:chat_to(Neustradamus, <<"Hi!">>)),
+                escalus:assert(is_chat_message, [<<"Hi!">>], escalus_client:wait_for_stanza(Neustradamus))
+            end);
+        _ ->
+            {skip, test_valid_only_for_fast_tls}
+    end.
 
 log_one_digest(Config) ->
     log_one([{escalus_auth_method, <<"DIGEST-MD5">>} | Config]).
@@ -427,10 +443,28 @@ configure_c2s_listener(Config) ->
     C2SPort = ct:get_config({hosts, mim, c2s_port}),
     [C2SListener = #{tls := TLSOpts}] =
         mongoose_helper:get_listeners(mim(), #{port => C2SPort, module => mongoose_c2s_listener}),
+    %% If in fast_tls group, retrieve fast_tls options from config, otherwise use the ones from node.
+    %% We do this, as in fast_tls group init we have already restarted c2s listener on node
+    %% and there is a race condition in retrieving them from node,
+    %% so it is better to take them from config
+    TLSOpts1 = proplists:get_value(tls_opts, Config, TLSOpts),
     %% replace starttls with tls
-    NewTLSOpts = TLSOpts#{mode := tls},
+    NewTLSOpts = TLSOpts1#{mode := tls},
     mongoose_helper:restart_listener(mim(), C2SListener#{tls := NewTLSOpts}),
-    [{c2s_listener, C2SListener} | Config].
+    [{c2s_listener, C2SListener#{tls := TLSOpts1}} | Config].
+
+configure_c2s_listener_with_fast_tls(Config) ->
+    C2SPort = ct:get_config({hosts, mim, c2s_port}),
+    [C2SListener = #{tls := _TLSOpts}] =
+        mongoose_helper:get_listeners(mim(), #{port => C2SPort, module => mongoose_c2s_listener}),
+    NewTLSOpts = #{module => fast_tls,mode => starttls,
+                   certfile => "priv/ssl/fake_server.pem",
+                   dhfile => "priv/ssl/fake_dh_server.pem",
+                   ciphers => "TLSv1.2:TLSv1.3",
+                   protocol_options => ["no_sslv2","no_sslv3","no_tlsv1","no_tlsv1_1"],
+                   verify_mode => none},
+    mongoose_helper:restart_listener(mim(), C2SListener#{tls := NewTLSOpts}),
+    [{c2s_listener, C2SListener}, {fast_tls, true}, {tls_opts, NewTLSOpts}| Config].
 
 create_tls_users(Config) ->
    Config1 = escalus:create_users(Config, escalus:get_users([alice, neustradamus])),
@@ -517,8 +551,14 @@ configure_and_fail_log_scram(Config, Sha, Mech) ->
     {expected_challenge, _, _} = fail_log_one([{escalus_auth_method, Mech} | Config]).
 
 configure_scram_plus_and_fail_log_scram(Config, Sha, Mech) ->
-    set_scram_sha(Config, Sha),
-    {expected_challenge, _, _} = fail_log_one_scram_plus([{escalus_auth_method, Mech} | Config]).
+    %% SCRAM PLUS tests are to be run for fast_tls only
+    case proplists:get_value(fast_tls, Config) of
+        true ->
+            set_scram_sha(Config, Sha),
+            {expected_challenge, _, _} = fail_log_one_scram_plus([{escalus_auth_method, Mech} | Config]);
+        _ ->
+            {skip, test_valid_only_for_fast_tls}
+    end.
 
 set_scram_sha(Config, Sha) ->
     NewAuthOpts = mongoose_helper:auth_opts_with_password_format({scram, [Sha]}),

doc/configuration/TLS-hardening.md

@@ -16,7 +16,9 @@ The former one is used primarily by MIM dependencies, while the latter is used o
 None of them is strictly better than the other.
 Below you may find a summary of the differences between them.
 
-* `fast_tls` is faster
+* `fast_tls` used to be faster, however with the progress of OTP TLS implementation
+and additional optimisations applied in MongooseIM this is no longer true.
+* `just_tls` may use slightly more processor time than `fast_tls`.
 * There are options that OTP TLS (a.k.a `just_tls` in the C2S listener configuration) supports exclusively:
     * Immediate connection drop when the client certificate is invalid
     * Certificate Revocation Lists
@@ -48,7 +50,7 @@ The remaining valid values are: `'tlsv1.1'`, `tlsv1`, `sslv3`.
 
 This setting affects the following MongooseIM components:
 
-* Raw XMPP over TCP connections, if a C2S listener is configured to use `just_tls`
+* Raw XMPP over TCP connections (C2S listener) in the default configuration uses `just_tls`
 * All outgoing connections (databases, AMQP, SIP etc.)
 * HTTP endpoints
 
@@ -60,7 +62,8 @@ By default, MongooseIM sets this option to `TLSv1.2:TLSv1.3` for each component.
 
 The list below enumerates all components that use Fast TLS and describes how to change this string.
 
-* `listen.c2s` - main user session abstraction + XMPP over TCP listener
+* `listen.c2s` - main user session abstraction + XMPP over TCP listener, when configured to use `fast_tls`
+    * Note that usage of `fast_tls` for C2S has been deprecated
     * Please consult the respective section in [Listener modules](../listeners/listen-c2s.md#listenc2stlsprotocol_options-only-for-fast_tls).
 * `listen.s2s` - incoming S2S connections (XMPP Federation)
     * Please consult the respective section in [Listener modules](../listeners/listen-s2s.md#tls-options-for-s2s).

doc/configuration/configuration-files.md

@@ -63,8 +63,8 @@ By default only the following applications can be found there:
 
 TLS is configured in one of two ways: some modules need a private key and certificate (chain) in __separate__ files, while others need both in a __single__ file. This is because recent additions use OTP's `ssl` library, while older modules use `p1_tls`, respectively.
 
-* Client-to-server connections need both in the __same__ `.pem` file
 * Server-to-server connections need both in the __same__ `.pem` file
+* Client-to-server connections need them in __separate__ files, unless `fast_tls` is used
 * BOSH, WebSockets and REST APIs need them in __separate__ files
 
 In order to create private key & certificate bundle, you may simply concatenate them.

doc/listeners/listen-c2s.md

@@ -89,10 +89,12 @@ This option determines how clients are supposed to set up the TLS encryption:
 
 ### `listen.c2s.tls.module`
 * **Syntax:** string, one of `"just_tls"`, `"fast_tls"`
-* **Default:** `"fast_tls"`
+* **Default:** `"just_tls"`
 * **Example:** `tls.module = "just_tls"`
 
-By default, the TLS library used for C2S connections is `fast_tls`, which uses OpenSSL-based NIFs. It is possible to change it to `just_tls` - Erlang TLS implementation provided by OTP. Some TLS-related options described here have different formats for these two libraries.
+By default, the TLS library used for C2S connections is `just_tls` - Erlang TLS implementation provided by OTP.
+Usage of `fast_tls`, which uses OpenSSL-based NIFs for C2S is deprecated, however it is still possible to use this option.
+Some TLS-related options described here have different formats for these two libraries.
 
 ### `listen.c2s.tls.verify_mode`
 * **Syntax:** string, one of `"peer"`, `"selfsigned_peer"`, `"none"`
@@ -162,6 +164,16 @@ Password to the X509 PEM file with the private key.
 * **Default:** `true`
 * **Example:** `tls.disconnect_on_failure = false`
 
+This option specifies what happens when client certificate is verified during TLS handshake.
+It therefore only applies when client certificate verification is enabled, that is `tls.verify_mode` is set to `"peer"` or `"selfsigned_peer"`.
+
+When set to `true`, client verification is performed during TLS handshake and in case of error the connection is aborted.
+Additionally empty client certificate is treated as an error.
+
+When set to `false`, TLS handshake will succeed even if there were errors in client certificate verification.
+This allows to use other methods of authentication (like SASL) later as part of XMPP stream.
+The above behaviour is the same as default `fast_tls` behaviour (not aborting TLS connection on verification errors).
+
 ### `listen.c2s.tls.versions` - only for `just_tls`
 * **Syntax:** array of strings
 * **Default:** not set, all supported versions are accepted

doc/migrations/6.3.1_6.x.y.md

@@ -0,0 +1,38 @@
+## Change of the default TLS library used for C2S connections
+
+As of this release, usage of `fast_tls` for Client to Server connections (C2S) has been deprecated.
+`fast_tls` will be removed in a future release.
+
+From now on the default TLS library for C2S is `just_tls`, which uses TLS implementation from Erlang OTP.
+In our load tests `just_tls` is as performant as `fast_tls` and also has better standards compliance.
+
+This deprecation affects only C2S, `fast_tls` remains as a TLS implementation for S2S.
+
+To continue using `fast_tls` for C2S in existing deployment after upgrade, make sure the
+option `tls.module` is set to `fast_tls` in `listen.c2s` section of your MongooseIM config.
+
+### Channel binding for TLS
+
+Note that `just_tls` currently does not implement `channel binding` for TLS, which is required for SCRAM_PLUS
+authentication methods. If you depend on using SCRAM_PLUS for authentication, you need to use `fast_tls`.
+We do plan to implement `channel binding` for `just_tls` (only for TLS 1.3) in the future.
+
+### TLS handshake
+
+There is a difference between `fast_tls` and `just_tls` in client authentication behaviour during TLS handshake.
+
+`fast_tls` doesn't verify client certificate during TLS handshake and relies on other mechanisms, like SASL,
+to authenticate client. It may involve client certificate, but is executed after TLS handshake succeeded,
+and in case of invalid certificate will result in an error reported in message stream.
+
+`just_tls` by default verifies client certificate during TLS handshake
+and aborts connection when client certificate is invalid. This is realised by the default settings in
+`just_tls` of `verify_mode` set to `peer` and `disconnect_on_failure` set to `true`.
+
+If you want to have the same behaviour for `just_tls` as it was in `fast_tls` regarding TLS handshake,
+set `tls.disconnect_on_failure` to `false`. This is required for example when using SASL for client authentication.
+
+It is also possible to completely disable client certificate verification during TLS
+handshake in `just_tls` by setting `tls.verify_mode` to `none`.
+
+For more information regarding configuration of TLS for C2S see [Listener modules](../listeners/listen-c2s/#tls-options-for-c2s)

rel/mim1.vars-toml.config

@@ -70,6 +70,7 @@
   max_stanza_size = 65536
   tls.certfile = \"priv/ssl/fake_server.pem\"
   tls.cacertfile = \"priv/ssl/cacert.pem\"
+  tls.disconnect_on_failure = false
   tls.mode = \"tls\""}.
 {listen_service,
   "[[listen.service]]

src/config/mongoose_config_spec.erl

@@ -698,7 +698,7 @@ tls(c2s, common) ->
                                                validate = {enum, [fast_tls, just_tls]}},
                        <<"mode">> => #option{type = atom,
                                              validate = {enum, [tls, starttls, starttls_required]}}},
-             defaults = #{<<"module">> => fast_tls,
+             defaults = #{<<"module">> => just_tls,
                           <<"mode">> => starttls},
              process = fun ?MODULE:process_c2s_tls/1};
 tls(c2s, just_tls) ->

test/common/config_parser_helper.erl

@@ -156,7 +156,7 @@ options("mongooseim-pgsql") ->
                 access => c2s,
                 shaper => c2s_shaper,
                 max_stanza_size => 65536,
-                tls => #{certfile => "priv/dc1.pem", dhfile => "priv/dh.pem",
+                tls => #{certfile => "priv/cert.pem", keyfile => "priv/dc1.pem",
                          cacertfile => "priv/ca.pem"}
                }),
        config([listen, c2s],
@@ -1166,7 +1166,7 @@ default_config([listen, c2s]) ->
                                      access => all,
                                      shaper => none};
 default_config([listen, c2s, tls]) ->
-    default_c2s_tls(fast_tls);
+    default_c2s_tls(just_tls);
 default_config([listen, s2s] = P) ->
     (common_xmpp_listener_config())#{module => ejabberd_s2s_in,
                                      shaper => none,

test/config_parser_SUITE.erl

@@ -508,7 +508,8 @@ listen_c2s(_Config) ->
 
 listen_c2s_fast_tls(_Config) ->
     T = fun(Opts) -> listen_raw(c2s, #{<<"port">> => 5222,
-                                       <<"tls">> => Opts}) end,
+                                       <<"tls">> => maps:merge(
+                                           #{<<"module">> => <<"fast_tls">>}, Opts)}) end,
     P = [listen, 1, tls],
     M = tls_ca_raw(),
     ?cfg(P, maps:merge(default_c2s_tls(fast_tls), tls_ca()), T(M)),
@@ -525,7 +526,7 @@ listen_c2s_fast_tls(_Config) ->
 
 listen_c2s_just_tls(_Config) ->
     T = fun(Opts) -> listen_raw(c2s, #{<<"port">> => 5222,
-                                       <<"tls">> => Opts#{<<"module">> => <<"just_tls">>}}) end,
+                                       <<"tls">> => Opts}) end,
     P = [listen, 1, tls],
     M = tls_ca_raw(),
     ?cfg(P, maps:merge(default_c2s_tls(just_tls), tls_ca()), T(M)),
@@ -569,7 +570,6 @@ listen_s2s_cacertfile_verify(_Config) ->
     ?err([#{reason := missing_cacertfile}], T(<<"required">>, #{})),
     ?err([#{reason := missing_cacertfile}], T(<<"required_trusted">>, #{})),
     %% setting `verify_mode` to `none` turns off `cacertfile` validation
-    VerifyModeNone = #{verify_mode => none},
     VerifyModeNoneRaw = #{<<"verify_mode">> => <<"none">>},
     ConfigWithVerifyModeNone = maps:merge(default_config([listen, s2s, tls]),
                                           #{verify_mode => none}),

test/config_parser_SUITE_data/mongooseim-pgsql.toml

@@ -84,8 +84,8 @@
   shaper = "c2s_shaper"
   max_stanza_size = 65536
   tls.mode = "starttls"
-  tls.certfile = "priv/dc1.pem"
-  tls.dhfile = "priv/dh.pem"
+  tls.certfile = "priv/cert.pem"
+  tls.keyfile = "priv/dc1.pem"
   tls.cacertfile = "priv/ca.pem"
 
 [[listen.c2s]]