Merge pull request #4452 from esl/feature/listeners

Feature/listeners

Author: jacekwegr

big_tests/Makefile

@@ -24,6 +24,9 @@ else
 TLS_DIST_OPTS :=
 endif
 
+REMSH_OPTS := -setcookie mongooseim \
+               $(TLS_DIST_OPTS)
+
 COMMON_OPTS := -sname test -setcookie mongooseim -hidden \
                $(TLS_DIST_OPTS) \
                -env REPO_DIR "$(ABS_REPO_DIR)" \
@@ -72,6 +75,11 @@ test: $(PREPARE)
 	$(RUN) erl -noinput $(COMMON_OPTS) $(ADD_OPTS) \
 			-s run_common_test main test=full spec=$(TESTSPEC)
 
+# Open remote shell to the test node
+remsh:
+	erl $(REMSH_OPTS) $(ADD_OPTS) \
+			-remsh "test@$(shell hostname -s)" -sname remshnode
+
 prepare: mim_ct_rest compile vm.dist.args
 	erlc -I_build/default/lib/exml/include \
 		 run_common_test.erl

big_tests/default.spec

@@ -99,7 +99,6 @@
 {suites, "tests", xep_0352_csi_SUITE}.
 {suites, "tests", domain_isolation_SUITE}.
 {suites, "tests", domain_removal_SUITE}.
-{suites, "tests", tcp_listener_SUITE}.
 {suites, "tests", cets_disco_SUITE}.
 {suites, "tests", start_node_id_SUITE}.
 {suites, "tests", tr_util_SUITE}.

big_tests/dynamic_domains.config

@@ -23,9 +23,10 @@
                  {cowboy_port, 5280},
                  {cowboy_secure_port, 5285},
                  {http_api_client_endpoint_port, 8089},
-                 {service_port, 8888},
-                 {kicking_service_port, 8666},
-                 {hidden_service_port, 8189},
+                 {component_port, 8888},
+                 {kicking_component_port, 8666},
+                 {hidden_component_port, 8189},
+                 {tls_component_port, 8190},
                  {gd_endpoint_port, 5555},
                  {http_notifications_port, 8000}]},
          {mim2, [{node, mongooseim2@localhost},
@@ -38,7 +39,7 @@
                  {c2s_tls_port, 5233},
                  {metrics_rest_port, 5289},
                  {gd_endpoint_port, 6666},
-                 {service_port, 8899}]},
+                 {component_port, 8899}]},
          {mim3, [{node, mongooseim3@localhost},
                  {domain, <<"domain.example.com">>},
                  {host_type, <<"test type">>},
@@ -110,6 +111,11 @@
         {server, <<"domain.example.com">>},
         {host, <<"localhost">>},
         {password, <<"nicniema">>}]},
+    {astrid, [
+        {username, <<"astrid">>},
+        {server, <<"sogndal">>},
+        {host, <<"localhost">>},
+        {password, <<"doctor">>}]},
     {geralt, [
         {username, <<"geralt">>},
         {server, <<"domain.example.com">>},
@@ -162,8 +168,7 @@
         {server, <<"domain.example.com">>},
         {host, <<"localhost">>},
         {password, <<"scramshaplus">>},
-        {starttls, required},
-        {tls_module, fast_tls}]},
+        {starttls, required}]},
     {hacker, [
         {username, <<"hacker">>},
         {server, <<"domain.example.com">>},

big_tests/dynamic_domains.spec

@@ -99,7 +99,6 @@
 {suites, "tests", xep_0352_csi_SUITE}.
 {suites, "tests", domain_removal_SUITE}.
 {suites, "tests", local_iq_SUITE}.
-{suites, "tests", tcp_listener_SUITE}.
 {suites, "tests", cets_disco_SUITE}.
 {suites, "tests", start_node_id_SUITE}.
 {suites, "tests", tr_util_SUITE}.
@@ -108,6 +107,7 @@
 %% to minimise impact on other tests
 {suites, "tests", auth_methods_for_c2s_SUITE}.
 {suites, "tests", cluster_commands_SUITE}.
+{suites, "tests", component_SUITE}.
 {suites, "tests", dynamic_domains_SUITE}.
 {suites, "tests", graphql_server_SUITE}.
 {suites, "tests", last_SUITE}.

big_tests/rebar.config

@@ -14,11 +14,7 @@
         {proper, "1.4.0"},
         {gun, "2.1.0"},
         {fusco, "0.1.1"},
-        {escalus, "4.3.0"},
-        %% We need to override `escalus` scram version, because 4.4 removes `fast_tls`, which is not
-        %% supported in master yet. When we can upgrade escalus, we can remove the explicit
-        %% `fast_scram` declaration from here.
-        {fast_scram, "0.7.0"},
+        {escalus, "4.4.1"},
         {cowboy, "2.12.0"},
         {csv, "3.0.3", {pkg, csve}},
         {amqp_client, "4.0.3"},

big_tests/rebar.lock

@@ -10,12 +10,12 @@
  {<<"ct_groups_summary_hook">>,
   {pkg,<<"ct_groups_summary_hook">>,<<"0.1.1">>},
   0},
- {<<"escalus">>,{pkg,<<"escalus">>,<<"4.3.0">>},0},
+ {<<"escalus">>,{pkg,<<"escalus">>,<<"4.4.1">>},0},
  {<<"esip">>,{pkg,<<"esip">>,<<"1.0.56">>},0},
  {<<"exml">>,{pkg,<<"hexml">>,<<"4.1.1">>},0},
  {<<"fast_pbkdf2">>,{pkg,<<"fast_pbkdf2">>,<<"2.0.0">>},1},
  {<<"fast_scram">>,{pkg,<<"fast_scram">>,<<"0.7.0">>},0},
- {<<"fast_tls">>,{pkg,<<"fast_tls">>,<<"1.1.21">>},1},
+ {<<"fast_tls">>,{pkg,<<"fast_tls">>,<<"1.1.22">>},1},
  {<<"fusco">>,{pkg,<<"fusco">>,<<"0.1.1">>},0},
  {<<"gun">>,{pkg,<<"gun">>,<<"2.1.0">>},0},
  {<<"jid">>,{pkg,<<"mongoose_jid">>,<<"2.2.0">>},0},
@@ -42,12 +42,12 @@
  {<<"credentials_obfuscation">>, <<"34E18B126B3AEFD6E8143776FBE1CECEEA6792307C99AC5EE8687911F048CFD7">>},
  {<<"csv">>, <<"69E7D9B3FDC72016644368762C6A3E6CBFEB85BCCADBF1BD99AB6C827E360E04">>},
  {<<"ct_groups_summary_hook">>, <<"21B94902B6CF2D345F4D790D34B49654E71CB8E570DCCC9C1C3616DAE720A7AA">>},
- {<<"escalus">>, <<"80301D57F4C863FA95F09AE148524508F18D7389B5070E2A5FFB7A38B51527C5">>},
+ {<<"escalus">>, <<"B32CC182F2246BA7B8B2853FF3A5A28F0F822540965053A7D3EF2894E0B17B5A">>},
  {<<"esip">>, <<"63C0FDC667BE751714E1E5C14621A9334F21B60AC1BB68BE889454CA9CA021B7">>},
  {<<"exml">>, <<"D46272177F3B779D75058005C744997034823AB3A237F9B30B56A583B063FDE3">>},
  {<<"fast_pbkdf2">>, <<"72CDEE3C10C6B9B40E31194DE946A883CEEF6CF1F37D7FC9FD1A9D87502723F5">>},
  {<<"fast_scram">>, <<"FAD4DD185E0DEFA34B0E30654AB0CC4048E9324BCF65BB261E0A86BF3C604F9F">>},
- {<<"fast_tls">>, <<"65D7D547A09EEFB37A1C0D04D8601FAC4F3E6E2C1EDE859A7787081670F9648D">>},
+ {<<"fast_tls">>, <<"44356B256AFAD4399C2FC5059A3066669DAFD8BD4E4E796C9C1CF8910DDD265E">>},
  {<<"fusco">>, <<"3DD6A90151DFEF30EA1937CC44E9A59177C0094918388D9BCAA2F2DC5E2AE4AA">>},
  {<<"gun">>, <<"B4E4CBBF3026D21981C447E9E7CA856766046EFF693720BA43114D7F5DE36E87">>},
  {<<"jid">>, <<"3761535829A0DBA5A6BBDF2EE6EA7BCD9A8F1ADA201EDB4D79E1E2B47D5FD3D6">>},
@@ -73,12 +73,12 @@
  {<<"credentials_obfuscation">>, <<"738ACE0ED5545D2710D3F7383906FC6F6B582D019036E5269C4DBD85DBCED566">>},
  {<<"csv">>, <<"741D1A55AABADAA3E0FE13051050101A73E90C4570B9F9403A939D9546813521">>},
  {<<"ct_groups_summary_hook">>, <<"997CDE48FEB0C398989E4091A341D4FFF43CFA06CDB8FB2E80CC4A0E0362691C">>},
- {<<"escalus">>, <<"8C4D39677D55C48B04EDE4EAC0FD0AD2A574BEDE7180761E6E3D3877900C7708">>},
+ {<<"escalus">>, <<"3389CF597E87F6C48314C4445B4B25A2678983B11C87A5C2308191A3BDA2F85A">>},
  {<<"esip">>, <<"9EF3660CEF93B623F7368DCD5C79F4E704358631909E6DD464E335378815DA1F">>},
  {<<"exml">>, <<"86C154396A9B21EEB0EDB9636D0E2D493FE0F59EAAF40FCFE795210F31119D8F">>},
  {<<"fast_pbkdf2">>, <<"74159FD09FB8BF5E97D25137C6C83C28E2CF7E97D7C127D83310DFD0904BD732">>},
  {<<"fast_scram">>, <<"39CB1D4C91C99E637E000A3864D274D8A9CB43D86EEC2495FE9546798DBFA015">>},
- {<<"fast_tls">>, <<"131542913937025E48CD80AA81F00359686D5501B75621E72026A87B5229505B">>},
+ {<<"fast_tls">>, <<"E65779AEFB7AB15C4755230FEF8077E687D20CC5A3984A5974F9F657E8E2485B">>},
  {<<"fusco">>, <<"6343551BD1E824F2A6CA85E1158C5B37C320FD449FBFEC7450A73F192AAF9022">>},
  {<<"gun">>, <<"52FC7FC246BFC3B00E01AEA1C2854C70A366348574AB50C57DFE796D24A0101D">>},
  {<<"jid">>, <<"0F2C69E4C142E21E7D87E6D70AC62ECF6EB00E97C86CF8CBF6BAD4B3BE8B0545">>},

big_tests/test.config

@@ -35,9 +35,10 @@
                  {cowboy_port, 5280},
                  {cowboy_secure_port, 5285},
                  {http_api_client_endpoint_port, 8089},
-                 {service_port, 8888},
-                 {kicking_service_port, 8666},
-                 {hidden_service_port, 8189},
+                 {component_port, 8888},
+                 {kicking_component_port, 8666},
+                 {hidden_component_port, 8189},
+                 {tls_component_port, 8190},
                  {gd_endpoint_port, 5555},
                  {http_notifications_port, 8000},
                  {blocks_hosts, [reg]}]},
@@ -48,7 +49,7 @@
                  {cluster, mim},
                  {c2s_tls_port, 5233},
                  {gd_endpoint_port, 6666},
-                 {service_port, 8899}]},
+                 {component_port, 8899}]},
          {mim3, [{node, mongooseim3@localhost},
                  {domain, <<"localhost">>},
                  {host_type, <<"localhost">>},
@@ -69,7 +70,7 @@
                  {domain, <<"reg1">>},
                  {host_type, <<"red1">>},
                  {vars, "reg1"},
-                 {service_port, 9990},
+                 {component_port, 9990},
                  {c2s_port, 5252},
                  {gd_endpoint_port, 7777},
                  {gd_extra_endpoint_port, 10000},
@@ -210,8 +211,7 @@
         {server, <<"localhost">>},
         {host, <<"localhost">>},
         {password, <<"scramshaplus">>},
-        {starttls, required},
-        {tls_module, fast_tls}
+        {starttls, required}
     ]},
     {alice3, [ %% used in dynamic_domains_SUITE
         {username, <<"alice">>},

big_tests/tests/bosh_SUITE.erl

@@ -107,7 +107,7 @@ acks_test_cases() ->
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
-    instrument_helper:start(instrumentation_events(), negative_instrumentation_events()),
+    instrument_helper:start(instrumentation_events()),
     Config1 = dynamic_modules:save_modules(host_type(), Config),
     escalus:init_per_suite([{escalus_user_db, {module, escalus_ejabberd}} | Config1]).
 
@@ -195,9 +195,6 @@ create_and_terminate_session(Config) ->
     [instrument_helper:assert(Event, Label, fun(#{byte_size := BS}) -> BS > 0 end)
      || {Event, Label} <- instrumentation_events(), Event =/= c2s_message_processed],
 
-    %% Verify C2S listener is not used
-    instrument_helper:assert_not_emitted(negative_instrumentation_events()),
-
     %% Assert the session was terminated.
     wait_for_zero_bosh_sessions().
 
@@ -952,14 +949,6 @@ wait_for_zero_bosh_sessions() ->
 
 instrumentation_events() ->
     instrument_helper:declared_events(mod_bosh, [])
-    ++ instrument_helper:declared_events(mongoose_c2s, [global])
-    ++ [{c2s_message_processed, #{host_type => host_type()}}].
-
-negative_instrumentation_events() ->
-    [{Name, #{}} || Name <- negative_instrumentation_events_names()].
-
-negative_instrumentation_events_names() ->
-    [c2s_tcp_data_out,
-     c2s_tcp_data_in,
-     c2s_tls_data_out,
-     c2s_tls_data_in].
+    ++ [{c2s_message_processed, #{host_type => domain_helper:host_type()}},
+        {xmpp_element_size_out, #{connection_type => c2s}, #{metrics => #{byte_size => histogram}}},
+        {xmpp_element_size_in, #{connection_type => c2s}, #{metrics => #{byte_size => histogram}}}].

big_tests/tests/ca_certificate_helper.erl

@@ -14,9 +14,12 @@ generate_cert(Config, #{cn := User} = CertSpec, BasicTemplateValues) ->
     ct:log("OpenSSL config: ~ts~n~ts", [UserConfig, OpenSSLConfig]),
     file:write_file(UserConfig, OpenSSLConfig),
     UserKey = filename:join(?config(priv_dir, Config), User ++ "_key.pem"),
+    MaybeCritical = maps:get(with_critical_extension, CertSpec, false),
     case maps:get(signed, CertSpec, ca) of
-	    ca -> generate_ca_signed_cert(Config, User, UserConfig, UserKey);
-	    self -> generate_self_signed_cert(Config, User, UserConfig, UserKey)
+	    ca -> 
+	        generate_ca_signed_cert(Config, User, UserConfig, UserKey, MaybeCritical);
+	    self -> 
+	        generate_self_signed_cert(Config, User, UserConfig, UserKey, MaybeCritical)
     end.
 
 prepare_template_values(User, XMPPAddrsIn) ->
@@ -35,11 +38,12 @@ make_xmpp_addr_entry(I, Addr) ->
     "otherName." ++ integer_to_list(I) ++ " = id-on-xmppAddr;UTF8:" ++ Addr.
 
 
-generate_ca_signed_cert(Config, Filename, ConfigCfg, KeyFilename) ->
+generate_ca_signed_cert(Config, Filename, ConfigCfg, KeyFilename, MaybeCritical) ->
     Csr = filename:join(?config(priv_dir, Config), Filename ++ ".csr"),
     Cmd = ["openssl req -config ", ConfigCfg, " -newkey rsa:2048 -sha256 -nodes -out ",
-           Csr, " -keyout ", KeyFilename, " -outform PEM"],
-    Out = os:cmd(Cmd),
+           Csr, " -keyout ", KeyFilename, " -outform PEM" | extensions(ca_signed, MaybeCritical)],
+    Out = os:cmd(Cmd ++ " && echo CREATED"),
+    verify_created(Cmd, Out),
     ct:log("generate_ca_signed_cert 1:~nCmd ~p~nOut ~ts", [Cmd, Out]),
     Cert = filename:join(?config(priv_dir, Config), Filename ++ "_cert.pem"),
     SignCmd = filename:join(?config(mim_data_dir, Config), "sign_cert.sh"),
@@ -50,11 +54,29 @@ generate_ca_signed_cert(Config, Filename, ConfigCfg, KeyFilename) ->
     #{key => KeyFilename,
       cert => Cert}.
 
-generate_self_signed_cert(Config, Filename, ConfigCfg, KeyFilename) ->
+generate_self_signed_cert(Config, Filename, ConfigCfg, KeyFilename, MaybeCritical) ->
     Cert = filename:join(?config(priv_dir, Config), Filename ++ "_self_signed_cert.pem"),
     Cmd = ["openssl req -config ", ConfigCfg, " -newkey rsa:2048 -sha256 -nodes -out ",
-           Cert, " -keyout ", KeyFilename, " -x509 -outform PEM -extensions client_req_extensions"],
-    OutLog = os:cmd(Cmd),
+           Cert, " -keyout ", KeyFilename, " -x509 -outform PEM", extensions(self_signed, MaybeCritical)],
+    OutLog = os:cmd(Cmd ++ " && echo CREATED"),
+    verify_created(Cmd, OutLog),
     ct:log("generate_self_signed_cert:~nCmd ~p~nOut ~ts", [Cmd, OutLog]),
     #{key => KeyFilename,
       cert => Cert}.
+
+verify_created(Cmd, Output) ->
+    case lists:suffix("CREATED\n", Output) of
+        true ->
+            ok;
+        false ->
+            ct:fail({failed_to_create_certificate, Cmd, Output})
+    end.
+
+extensions(ca_signed, false) ->
+    [];
+extensions(ca_signed, true) ->
+    " -extensions critical_extensions";
+extensions(self_signed, false) ->
+    " -extensions client_req_extensions";
+extensions(self_signed, true) ->
+    " -extensions self_signed_critical_extensions".