Merge pull request #4524 from esl/startup_optimisations

Rework startup order and FIPS config

Author: chrzaszcz

Makefile

@@ -41,7 +41,7 @@ rock:
 
 ## Don't allow these files to go out of sync!
 configure.out rel/configure.vars.config:
-	./tools/configure with-all without-jingle-sip
+	./tools/configure
 
 devrel: $(DEVNODES)
 

big_tests/rebar.config

@@ -1,6 +1,5 @@
 {erl_opts, [debug_info,
-            {i, ["include"]},
-            {platform_define, "^(20)", fun_stacktrace}]}.
+            {i, ["include"]}]}.
 
 {require_min_otp_vsn, "21"}.
 

big_tests/tests/jingle_SUITE.erl

@@ -56,20 +56,15 @@ suite() ->
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
-    case rpc(mim(), application, get_application, [nksip]) of
-        {ok, nksip} ->
-            distributed_helper:add_node_to_cluster(mim2(), Config),
-            start_nksip_in_mim_nodes(),
-            application:ensure_all_started(esip),
-            spawn(fun() -> ets:new(jingle_sip_translator, [public, named_table]),
-                           ets:new(jingle_sip_translator_bindings, [public, named_table]),
-                           receive stop -> ok end end),
-            esip:add_listener(12345, tcp, []),
-            esip:set_config_value(module, jingle_sip_translator),
-            escalus:init_per_suite(Config);
-        undefined ->
-            {skip, build_was_not_configured_with_jingle_sip}
-    end.
+    distributed_helper:add_node_to_cluster(mim2(), Config),
+    start_nksip_in_mim_nodes(),
+    application:ensure_all_started(esip),
+    spawn(fun() -> ets:new(jingle_sip_translator, [public, named_table]),
+                   ets:new(jingle_sip_translator_bindings, [public, named_table]),
+                   receive stop -> ok end end),
+    esip:add_listener(12345, tcp, []),
+    esip:set_config_value(module, jingle_sip_translator),
+    escalus:init_per_suite(Config).
 
 start_nksip_in_mim_nodes() ->
     Pid1 = start_nskip_in_parallel(mim(), #{}),

doc/developers-guide/OpenSSL-and-FIPS.md

@@ -51,7 +51,7 @@ where `Value` is either `true` or `false`.
 When MongooseIM starts, it prints the following log message if FIPS mode is enabled
 
 ```
-2015-02-25 14:30:54.501 [warning] <0.242.0>@mongoose_fips:do_notify:37 FIPS mode enabled
+2015-02-25 14:30:54.501 [warning] <0.242.0>@mongoose_fips:notify:15 FIPS mode enabled
 ```
 
 ### Run-time check
@@ -64,9 +64,9 @@ mongoose_fips:status().
 
 The function returns:
 
+* not_supported - erlang compiled without fips support
 * not_enabled - fips_mode is not set to true in etc/app.config
 * enabled - fips_mode is set to true in etc/app.config
-* not_supported - erlang compiled without fips support
 
 ### Cipher suites difference
 

doc/developers-guide/release_config.md

@@ -26,15 +26,6 @@ Writes configure.out file as output - this file can be sourced with:
 
 Writes rel/configure.vars.config which can be used as Reltool input.
 
-3rd party apps:
-
-    with-none           include no 3rd party drivers
-    with-all            include all drivers
-    with-mysql          include mysql driver
-    with-odbc           include an ODBC driver (requires unixodbc to compile)
-    with-pgsql          include pgsql driver
-    with-redis          include redis driver
-
 Options:
 
     prefix    Installation PREFIX directory. Default: /usr/local
@@ -44,13 +35,6 @@ Options:
 
 This script is also accessible via the make `configure` target.
 
-### Example
-
-If `mysql` and `redis` are the only drivers that should be included in the
-release, run the following command before `make rel`:
-
-    $ ./tools/configure with-mysql with-redis
-
 You only need to run the `./tools/configure` command once (unless changing the release's config is needed to
 include some other dependencies).
 
@@ -63,7 +47,7 @@ Results should be similar to those described below.
 On Mac:
 
 ```sh
-./tools/configure with-all user=erszcz prefix=/tmp/mim-sandbox-system system=yes
+./tools/configure user=erszcz prefix=/tmp/mim-sandbox-system system=yes
 cat configure.out rel/configure.vars.config
 RUNNER_GROUP=staff make install
 ```

doc/modules/mod_jingle_sip.md

@@ -83,20 +83,6 @@ This may be useful in web applications when there is a need to handle the call i
 In order to get the `session-initiate`, which was not answered yet, the user can send a `get` Jingle stanza to self with action set to `existing-session-initiate`.
 As a result, MongooseIM will resend the original `session-initiate` request to the device which sent the query.
 
-## Prerequisites
-
-By default, MongooseIM is built without SIP support.
-In order to build the server with SIP support, please use `tools/configure` script before the release generation.
-You may either pick only certain drivers (with SIP included) or simply use `with-all` option. Examples:
-
-```
-tools/configure with-mysql with-jingle-sip
-tools/configure with-all without-odbc
-tools/configure with-all
-```
-
-MongooseIM packages are built with Jingle/SIP support.
-
 !!! warning
     This module does not support [dynamic domains](../configuration/general.md#generalhost_types).
 

doc/tutorials/Jingle-SIP-setup.md

@@ -8,9 +8,6 @@ We are going to use the following open source software:
 
 * MongooseIM - [https://github.com/esl/MongooseIM](https://github.com/esl/MongooseIM)
     * see [How-to-build](How-to-build.md) for details on building.
-      It's important to remember to run the configuration script with `with-jingle-sip` flag set:
-      `tools/configure with-jingle-sip`.
-      Without this, third party dependencies required by the Jingle/SIP translator will not be included in the release.
 
 * Routr (SIP server) - [https://routr.io](https://routr.io)
     * I recommend downloading binaries for your system from [official source](https://routr.io/docs/introduction/installation/#custom).

rebar.config

@@ -1,9 +1,5 @@
 {erl_opts, [debug_info,
-            {i, ["include"]},
-            {d, xml_nif},
-            {d, namespaced_types},
-            {platform_define, "^(20)", fun_stacktrace},
-            {platform_define, "^(25)", gen_server_request_id}]}.
+            {i, ["include"]}]}.
 
 %% For behaviour info
 {erl_first_files, [
@@ -25,7 +21,7 @@
     mod_global_distrib_mapping_backend,
     mod_pubsub_db_backend,
     mod_shared_roster,
-    ejabberd_c2s, ejabberd_c2s_state, ejabberd_local,
+    ejabberd_local,
     mongoose_c2s,
     mongoose_c2s_acc,
     %% Deprecated functions
@@ -113,8 +109,7 @@
   {luerl, "1.3.0"}
  ]}.
 
-{relx, [{release, { mongooseim, {cmd, "cat VERSION | tr -d '\r\n'"} },
-         []}, %%Apps list is dynamicaly set by rebar.config.script
+{relx, [{release, {mongooseim, {file, "VERSION"}}, []}, %%Apps list is dynamicaly set by rebar.config.script
 
         %% Disable some warnings in systools:make_script/2 which are extremely slow (saves 30 seconds)
         {src_tests, false},
@@ -224,7 +219,8 @@
     {plt_extra_apps, [jid, cowboy, cowlib, lasse, p1_utils, ranch, gen_fsm_compat, epgsql, cqerl,
                       common_test, tools, amqp_client, jiffy, erl_csv, inets, compiler, jsx,
                       rabbit_common, mysql, eredis, erlcloud, eodbc, telemetry,
-                      nksip, nklib, nkservice, nkpacket, prometheus]}]}.
+                      nksip, nklib, nkservice, nkpacket, prometheus, exometer_core,
+                      mnesia, cets, cpool, tirerl, erlang_doctor]}]}.
 
 {cover_print_enabled, true}.
 {cover_export_enabled, true}.

rebar.config.script

@@ -1,88 +1,39 @@
-os:putenv("EXOMETER_PACKAGES", "(minimal)"),
-
-MaybeReadFromConfig =
-fun ({ok, Content}) ->
-        Parts = binary:split(Content, <<"\n">>, [global]),
-        %% find only this line which starts with "export APPS="
-        [Apps] = [Item || <<"export APPS=\"",Item/binary>> <- Parts],
-        %% remove trailing " and conver to string
-        binary_to_list(binary:part(Apps, 0, size(Apps) - 1));
-    (_) ->
-        io:format("WARNING! The configure.out file was not created, using minimal configuration~n"),
-        ""
-end,
-
-ReadFromConfig =
-fun(Path) ->
-        case filelib:is_file(Path) of
-            true ->
-                MaybeReadFromConfig(file:read_file(Path));
-            _ ->
-                Script = filename:join(["tools", "configure"]),
-                os:cmd(Script ++ " with-all without-jingle-sip"),
-                MaybeReadFromConfig(file:read_file("configure.out"))
-        end
-end,
-
-GetEnvApps =
-fun() ->
-        case os:getenv("APPS") of
-            false ->
-                ConfigurePath = "configure.out",
-                ReadFromConfig(ConfigurePath);
-            EnvApps ->
-                EnvApps
-        end
-end,
-
-DevAppsToInclude =
-fun() ->
-        case os:getenv("DEVNODE") of
-            false -> [];
-            _ -> [meck]
-        end
-end,
-
-FirstApps = [kernel],
-RequiredApps = [mongooseim, inets, tools, compiler],
-OptionalApps = [{nksip, [nkservice, nkpacket, nklib]}],
-
-EnvApps = GetEnvApps(),
-
-PrepareApp =
-fun(App) ->
-        case lists:keyfind(App, 1, OptionalApps) of
-            false ->
-                [App];
-            {App, DepsOfApp} ->
-                lists:map(fun(V) -> {V, load} end, [App | DepsOfApp])
-        end
-end,
-
-SetupIncludedApps =
-fun(Config, EnvApps) ->
-        RelxCfg = proplists:get_value(relx, Config),
-        {release, Desc, _Apps} = lists:keyfind(release, 1, RelxCfg),
-        EnvAppsToInclude = [ list_to_atom(App) || App <- string:tokens(EnvApps, " \n\r") ],
-        AppsToIncludeIn = lists:usort(RequiredApps ++ DevAppsToInclude() ++ EnvAppsToInclude),
-        Apps = lists:subtract(AppsToIncludeIn, FirstApps),
-        NewReleaseCfg = {release, Desc, FirstApps ++ lists:flatmap(PrepareApp, Apps)},
-        NewRelxCfg = lists:keyreplace(release, 1, RelxCfg, NewReleaseCfg),
-        lists:keyreplace(relx, 1, Config, {relx, NewRelxCfg})
-end,
-
-MaybeFIPSSupport = fun(Config) ->
-    code:ensure_loaded(crypto),
-    case erlang:function_exported(crypto, info_fips, 0) of
-        false ->
-            Config;
-        _ ->
-            io:format("compiling with FIPS mode enabled~n"),
-            {erl_opts, ErlOpts} = lists:keyfind(erl_opts, 1, Config),
-            NewErlOpts = [{d, fips_mode} | ErlOpts],
-            lists:keyreplace(erl_opts, 1, Config, {erl_opts, NewErlOpts})
-    end
-end,
-
-Config0 = SetupIncludedApps(CONFIG, EnvApps),
-MaybeFIPSSupport(Config0).
+BaseApps = [kernel, stdlib, sasl,
+            crypto, inets, public_key, ssl,
+            tools, runtime_tools,
+            mongooseim,
+            {mnesia, none},
+            {erlang_doctor, none},
+            {syslogger, none},
+            {cpool, none},
+            {backoff, none},
+            {mysql, none},
+            {eodbc, none},
+            {epgsql, none},
+            {eredis, none},
+            {cqerl, none},
+            {tirerl, none},
+            {erlcloud, none},
+            {amqp_client, none},
+            {exometer_core, none},
+            {exometer_report_graphite, none},
+            {exometer_report_statsd, none},
+            {prometheus, none},
+            {prometheus_httpd, none},
+            {prometheus_cowboy, none},
+            {nksip, none},
+            {nkservice, none},
+            {nkpacket, none},
+            {nklib, none}
+           ],
+
+Apps = case os:getenv("DEVNODE") of
+           false -> BaseApps;
+           _ -> [meck | BaseApps]
+       end,
+
+{relx, RelxCfg} = lists:keyfind(relx, 1, CONFIG),
+{release, Desc, _Apps} = lists:keyfind(release, 1, RelxCfg),
+NewReleaseCfg = {release, Desc, Apps},
+NewRelxCfg = lists:keyreplace(release, 1, RelxCfg, NewReleaseCfg),
+lists:keyreplace(relx, 1, CONFIG, {relx, NewRelxCfg}).

src/async_pools/mongoose_aggregator_worker.erl

@@ -16,18 +16,12 @@
 
 -behaviour(gen_server).
 
--ifdef(gen_server_request_id).
--type request_id() :: gen_server:request_id().
--else.
--type request_id() :: term().
--endif.
-
 -callback aggregate(mongoose_async_pools:task(),
                     mongoose_async_pools:task(),
                     mongoose_async_pools:pool_extra()) ->
     {ok, mongoose_async_pools:task()} | {error, term()}.
 -callback request(mongoose_async_pools:task(), mongoose_async_pools:pool_extra()) ->
-    request_id() | drop.
+    gen_server:request_id() | drop.
 -callback verify(term(), mongoose_async_pools:task(), mongoose_async_pools:pool_extra()) ->
     term().
 -optional_callbacks([verify/3]).
@@ -43,7 +37,7 @@
          code_change/3,
          format_status/1]).
 
--type request() :: no_request_pending | {request_id(), mongoose_async_pools:task()}.
+-type request() :: no_request_pending | {gen_server:request_id(), mongoose_async_pools:task()}.
 -record(state, {
           host_type :: mongooseim:host_type(),
           pool_id :: mongoose_async_pools:pool_id(),