feat: add param to set html size threshold (#238)

* feat: add param to set html size threshold

* fix

* more changes

* fiox

Author: vdayanand

src/DocumentationGenerator.jl

@@ -61,7 +61,7 @@ function try_use_package(packagespec, envdir)
     return succeeded
 end
 
-function build_package_docs(packagespec::Pkg.Types.PackageSpec, buildpath, registry; src_prefix="", href_prefix="")
+function build_package_docs(packagespec::Pkg.Types.PackageSpec, buildpath, registry; src_prefix="", href_prefix="", html_size_threshold_bytes=nothing)
     type, uri = doctype(packagespec, registry)
 
     @info("$(packagespec.name) specifies docs of type $(type).")
@@ -71,7 +71,7 @@ function build_package_docs(packagespec::Pkg.Types.PackageSpec, buildpath, regis
         elseif type == "git-repo"
             build_git_docs(packagespec, buildpath, uri; src_prefix=src_prefix, href_prefix=href_prefix)
         elseif type == "vendored"
-            build_local_docs(packagespec, buildpath, uri; src_prefix=src_prefix, href_prefix=href_prefix)
+            build_local_docs(packagespec, buildpath, uri; src_prefix=src_prefix, href_prefix=href_prefix, html_size_threshold_bytes=html_size_threshold_bytes)
         else
             @error("Invalid doctype specified: $(type).")
             Dict(
@@ -102,7 +102,8 @@ function build_documentation(
         registry = joinpath(homedir(), ".julia/registries/General"),
         timeout = RUNNER_TIMEOUT,
         max_timeout = RUNNER_MAX_TIMEOUT,
-        kill_timeout = RUNNER_KILL_TIMEOUT
+        kill_timeout = RUNNER_KILL_TIMEOUT,
+        html_size_threshold_bytes=nothing
     )
 
     has_xvfb = try
@@ -149,7 +150,7 @@ function build_documentation(
                                            update_only = update_only,
                                            timeout = timeout,
                                            max_timeout = max_timeout,
-                                           kill_timeout = kill_timeout)
+                                           kill_timeout = kill_timeout, html_size_threshold_bytes=html_size_threshold_bytes)
                     push!(process_queue, proc)
                 end
         end
@@ -290,6 +291,7 @@ function start_builder(package, version;
         timeout = RUNNER_TIMEOUT,
         max_timeout = RUNNER_MAX_TIMEOUT,
         kill_timeout = RUNNER_KILL_TIMEOUT,
+        html_size_threshold_bytes=nothing
     )
 
     workerfile = joinpath(@__DIR__, "workerfile.jl")
@@ -332,6 +334,7 @@ function start_builder(package, version;
                 $href_prefix
                 $(server_type)
                 $(isempty(api_url) ? "-" : api_url)
+                $(isnothing(html_size_threshold_bytes) ? "-" : html_size_threshold_bytes)
                 $(update_only ? "update" : "build")
 
         ```

src/builders.jl

@@ -4,14 +4,15 @@ using HTMLSanitizer
 using Highlights
 using Downloads
 using Documenter
-function build_git_docs(packagespec, buildpath, uri; src_prefix="", href_prefix="")
+
+function build_git_docs(packagespec, buildpath, uri; src_prefix="", href_prefix="", html_size_threshold_bytes=nothing)
     pkgname = packagespec.name
     return mktempdir() do dir
         return cd(dir) do
             run(`git clone --depth=1 $(uri) $(pkgname)`)
             docsproject = joinpath(dir, pkgname)
             return cd(docsproject) do
-                return build_local_docs(packagespec, buildpath, nothing, docsproject, gitdirdocs = true; src_prefix=src_prefix, href_prefix=href_prefix)
+                return build_local_docs(packagespec, buildpath, nothing, docsproject, gitdirdocs = true; src_prefix=src_prefix, href_prefix=href_prefix, html_size_threshold_bytes=html_size_threshold_bytes)
             end
         end
     end
@@ -84,7 +85,7 @@ function maybe_redirect(uri)
     return uri
 end
 
-function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdirdocs = false, src_prefix="", href_prefix="")
+function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdirdocs = false, src_prefix="", href_prefix="", html_size_threshold_bytes=nothing)
     uri = something(uri, "docs")
     mktempdir() do envdir
         pkgname = packagespec.name
@@ -110,7 +111,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
             for docdir in joinpath.(pkgroot, unique([uri, "docs", "doc"]))
                 if isdir(docdir)
                     @info("Building vendored Documenter.jl documentation at $(docdir).")
-                    output = build_documenter(packagespec, docdir)
+                    output = build_documenter(packagespec, docdir; html_size_threshold_bytes)
                     @info("Documentation built at $(output).")
                     if output !== nothing
                         @info("Copying build documentation from $(output) to $(buildpath)")
@@ -139,7 +140,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
 
         # fallback docs (readme & docstrings)
         return mktempdir() do docsdir
-            output = build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg)
+            output = build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg, html_size_threshold_bytes)
             if output !== nothing
                 cp(output, buildpath, force = true)
                 return Dict(
@@ -161,7 +162,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
     end
 end
 
-function build_legacy_documenter(packagespec, docdir)
+function build_legacy_documenter(packagespec, docdir; html_size_threshold_bytes=nothing)
     open(joinpath(docdir, "Project.toml"), "w") do io
         println(io, """
             [deps]
@@ -171,16 +172,16 @@ function build_legacy_documenter(packagespec, docdir)
             Documenter = "~0.20"
         """)
     end
-    build_documenter(packagespec, docdir)
+    build_documenter(packagespec, docdir; html_size_threshold_bytes)
 end
 
-function build_documenter(packagespec, docdir)
+function build_documenter(packagespec, docdir; html_size_threshold_bytes=nothing)
     pkgdir = normpath(joinpath(docdir, ".."))
     cd(pkgdir) do
         docsproject = joinpath(docdir, "Project.toml")
         docsmanifest = joinpath(docdir, "Manifest.toml")
         if !isfile(docsproject)
-            return build_legacy_documenter(packagespec, docdir)
+            return build_legacy_documenter(packagespec, docdir; html_size_threshold_bytes)
         end
 
         # fix permissions to allow us to add the main pacakge to the docs project
@@ -190,7 +191,7 @@ function build_documenter(packagespec, docdir)
         chmod(joinpath(pkgdir, "Project.toml"), 0o660)
         isfile(joinpath(pkgdir, "Manifest.toml")) && chmod(joinpath(pkgdir, "Manifest.toml"), 0o660)
 
-        rundcocumenter = joinpath(@__DIR__, "rundocumenter.jl")
+        rundocumenter = joinpath(@__DIR__, "rundocumenter.jl")
 
         makefile = joinpath(docdir, "make.jl")
         if !isfile(makefile)
@@ -203,15 +204,17 @@ function build_documenter(packagespec, docdir)
             makefile = joinpath(docdir, jlfiles[1])
             @info("Using $(makefile) to generate Documenter docs.")
         end
-        _, builddir = fix_makefile(makefile)
+        _, builddir = fix_makefile(makefile; html_size_threshold_bytes)
         pkgimagesopt = VERSION >= v"1.9" ? "--pkgimages=no" : ""
+        opt_ser = isnothing(html_size_threshold_bytes) ? "-" : html_size_threshold_bytes
         cmd = ```
             $(julia())
                 --project="$(docdir)"
                 $(isempty(pkgimagesopt) ? [] : pkgimagesopt)
-                $(rundcocumenter)
+                $(rundocumenter)
                 $(pkgdir)
                 $(makefile)
+                $(opt_ser)
             ```
 
         try
@@ -229,7 +232,7 @@ function build_documenter(packagespec, docdir)
     end
 end
 
-function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg)
+function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg, html_size_threshold_bytes=nothing)
     @info("Generating readme-only fallback docs.")
 
     if pkgroot === nothing || !ispath(pkgroot)
@@ -265,7 +268,11 @@ function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, c
             end
         end
     end
-
+    doc_format_str = if isnothing(html_size_threshold_bytes)
+        "Documenter.HTML()"
+    else
+        "Documenter.HTML(;size_threshold=$html_size_threshold_bytes)"
+    end
     makejl_str = """
     using Pkg
     Pkg.add(name="Documenter", version="1")
@@ -275,7 +282,7 @@ function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, c
     using $(pkgname)
 
     makedocs(
-        format = Documenter.HTML(),
+        format = $(doc_format_str),
         sitename = "$(pkgname).jl",
         modules = [$(pkgname)],
         root = "$(docsdir)",

src/rundocumenter.jl

@@ -5,12 +5,14 @@ let anonymous_module = Module()
     Base.include(anonymous_module, joinpath(@__DIR__, "utils", "rewrite.jl"))
     @eval anonymous_module begin
         function main(args)
-            if length(args) == 2
+            if length(args) == 3
                 pkgdir = args[1]
                 makefile = args[2]
+                html_size_threshold_bytes = args[3] == "-" ? nothing : tryparse(Int, args[3])
             else
                 makefile = joinpath(pwd(), "make.jl")
                 pkgdir = normpath(joinpath(pwd(), ".."))
+                html_size_threshold_bytes = nothing
                 @info("No directory specified. Falling back to pkgdir=`$(pkgdir)` and makefile=`$(makefile)`.")
             end
             docsdir = dirname(makefile)
@@ -51,7 +53,7 @@ let anonymous_module = Module()
 
             @info("Using Documenter version $(documenter_version).")
 
-            expr, bpath = fix_makefile(makefile, documenter_version)
+            expr, bpath = fix_makefile(makefile, documenter_version; html_size_threshold_bytes)
 
             @info("`cd`ing to `$(docsdir)` and setting `tls[:SOURCE_PATH]` to `$(makefile)`.")
             task_local_storage()[:SOURCE_PATH] = makefile

src/utils/rewrite.jl

@@ -15,7 +15,7 @@ function parseall(str)
 end
 
 """
-    fix_makefile(makefile)
+    fix_makefile(makefile; html_size_threshold_bytes=nothing)
 
 Takes in the path to a Documenter.jl-compatible `make.jl` file and
 1. Removes all calls to `deploydocs`.
@@ -24,7 +24,7 @@ Takes in the path to a Documenter.jl-compatible `make.jl` file and
 
 Return a tuple of `new_make_expr, buildpath`.
 """
-function fix_makefile(makefile, documenter_version = v"1.8.1")
+function fix_makefile(makefile, documenter_version = v"1.8.1"; html_size_threshold_bytes=nothing)
     # default output path:
     buildpath = joinpath(dirname(makefile), "build")
     should_break = false
@@ -51,7 +51,12 @@ function fix_makefile(makefile, documenter_version = v"1.8.1")
             has_remotes = false
             has_repo = false
             has_warnonly = false
-            html = documenter_version < v"0.21" ? QuoteNode(:html) : :(Documenter.HTML())
+            docs_exp = if isnothing(html_size_threshold_bytes)
+                :(Documenter.HTML())
+            else
+                :(Documenter.HTML(;size_threshold=$html_size_threshold_bytes))
+            end
+            html = documenter_version < v"0.21" ? QuoteNode(:html) : docs_exp
 
             fixkwarg = argument -> begin
                 if Meta.isexpr(argument, :kw)

src/workerfile.jl

@@ -4,9 +4,10 @@ using DocumentationGenerator
 
 Pkg.status()
 
-function build(uuid, name, url, version, buildpath, registry, deployment_url, src_prefix, href_prefix, server_type, api_url, args...)
+function build(uuid, name, url, version, buildpath, registry, deployment_url, src_prefix, href_prefix, server_type, api_url, html_size_threshold_bytes, args...)
     packagespec = PackageSpec(uuid = uuid, name = name, version = VersionNumber(version))
     api_url = api_url == "-" ? "" : api_url
+    html_size_threshold_bytes = html_size_threshold_bytes == "-" ? nothing : tryparse(Int, html_size_threshold_bytes)
     withenv(
         "DOCUMENTATIONGENERATOR" => "true",
         "CI" => "true",
@@ -20,7 +21,7 @@ function build(uuid, name, url, version, buildpath, registry, deployment_url, sr
         end
         new_metadata = DocumentationGenerator.package_metadata(packagespec, url, server_type; api_url = api_url)
         merge!(metadata, new_metadata)
-        build_meta = DocumentationGenerator.build_package_docs(packagespec, buildpath, registry; src_prefix=src_prefix, href_prefix=href_prefix)
+        build_meta = DocumentationGenerator.build_package_docs(packagespec, buildpath, registry; src_prefix=src_prefix, href_prefix=href_prefix, html_size_threshold_bytes=html_size_threshold_bytes)
         merge!(metadata, build_meta)
 
         isdir(buildpath) || mkpath(buildpath)