Skip to content

feat: add param to set html size threshold #238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 6, 2025
Merged

feat: add param to set html size threshold #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
15449cc
feat: add param to set html size threshold
vdayanand Aug 5, 2025
d1fa7ff
fix
vdayanand Aug 6, 2025
2b84777
more changes
vdayanand Aug 6, 2025
51020dd
fiox
vdayanand Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions src/DocumentationGenerator.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ function try_use_package(packagespec, envdir)
return succeeded
end

function build_package_docs(packagespec::Pkg.Types.PackageSpec, buildpath, registry; src_prefix="", href_prefix="")
function build_package_docs(packagespec::Pkg.Types.PackageSpec, buildpath, registry; src_prefix="", href_prefix="", html_size_threshold_bytes=nothing)
type, uri = doctype(packagespec, registry)

@info("$(packagespec.name) specifies docs of type $(type).")
Expand All @@ -71,7 +71,7 @@ function build_package_docs(packagespec::Pkg.Types.PackageSpec, buildpath, regis
elseif type == "git-repo"
build_git_docs(packagespec, buildpath, uri; src_prefix=src_prefix, href_prefix=href_prefix)
elseif type == "vendored"
build_local_docs(packagespec, buildpath, uri; src_prefix=src_prefix, href_prefix=href_prefix)
build_local_docs(packagespec, buildpath, uri; src_prefix=src_prefix, href_prefix=href_prefix, html_size_threshold_bytes=html_size_threshold_bytes)
else
@error("Invalid doctype specified: $(type).")
Dict(
Expand Down Expand Up @@ -102,7 +102,8 @@ function build_documentation(
registry = joinpath(homedir(), ".julia/registries/General"),
timeout = RUNNER_TIMEOUT,
max_timeout = RUNNER_MAX_TIMEOUT,
kill_timeout = RUNNER_KILL_TIMEOUT
kill_timeout = RUNNER_KILL_TIMEOUT,
html_size_threshold_bytes=nothing
)

has_xvfb = try
Expand Down Expand Up @@ -149,7 +150,7 @@ function build_documentation(
update_only = update_only,
timeout = timeout,
max_timeout = max_timeout,
kill_timeout = kill_timeout)
kill_timeout = kill_timeout, html_size_threshold_bytes=html_size_threshold_bytes)
push!(process_queue, proc)
end
end
Expand Down Expand Up @@ -290,6 +291,7 @@ function start_builder(package, version;
timeout = RUNNER_TIMEOUT,
max_timeout = RUNNER_MAX_TIMEOUT,
kill_timeout = RUNNER_KILL_TIMEOUT,
html_size_threshold_bytes=nothing
)

workerfile = joinpath(@__DIR__, "workerfile.jl")
Expand Down Expand Up @@ -332,6 +334,7 @@ function start_builder(package, version;
$href_prefix
$(server_type)
$(isempty(api_url) ? "-" : api_url)
$(isnothing(html_size_threshold_bytes) ? "-" : html_size_threshold_bytes)
$(update_only ? "update" : "build")

```
Expand Down
37 changes: 22 additions & 15 deletions src/builders.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,15 @@ using HTMLSanitizer
using Highlights
using Downloads
using Documenter
function build_git_docs(packagespec, buildpath, uri; src_prefix="", href_prefix="")

function build_git_docs(packagespec, buildpath, uri; src_prefix="", href_prefix="", html_size_threshold_bytes=nothing)
pkgname = packagespec.name
return mktempdir() do dir
return cd(dir) do
run(`git clone --depth=1 $(uri) $(pkgname)`)
docsproject = joinpath(dir, pkgname)
return cd(docsproject) do
return build_local_docs(packagespec, buildpath, nothing, docsproject, gitdirdocs = true; src_prefix=src_prefix, href_prefix=href_prefix)
return build_local_docs(packagespec, buildpath, nothing, docsproject, gitdirdocs = true; src_prefix=src_prefix, href_prefix=href_prefix, html_size_threshold_bytes=html_size_threshold_bytes)
end
end
end
Expand Down Expand Up @@ -84,7 +85,7 @@ function maybe_redirect(uri)
return uri
end

function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdirdocs = false, src_prefix="", href_prefix="")
function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdirdocs = false, src_prefix="", href_prefix="", html_size_threshold_bytes=nothing)
uri = something(uri, "docs")
mktempdir() do envdir
pkgname = packagespec.name
Expand All @@ -110,7 +111,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
for docdir in joinpath.(pkgroot, unique([uri, "docs", "doc"]))
if isdir(docdir)
@info("Building vendored Documenter.jl documentation at $(docdir).")
output = build_documenter(packagespec, docdir)
output = build_documenter(packagespec, docdir; html_size_threshold_bytes)
@info("Documentation built at $(output).")
if output !== nothing
@info("Copying build documentation from $(output) to $(buildpath)")
Expand Down Expand Up @@ -139,7 +140,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir

# fallback docs (readme & docstrings)
return mktempdir() do docsdir
output = build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg)
output = build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg, html_size_threshold_bytes)
if output !== nothing
cp(output, buildpath, force = true)
return Dict(
Expand All @@ -161,7 +162,7 @@ function build_local_docs(packagespec, buildpath, uri, pkgroot = nothing; gitdir
end
end

function build_legacy_documenter(packagespec, docdir)
function build_legacy_documenter(packagespec, docdir; html_size_threshold_bytes=nothing)
open(joinpath(docdir, "Project.toml"), "w") do io
println(io, """
[deps]
Expand All @@ -171,16 +172,16 @@ function build_legacy_documenter(packagespec, docdir)
Documenter = "~0.20"
""")
end
build_documenter(packagespec, docdir)
build_documenter(packagespec, docdir; html_size_threshold_bytes)
end

function build_documenter(packagespec, docdir)
function build_documenter(packagespec, docdir; html_size_threshold_bytes=nothing)
pkgdir = normpath(joinpath(docdir, ".."))
cd(pkgdir) do
docsproject = joinpath(docdir, "Project.toml")
docsmanifest = joinpath(docdir, "Manifest.toml")
if !isfile(docsproject)
return build_legacy_documenter(packagespec, docdir)
return build_legacy_documenter(packagespec, docdir; html_size_threshold_bytes)
end

# fix permissions to allow us to add the main pacakge to the docs project
Expand All @@ -190,7 +191,7 @@ function build_documenter(packagespec, docdir)
chmod(joinpath(pkgdir, "Project.toml"), 0o660)
isfile(joinpath(pkgdir, "Manifest.toml")) && chmod(joinpath(pkgdir, "Manifest.toml"), 0o660)

rundcocumenter = joinpath(@__DIR__, "rundocumenter.jl")
rundocumenter = joinpath(@__DIR__, "rundocumenter.jl")

makefile = joinpath(docdir, "make.jl")
if !isfile(makefile)
Expand All @@ -203,15 +204,17 @@ function build_documenter(packagespec, docdir)
makefile = joinpath(docdir, jlfiles[1])
@info("Using $(makefile) to generate Documenter docs.")
end
_, builddir = fix_makefile(makefile)
_, builddir = fix_makefile(makefile; html_size_threshold_bytes)
pkgimagesopt = VERSION >= v"1.9" ? "--pkgimages=no" : ""
opt_ser = isnothing(html_size_threshold_bytes) ? "-" : html_size_threshold_bytes
cmd = ```
$(julia())
--project="$(docdir)"
$(isempty(pkgimagesopt) ? [] : pkgimagesopt)
$(rundcocumenter)
$(rundocumenter)
$(pkgdir)
$(makefile)
$(opt_ser)
```

try
Expand All @@ -229,7 +232,7 @@ function build_documenter(packagespec, docdir)
end
end

function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg)
function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, could_use_pkg, html_size_threshold_bytes=nothing)
@info("Generating readme-only fallback docs.")

if pkgroot === nothing || !ispath(pkgroot)
Expand Down Expand Up @@ -265,7 +268,11 @@ function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, c
end
end
end

doc_format_str = if isnothing(html_size_threshold_bytes)
"Documenter.HTML()"
else
"Documenter.HTML(;size_threshold=$html_size_threshold_bytes)"
end
makejl_str = """
using Pkg
Pkg.add(name="Documenter", version="1")
Expand All @@ -275,7 +282,7 @@ function build_readme_docs(pkgname, pkgroot, docsdir, src_prefix, href_prefix, c
using $(pkgname)

makedocs(
format = Documenter.HTML(),
format = $(doc_format_str),
sitename = "$(pkgname).jl",
modules = [$(pkgname)],
root = "$(docsdir)",
Expand Down
6 changes: 4 additions & 2 deletions src/rundocumenter.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ let anonymous_module = Module()
Base.include(anonymous_module, joinpath(@__DIR__, "utils", "rewrite.jl"))
@eval anonymous_module begin
function main(args)
if length(args) == 2
if length(args) == 3
pkgdir = args[1]
makefile = args[2]
html_size_threshold_bytes = args[3] == "-" ? nothing : tryparse(Int, args[3])
else
makefile = joinpath(pwd(), "make.jl")
pkgdir = normpath(joinpath(pwd(), ".."))
html_size_threshold_bytes = nothing
@info("No directory specified. Falling back to pkgdir=`$(pkgdir)` and makefile=`$(makefile)`.")
end
docsdir = dirname(makefile)
Expand Down Expand Up @@ -51,7 +53,7 @@ let anonymous_module = Module()

@info("Using Documenter version $(documenter_version).")

expr, bpath = fix_makefile(makefile, documenter_version)
expr, bpath = fix_makefile(makefile, documenter_version; html_size_threshold_bytes)

@info("`cd`ing to `$(docsdir)` and setting `tls[:SOURCE_PATH]` to `$(makefile)`.")
task_local_storage()[:SOURCE_PATH] = makefile
Expand Down
11 changes: 8 additions & 3 deletions src/utils/rewrite.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ function parseall(str)
end

"""
fix_makefile(makefile)
fix_makefile(makefile; html_size_threshold_bytes=nothing)

Takes in the path to a Documenter.jl-compatible `make.jl` file and
1. Removes all calls to `deploydocs`.
Expand All @@ -24,7 +24,7 @@ Takes in the path to a Documenter.jl-compatible `make.jl` file and

Return a tuple of `new_make_expr, buildpath`.
"""
function fix_makefile(makefile, documenter_version = v"1.8.1")
function fix_makefile(makefile, documenter_version = v"1.8.1"; html_size_threshold_bytes=nothing)
# default output path:
buildpath = joinpath(dirname(makefile), "build")
should_break = false
Expand All @@ -51,7 +51,12 @@ function fix_makefile(makefile, documenter_version = v"1.8.1")
has_remotes = false
has_repo = false
has_warnonly = false
html = documenter_version < v"0.21" ? QuoteNode(:html) : :(Documenter.HTML())
docs_exp = if isnothing(html_size_threshold_bytes)
:(Documenter.HTML())
else
:(Documenter.HTML(;size_threshold=$html_size_threshold_bytes))
end
html = documenter_version < v"0.21" ? QuoteNode(:html) : docs_exp

fixkwarg = argument -> begin
if Meta.isexpr(argument, :kw)
Expand Down
5 changes: 3 additions & 2 deletions src/workerfile.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ using DocumentationGenerator

Pkg.status()

function build(uuid, name, url, version, buildpath, registry, deployment_url, src_prefix, href_prefix, server_type, api_url, args...)
function build(uuid, name, url, version, buildpath, registry, deployment_url, src_prefix, href_prefix, server_type, api_url, html_size_threshold_bytes, args...)
packagespec = PackageSpec(uuid = uuid, name = name, version = VersionNumber(version))
api_url = api_url == "-" ? "" : api_url
html_size_threshold_bytes = html_size_threshold_bytes == "-" ? nothing : tryparse(Int, html_size_threshold_bytes)
withenv(
"DOCUMENTATIONGENERATOR" => "true",
"CI" => "true",
Expand All @@ -20,7 +21,7 @@ function build(uuid, name, url, version, buildpath, registry, deployment_url, sr
end
new_metadata = DocumentationGenerator.package_metadata(packagespec, url, server_type; api_url = api_url)
merge!(metadata, new_metadata)
build_meta = DocumentationGenerator.build_package_docs(packagespec, buildpath, registry; src_prefix=src_prefix, href_prefix=href_prefix)
build_meta = DocumentationGenerator.build_package_docs(packagespec, buildpath, registry; src_prefix=src_prefix, href_prefix=href_prefix, html_size_threshold_bytes=html_size_threshold_bytes)
merge!(metadata, build_meta)

isdir(buildpath) || mkpath(buildpath)
Expand Down