From 5d4c3f242bdd4473b48efd22a5e9148618e65962 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 14:53:32 -0400 Subject: [PATCH 01/13] Rename make_docs.jl make_docs.jl only deals with the registry, so rename to populat_registry.jl --- docs/make.jl | 2 +- docs/{make_docs.jl => populate_registry.jl} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename docs/{make_docs.jl => populate_registry.jl} (97%) diff --git a/docs/make.jl b/docs/make.jl index b340552d..618b0664 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,7 +1,7 @@ using Documenter using FileIO -include("make_docs.jl") +include("populate_registry.jl") makedocs( sitename = "FileIO", diff --git a/docs/make_docs.jl b/docs/populate_registry.jl similarity index 97% rename from docs/make_docs.jl rename to docs/populate_registry.jl index 77ad90eb..413bc3e4 100644 --- a/docs/make_docs.jl +++ b/docs/populate_registry.jl @@ -90,7 +90,7 @@ fs = open(joinpath(pkgdir(FileIO), "docs", "src", "registry.md"), "w") println(fs, """ | Format Name | extensions | IO library | detection or magic number | - | ----------- | ---------- | ---------- | ---------- |""") + | ----------- | ---------- | ---------- | ------------------------- |""") include(joinpath(pkgdir(FileIO), "src", "registry.jl")) close(fs) From 86f0af0a7e7485f8f8724d419b93800fb49fd3e4 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 14:54:09 -0400 Subject: [PATCH 02/13] Use Documenter.jl v1 to build documentation site --- docs/Project.toml | 2 +- docs/make.jl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index f100b5c6..8498d31b 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -5,5 +5,5 @@ ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19" PNGFiles = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883" [compat] -Documenter = "0.26" +Documenter = "1" HTTP = "0.9" diff --git a/docs/make.jl b/docs/make.jl index 618b0664..369b1351 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -15,6 +15,7 @@ makedocs( "world_age_issue.md", "reference.md", ], + checkdocs = :exports, ) deploydocs( From c51195931347ab95ccce6f498422dc647b8c3fdf Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:04:51 -0400 Subject: [PATCH 03/13] Add method and type headers to docstrings --- src/error_handling.jl | 6 ++++++ src/query.jl | 24 +++++++++++++++++------- src/registry.jl | 3 ++- src/registry_setup.jl | 10 +++++++--- src/types.jl | 43 +++++++++++++++++++++++++++---------------- 5 files changed, 59 insertions(+), 27 deletions(-) diff --git a/src/error_handling.jl b/src/error_handling.jl index 8e4b04b9..d688003b 100644 --- a/src/error_handling.jl +++ b/src/error_handling.jl @@ -1,4 +1,6 @@ """ + LoaderError <: Exception + `LoaderError` should be thrown when loader library code fails, and other libraries should be given the chance to recover from the error. Reports the library name and an error message: LoaderError("ImageMagick", "Foo not available") @@ -12,6 +14,8 @@ Base.showerror(io::IO, e::LoaderError) = println(IOContext(io, :limit=>true), e. e.msg, "\n due to ", e.ex, "\n Will try next loader.") """ + WriterError <: Exception + `WriterError` should be thrown when writer library code fails, and other libraries should be given the chance to recover from the error. Reports the library name and an error message: WriterError("ImageMagick", "Foo not available") @@ -34,6 +38,8 @@ end Base.showerror(io::IO, e::SpecError) = print(io, e.mod, " is missing $(e.call) and fileio_$(e.call)") """ + handle_exceptions(exceptions::Vector, action) + Handles a list of thrown errors after no IO library was found working """ function handle_exceptions(exceptions::Vector, action) diff --git a/src/query.jl b/src/query.jl index 27a40979..5e17d80a 100644 --- a/src/query.jl +++ b/src/query.jl @@ -1,7 +1,9 @@ # This file contains code that helps to query from the registry to determine the format """ -`unknown(f)` returns true if the format of `f` is unknown. + unknown(f) + +Returns true if the format of `f` is unknown. """ unknown(@nospecialize(f::Union{Formatted,Type})) = unknown(formatname(f)::Symbol) unknown(name::Symbol) = name === :UNKNOWN @@ -10,18 +12,25 @@ const unknown_df = DataFormat{:UNKNOWN} """ -`info(fmt)` returns the magic bytes/extension information for -`fmt`. + info(fmt) + +Returns the magic bytes/extension information for `fmt`. """ info(@nospecialize(f::Union{Formatted,Type})) = info(formatname(f)::Symbol) info(sym::Symbol) = sym2info[sym] -"`magic(fmt)` returns the magic bytes of format `fmt`" +""" + magic(fmt) + +Returns the magic bytes of format `fmt` +""" magic(@nospecialize(fmt::Type)) = magic(formatname(fmt)::Symbol) magic(sym::Symbol) = info(sym)[1] """ -`skipmagic(s::Stream)` sets the position of `s` to be just after the magic bytes. + skipmagic(s::Stream) + +Sets the position of `s` to be just after the magic bytes. For a plain IO object, you can use `skipmagic(io, fmt)`. """ skipmagic(@nospecialize(s::Stream)) = (skipmagic(stream(s), formatname(s)::Symbol); s) @@ -200,8 +209,9 @@ end """ -`query(io, [filename])` returns a `Stream` object with information about the -format inferred from the magic bytes. + query(io, [filename]) + +Returns a `Stream` object with information about the format inferred from the magic bytes. """ function query(io::IO, filename = nothing) sym = querysym(io) diff --git a/src/registry.jl b/src/registry.jl index 1e1bf1cf..cef5fe87 100644 --- a/src/registry.jl +++ b/src/registry.jl @@ -231,7 +231,8 @@ function detectavi(io) end add_format(format"AVI", detectavi, ".avi", [idImageMagick], [idVideoIO]) -""" detectisom(io) +""" + detectisom(io) Detect ISO/IEC 14496-12 ISO/IEC base media format files. These files start with a 32-bit big-endian length, and then the string 'ftyp' which is followed by diff --git a/src/registry_setup.jl b/src/registry_setup.jl index 0adca9ff..c3b1aaad 100644 --- a/src/registry_setup.jl +++ b/src/registry_setup.jl @@ -87,9 +87,11 @@ function add_format(fmt, magic, extension, load_save_libraries...) end """ -`add_format(fmt, magic, extension)` registers a new [`DataFormat`](@ref). -For example: + add_format(fmt, magic, extension) + +registers a new [`DataFormat`](@ref). +For example: add_format(format"TIFF", (UInt8[0x4d,0x4d,0x00,0x2b], UInt8[0x49,0x49,0x2a,0x00]), [".tiff", ".tif"]) add_format(format"PNG", [0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a], ".png") add_format(format"NRRD", "NRRD", [".nrrd",".nhdr"]) @@ -154,7 +156,9 @@ function add_format(sym::Symbol, @nospecialize(magic::Function), extension) end """ -`del_format(fmt::DataFormat)` deletes `fmt` from the format registry. + del_format(fmt::DataFormat) + +deletes `fmt` from the format registry. """ del_format(@nospecialize(fmt::Type)) = del_format(formatname(fmt)::Symbol) function del_format(sym::Symbol) diff --git a/src/types.jl b/src/types.jl index a66640df..89c9f145 100644 --- a/src/types.jl +++ b/src/types.jl @@ -5,9 +5,10 @@ ## DataFormat: """ -`DataFormat{sym}()` indicates a known binary or text format of kind `sym`, -where `sym` is always a symbol. For example, a .csv file might have -`DataFormat{:CSV}()`. + DataFormat{sym}() + +Indicates a known binary or text format of kind `sym`, where `sym` +is always a symbol. For example, a .csv file might have `DataFormat{:CSV}()`. An easy way to write `DataFormat{:CSV}` is `format"CSV"`. """ @@ -27,9 +28,10 @@ formatname(::Formatted{F}) where F<:DataFormat = formatname(F) ## File: """ -`File{fmt}(filename)` indicates that `filename` is a file of known -[`DataFormat`](@ref) `fmt`. For example, `File{format"PNG"}(filename)` would indicate a PNG -file. + File{fmt}(filename) + +Indicates that `filename` is a file of known [`DataFormat`](@ref) `fmt`. +For example, `File{format"PNG"}(filename)` would indicate a PNG file. !!! compat `File{fmt}(filename)` requires FileIO 1.6 or higher. The deprecated syntax `File(fmt, filename)` works @@ -46,28 +48,33 @@ File{F}(file) where F<:DataFormat = File{F,typeof(file)}(file) # The docs are separated from the definition because of https://github.com/JuliaLang/julia/issues/34122 filename(@nospecialize(f::File)) = f.filename """ -`filename(file)` returns the filename associated with [`File`](@ref) `file`. + filename(file) + +Returns the filename associated with [`File`](@ref) `file`. """ filename(::File) file_extension(@nospecialize(f::File)) = splitext(filename(f))[2] """ -`file_extension(file)` returns the file extension associated with [`File`](@ref) `file`. + file_extension(file) + +Returns the file extension associated with [`File`](@ref) `file`. """ file_extension(::File) ## Stream: """ -`Stream{fmt}(io, filename=nothing)` indicates that the stream `io` is -written in known format [`DataFormat`](@ref) `fmt`. -For example, `Stream{format"PNG"}(io)` would indicate PNG format. + Stream{fmt}(io, filename=nothing) + +Indicates that the stream `io` is written in known format [`DataFormat`](@ref) +`fmt`. For example, `Stream{format"PNG"}(io)` would indicate PNG format. If known, the optional `filename` argument can be used to improve error messages, etc. !!! compat - `Stream{fmt}(io, ...)` requires FileIO 1.6 or higher. The deprecated syntax `Stream(fmt, io, ...)` works - on all FileIO 1.x releases. + `Stream{fmt}(io, ...)` requires FileIO 1.6 or higher. + The deprecated syntax `Stream(fmt, io, ...)` works on all FileIO 1.x releases. """ struct Stream{F <: DataFormat, IOtype <: IO, Name} <: Formatted{F} io::IOtype @@ -92,8 +99,10 @@ stream(::Stream) filename(@nospecialize(s::Stream)) = s.filename """ -`filename(stream)` returns a string of the filename -associated with [`Stream`](@ref) `stream`, or nothing if there is no file associated. + filename(stream) + +Returns a string of the filename associated with [`Stream`](@ref) `stream`, +or nothing if there is no file associated. """ filename(::Stream) @@ -103,7 +112,9 @@ function file_extension(@nospecialize(f::Stream)) splitext(fname)[2] end """ -`file_extension(file)` returns a nullable-string for the file extension associated with [`Stream`](@ref) `stream`. + file_extension(file) + +Returns a nullable-string for the file extension associated with [`Stream`](@ref) `stream`. """ file_extension(::Stream) From 1fe015dc65cdea882b0c4e225a5d663f172e2b93 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:07:34 -0400 Subject: [PATCH 04/13] Add fence language tags --- docs/src/implementing.md | 2 +- docs/src/registering.md | 2 +- docs/src/world_age_issue.md | 14 +++++++------- src/error_handling.jl | 4 ++++ src/registry_setup.jl | 20 +++++++++++--------- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/src/implementing.md b/docs/src/implementing.md index 160617aa..ccecf236 100644 --- a/docs/src/implementing.md +++ b/docs/src/implementing.md @@ -87,7 +87,7 @@ closing any streams you opened in order to read or write the file. If you are given a `Stream`, your `close` method should only do the clean up for your reader or writer type, not close the stream. -```jl +```julia struct WAVReader io::IO ownstream::Bool diff --git a/docs/src/registering.md b/docs/src/registering.md index 6063ec8e..d09b265e 100644 --- a/docs/src/registering.md +++ b/docs/src/registering.md @@ -14,7 +14,7 @@ You'll need to [`pkg> dev FileIO`](https://julialang.github.io/Pkg.jl/v1/managin Before going into detail explaining the arguments of `add_format`, here is a real example that could be used to register an I/O package for one of the [Netpbm image formats](https://en.wikipedia.org/wiki/Netpbm#File_formats): -``` +```julia add_format(format"PPMBinary", "P6", ".ppm", [:Netpbm => UUID("f09324ee-3d7c-5217-9330-fc30815ba969")] ``` diff --git a/docs/src/world_age_issue.md b/docs/src/world_age_issue.md index 172744e7..d4bcdd2d 100644 --- a/docs/src/world_age_issue.md +++ b/docs/src/world_age_issue.md @@ -13,7 +13,7 @@ thing related to image IO. To avoid such unnecessary loading latency, FileIO defers package loading until it's actually used. For instance, when you use FileIO, you'll probably observe something like this: -```julia +```julia-repl julia> using TestImages, FileIO julia> path = testimage("cameraman"; download_only=true) @@ -38,12 +38,12 @@ after initial compilation finishes) than the one you called them from. Let's demonstrate the problem concretely. In case you don't have a suitable file to play with, let's first create one: -```julia +```julia-repl julia> using IndirectArrays, ImageCore julia> img = IndirectArray(rand(1:5, 4, 4), rand(RGB, 5)) 4×4 IndirectArray{RGB{Float64}, 2, Int64, Matrix{Int64}, Vector{RGB{Float64}}}: -... +[...] julia> save("indexed_image.png", img) ``` @@ -51,7 +51,7 @@ julia> save("indexed_image.png", img) Now, **reopen a new julia REPL** (this is crucial for demonstrating the problem) and call `load` from **within a function** (this is also crucial): -```julia +```julia-repl julia> using FileIO julia> f() = size(load("indexed_image.png")) @@ -95,7 +95,7 @@ which you called `f()`. This leads to the observed error. The good news is it's easy to fix, just try calling `f()` again: -```julia +```julia-repl julia> f() (4, 4) ``` @@ -113,7 +113,7 @@ around this world-age dispatch problem. Literally, `invokelatest` dispatches the the latest world age (which may be newer than when you typed `f()` at the REPL). **In a fresh Julia session**, -```julia +```julia-repl julia> using FileIO julia> f() = Base.invokelatest(size, load("indexed_image.png")) @@ -138,7 +138,7 @@ Another solution to the world age issue is simple and doesn't have long-term dow load the needed packages**. For instance, if you're seeing world age issue complaining methods related to `IndirectArray`, then load IndirectArrays eagerly: -```julia +```julia-repl julia> using FileIO, IndirectArrays # try this on a new Julia REPL julia> f() = size(load("indexed_image.png")) diff --git a/src/error_handling.jl b/src/error_handling.jl index d688003b..6bb5286f 100644 --- a/src/error_handling.jl +++ b/src/error_handling.jl @@ -3,7 +3,9 @@ `LoaderError` should be thrown when loader library code fails, and other libraries should be given the chance to recover from the error. Reports the library name and an error message: +```julia LoaderError("ImageMagick", "Foo not available") +``` """ struct LoaderError <: Exception library::String @@ -18,7 +20,9 @@ Base.showerror(io::IO, e::LoaderError) = println(IOContext(io, :limit=>true), e. `WriterError` should be thrown when writer library code fails, and other libraries should be given the chance to recover from the error. Reports the library name and an error message: +```julia WriterError("ImageMagick", "Foo not available") +``` """ struct WriterError <: Exception library::String diff --git a/src/registry_setup.jl b/src/registry_setup.jl index c3b1aaad..0934313a 100644 --- a/src/registry_setup.jl +++ b/src/registry_setup.jl @@ -92,20 +92,22 @@ end registers a new [`DataFormat`](@ref). For example: - add_format(format"TIFF", (UInt8[0x4d,0x4d,0x00,0x2b], UInt8[0x49,0x49,0x2a,0x00]), [".tiff", ".tif"]) - add_format(format"PNG", [0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a], ".png") - add_format(format"NRRD", "NRRD", [".nrrd",".nhdr"]) +```julia +add_format(format"TIFF", (UInt8[0x4d,0x4d,0x00,0x2b], UInt8[0x49,0x49,0x2a,0x00]), [".tiff", ".tif"]) +add_format(format"PNG", [0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a], ".png") +add_format(format"NRRD", "NRRD", [".nrrd",".nhdr"]) +``` Note that extensions, magic numbers, and format-identifiers are case-sensitive. You can also specify particular packages that support the format with `add_format(fmt, magic, extension, pkgspecifiers...)`, where example `pkgspecifiers` are: - - add_format(fmt, magic, extension, [:PkgA=>UUID(...)]) # only PkgA supports the format (load & save) - add_format(fmt, magic, extension, [:PkgA=>uuidA], [:PkgB=>uuidB]) # try PkgA first, but if it fails try PkgB - add_format(fmt, magic, extension, [:PkgA=>uuidA, LOAD], [:PkgB=>uuidB]) # try PkgA first for `load`, otherwise use PkgB - add_format(fmt, magic, extension, [:PkgA=>uuidA, OSX], [:PkgB=>uuidB]) # use PkgA on OSX, and PkgB otherwise - +```julia +add_format(fmt, magic, extension, [:PkgA=>UUID(...)]) # only PkgA supports the format (load & save) +add_format(fmt, magic, extension, [:PkgA=>uuidA], [:PkgB=>uuidB]) # try PkgA first, but if it fails try PkgB +add_format(fmt, magic, extension, [:PkgA=>uuidA, LOAD], [:PkgB=>uuidB]) # try PkgA first for `load`, otherwise use PkgB +add_format(fmt, magic, extension, [:PkgA=>uuidA, OSX], [:PkgB=>uuidB]) # use PkgA on OSX, and PkgB otherwise +``` The `uuid`s are all of type `UUID` and can be obtained from the package's `Project.toml` file. You can combine `LOAD`, `SAVE`, `OSX`, `Unix`, `Windows` and `Linux` arbitrarily to narrow `pkgspecifiers`. From b231e257b20ed22508685b07cd3980ae716dc192 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:07:50 -0400 Subject: [PATCH 05/13] Remove REPL prompt from install instructions The code is more likely to be copy-pasted --- docs/src/index.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index 01862fe3..93a3bc6f 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -13,9 +13,8 @@ provide support for standard file formats through functions named Install FileIO within Julia via ```julia -julia> using Pkg - -julia> Pkg.add("FileIO") +using Pkg +Pkg.add("FileIO") ``` ## Usage From 037496ae381ee6fb39fbe6ccff7116f308f69d4d Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:08:25 -0400 Subject: [PATCH 06/13] Export info `info()` is listed in FileIO module docstring --- src/FileIO.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/FileIO.jl b/src/FileIO.jl index e957c6f0..3a19da25 100644 --- a/src/FileIO.jl +++ b/src/FileIO.jl @@ -11,6 +11,7 @@ export DataFormat, add_saver, filename, file_extension, + info, load, loadstreaming, magic, From 7319162e9ffc2ebdb8080476722f733304cf5eb3 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:10:15 -0400 Subject: [PATCH 07/13] Wrap lines --- src/query.jl | 3 ++- src/registry.jl | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/query.jl b/src/query.jl index 5e17d80a..450be678 100644 --- a/src/query.jl +++ b/src/query.jl @@ -120,7 +120,8 @@ function querysym(filename; checkfile::Bool=true) end match(io, magic) && return sym end - badmagic && error("Some formats with extension ", ext, " have no magic bytes; use `File{format\"FMT\"}(filename)` to resolve the ambiguity.") + badmagic && error("Some formats with extension ", ext, + " have no magic bytes; use `File{format\"FMT\"}(filename)` to resolve the ambiguity.") fmt = querysym_all(io)[1] return fmt === :UNKNOWN ? syms[1] : fmt end diff --git a/src/registry.jl b/src/registry.jl index cef5fe87..0574dffc 100644 --- a/src/registry.jl +++ b/src/registry.jl @@ -21,10 +21,12 @@ const idNPZ = :NPZ => UUID("15e1cf62-19b3-5cfa-8e77-841668bca605") # data formats add_format(format"JLD", (unsafe_wrap(Vector{UInt8}, "Julia data file (HDF5), version 0.0"), - unsafe_wrap(Vector{UInt8}, "Julia data file (HDF5), version 0.1")), ".jld", [:JLD => UUID("4138dd39-2aa7-5051-a626-17a0bb65d9c8")]) + unsafe_wrap(Vector{UInt8}, "Julia data file (HDF5), version 0.1")), + ".jld", [:JLD => UUID("4138dd39-2aa7-5051-a626-17a0bb65d9c8")]) add_format(format"JLD2", (unsafe_wrap(Vector{UInt8},"Julia data file (HDF5), version 0.2"), - unsafe_wrap(Vector{UInt8}, "HDF5-based Julia Data Format, version ")), ".jld2", [:JLD2 => UUID("033835bb-8acc-5ee8-8aae-3f567f8a3819")]) -add_format(format"BSON",(),".bson", [:BSON => UUID("fbb218c0-5317-5bc6-957e-2ee96dd4b1f0")]) + unsafe_wrap(Vector{UInt8}, "HDF5-based Julia Data Format, version ")), + ".jld2", [:JLD2 => UUID("033835bb-8acc-5ee8-8aae-3f567f8a3819")]) +add_format(format"BSON", (), ".bson", [:BSON => UUID("fbb218c0-5317-5bc6-957e-2ee96dd4b1f0")]) add_format(format"JLSO", (), ".jlso", [:JLSO => UUID("9da8a3cd-07a3-59c0-a743-3fdc52c30d11")]) add_format(format"NPY", "\x93NUMPY", ".npy", [idNPZ]) add_format(format"NPZ", "PK\x03\x04", ".npz", [idNPZ]) @@ -289,7 +291,12 @@ add_format(format"WAV", detectwav, ".wav", [:WAV => UUID("8149f6b0-98f6-5db9-b78 add_format(format"FLAC", "fLaC", ".flac", [:FLAC => UUID("abae9e3b-a9a0-4778-b5c6-ca109b507d99")], [idLibSndFile]) ## Profile data -add_format(format"JLPROF", [0x4a, 0x4c, 0x50, 0x52, 0x4f, 0x46, 0x01, 0x00], ".jlprof", [:FlameGraphs => UUID("08572546-2f56-4bcf-ba4e-bab62c3a3f89")]) # magic is "JLPROF" followed by [0x01, 0x00] +add_format( + format"JLPROF", + [0x4a, 0x4c, 0x50, 0x52, 0x4f, 0x46, 0x01, 0x00], + ".jlprof", + [:FlameGraphs => UUID("08572546-2f56-4bcf-ba4e-bab62c3a3f89")] +) # magic is "JLPROF" followed by [0x01, 0x00] ### Complex cases @@ -375,7 +382,8 @@ end add_format(format"bedGraph", detect_bedgraph, [".bedgraph"], [:BedgraphFiles => UUID("85eb9095-274b-55ce-be28-9e90f41ac741")]) # Handle OME-TIFFs, which are identical to normal TIFFs with the primary difference being the filename and embedded XML metadata -const tiff_magic = (UInt8[0x4d,0x4d,0x00,0x2a], UInt8[0x4d,0x4d,0x00,0x2b], UInt8[0x49,0x49,0x2a,0x00],UInt8[0x49,0x49,0x2b,0x00]) +const tiff_magic = (UInt8[0x4d,0x4d,0x00,0x2a], UInt8[0x4d,0x4d,0x00,0x2b], + UInt8[0x49,0x49,0x2a,0x00], UInt8[0x49,0x49,0x2b,0x00]) function detecttiff(io) getlength(io) >= 4 || return false magic = read!(io, Vector{UInt8}(undef, 4)) From c201e5cb219a8021bc2e81e199b7be31937cc681 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:10:37 -0400 Subject: [PATCH 08/13] Add Downloads to test dependencies Base.download() is deprecated --- Project.toml | 4 +++- test/query.jl | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index 2932ea87..4b3ec42e 100644 --- a/Project.toml +++ b/Project.toml @@ -16,6 +16,7 @@ Aqua = "0.8" CSVFiles = "1" CodecZlib = "0.5, 0.6, 0.7" ColorTypes = "0.11" +Downloads = "1.6" FilePathsBase = "0.9" HTTP = "0.6, 1" Random = "<0.0.1, 0.7, 1" @@ -26,10 +27,11 @@ Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CSVFiles = "5d742f6a-9f54-50ce-8119-2520741973ca" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "ColorTypes", "CodecZlib", "CSVFiles", "FilePathsBase", "HTTP", "Random", "Test"] +test = ["Aqua", "ColorTypes", "CodecZlib", "CSVFiles", "Downloads", "FilePathsBase", "HTTP", "Random", "Test"] diff --git a/test/query.jl b/test/query.jl index 1c231055..a569a09a 100644 --- a/test/query.jl +++ b/test/query.jl @@ -1,6 +1,7 @@ using FileIO using Test using Random +import Downloads @testset "OS" begin if Sys.islinux() @@ -399,19 +400,19 @@ let file_dir = joinpath(@__DIR__, "files"), file_path = Path(file_dir) @test typeof(q) <: File{format"AVI"} end @testset "MP4 detection" begin - f = download("https://archive.org/download/LadybirdOpeningWingsCCBYNatureClip/Ladybird%20opening%20wings%20CC-BY%20NatureClip.mp4") + f = Downloads.download("https://archive.org/download/LadybirdOpeningWingsCCBYNatureClip/Ladybird%20opening%20wings%20CC-BY%20NatureClip.mp4") q = query(f) @test typeof(q) <: File{format"MP4"} end if Base.VERSION >= v"1.6" || !Sys.iswindows() # FIXME: Windows fails to download the files on Julia 1.0 @testset "OGG detection" begin - f = download("https://upload.wikimedia.org/wikipedia/commons/8/87/Annie_Oakley_shooting_glass_balls%2C_1894.ogv") + f = Downloads.download("https://upload.wikimedia.org/wikipedia/commons/8/87/Annie_Oakley_shooting_glass_balls%2C_1894.ogv") q = query(f) @test typeof(q) <: File{format"OGG"} end @testset "MATROSKA detection" begin - f = download("https://upload.wikimedia.org/wikipedia/commons/1/13/Artist%E2%80%99s_impression_of_the_black_hole_inside_NGC_300_X-1_%28ESO_1004c%29.webm") + f = Downloads.download("https://upload.wikimedia.org/wikipedia/commons/1/13/Artist%E2%80%99s_impression_of_the_black_hole_inside_NGC_300_X-1_%28ESO_1004c%29.webm") q = query(f) @test typeof(q) <: File{format"MATROSKA"} end From c8f3ba1d241c29ab1afe184024d11eb27dee36b5 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:11:05 -0400 Subject: [PATCH 09/13] Add markdown ref links to types and methods --- src/FileIO.jl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/FileIO.jl b/src/FileIO.jl index 3a19da25..c362f2f7 100644 --- a/src/FileIO.jl +++ b/src/FileIO.jl @@ -42,26 +42,26 @@ include("registry.jl") `FileIO` API (brief summary, see individual functions for more detail): - `format"PNG"`: specifies a particular defined format -- `File{fmt}` and `Stream{fmt}`: types of objects that declare that a resource has a particular format `fmt` +- [`File{fmt}`](@ref) and [`Stream{fmt}`](@ref): types of objects that declare that a resource has a particular format `fmt` -- `load([filename|stream])`: read data in formatted file, inferring the format +- [`load([filename|stream])`](@ref): read data in formatted file, inferring the format - `load(File{format"PNG"}(filename))`: specify the format manually -- `loadstreaming([filename|stream])`: similar to `load`, except that it returns an object that can be read from -- `save(filename, data...)` for similar operations involving saving data -- `savestreaming([filename|stream])`: similar to `save`, except that it returns an object that can be written to +- [`loadstreaming([filename|stream])`](@ref): similar to `load`, except that it returns an object that can be read from +- [`save(filename, data...)`](@ref) for similar operations involving saving data +- [`savestreaming([filename|stream])`](@ref): similar to `save`, except that it returns an object that can be written to - `io = open(f::File, args...)` opens a file -- `io = stream(s::Stream)` returns the IOStream from the query object `s` +- [`io = stream(s::Stream)`](@ref stream) returns the IOStream from the query object `s` -- `query([filename|stream])`: attempt to infer the format of `filename` -- `unknown(q)` returns true if a query can't be resolved -- `skipmagic(io, fmt)` sets the position of `io` to just after the magic bytes -- `magic(fmt)` returns the magic bytes for format `fmt` -- `info(fmt)` returns `(magic, extensions)` for format `fmt` +- [`query([filename|stream])`](@ref): attempt to infer the format of `filename` +- [`unknown(q)`](@ref) returns true if a query can't be resolved +- [`skipmagic(io, fmt)`](@ref) sets the position of `io` to just after the magic bytes +- [`magic(fmt)`](@ref) returns the magic bytes for format `fmt` +- [`info(fmt)`](@ref) returns `(magic, extensions)` for format `fmt` -- `add_format(fmt, magic, extension, libraries...)`: register a new format -- `add_loader(fmt, :Package)`: indicate that `Package` supports loading files of type `fmt` -- `add_saver(fmt, :Package)`: indicate that `Package` supports saving files of type `fmt` +- [`add_format(fmt, magic, extension, libraries...)`](@ref): register a new format +- [`add_loader(fmt, :Package)`](@ref): indicate that `Package` supports loading files of type `fmt` +- [`add_saver(fmt, :Package)`](@ref): indicate that `Package` supports saving files of type `fmt` """ FileIO From 7c4b88aa5a9147b04c45f03d3097ef70513614ff Mon Sep 17 00:00:00 2001 From: abhro <5664668+abhro@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:06:46 -0400 Subject: [PATCH 10/13] Loosen Downloads compat for tests --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 4b3ec42e..d8f96ad5 100644 --- a/Project.toml +++ b/Project.toml @@ -16,7 +16,7 @@ Aqua = "0.8" CSVFiles = "1" CodecZlib = "0.5, 0.6, 0.7" ColorTypes = "0.11" -Downloads = "1.6" +Downloads = "1.4" FilePathsBase = "0.9" HTTP = "0.6, 1" Random = "<0.0.1, 0.7, 1" From a33d30df84639e3fd312afc0a2e426b9a1e2d732 Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:51:21 -0400 Subject: [PATCH 11/13] Revert "Export info" This reverts commit 037496ae381ee6fb39fbe6ccff7116f308f69d4d. --- src/FileIO.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/FileIO.jl b/src/FileIO.jl index c362f2f7..b4a555c6 100644 --- a/src/FileIO.jl +++ b/src/FileIO.jl @@ -11,7 +11,6 @@ export DataFormat, add_saver, filename, file_extension, - info, load, loadstreaming, magic, From d21472a24f23283688a021e893ddd8d6c306cecb Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:54:24 -0400 Subject: [PATCH 12/13] Add docstring for info() to API reference page --- docs/src/reference.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/reference.md b/docs/src/reference.md index d6ef5a55..f5991bb6 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -4,3 +4,6 @@ Modules = [FileIO] Private = false ``` +```@docs +FileIO.info +``` From f0be08c3cbe4ce5b6fe2aa793b2c09d437eec25d Mon Sep 17 00:00:00 2001 From: Abhro R <5664668+abhro@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:54:43 -0400 Subject: [PATCH 13/13] Reformat docstring for stream() --- src/types.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/types.jl b/src/types.jl index 89c9f145..54697290 100644 --- a/src/types.jl +++ b/src/types.jl @@ -94,7 +94,11 @@ Stream{F}(file::File, io::IO) where {F<:DataFormat} = Stream{F, typeof(io)}(file Stream(file::File{F}, io::IO) where {F<:DataFormat} = Stream{F}(io, filename(file)) stream(@nospecialize(s::Stream)) = s.io -"`stream(s)` returns the stream associated with [`Stream`](@ref) `s`" +""" + stream(s) + +Returns the stream associated with [`Stream`](@ref) `s`. +""" stream(::Stream) filename(@nospecialize(s::Stream)) = s.filename