feat(dependencies): Remove dependencies on Bibliography.jl and co.

Project.toml

@@ -6,8 +6,6 @@ version = "0.1.0"
 [deps]
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 BaseDirs = "18cc8868-cbac-4acf-b575-c8ff214dc66f"
-BibInternal = "2027ae74-3657-4b95-ae00-e2f7d55c3e64"
-Bibliography = "f1be7e48-bf82-45af-a471-ae754a193061"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
@@ -21,8 +19,6 @@ YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 Aqua = "0.8"
 ArgParse = "1"
 BaseDirs = "1"
-BibInternal = "0.3"
-Bibliography = "0.2"
 DataFrames = "1"
 Dates = "1"
 DocStringExtensions = "0.9"

README.md

@@ -4,3 +4,59 @@
 [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://klafyvel.github.io/ZoteroToPapis.jl/dev/)
 [![Build Status](https://github.com/klafyvel/ZoteroToPapis.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/klafyvel/ZoteroToPapis.jl/actions/workflows/CI.yml?query=branch%3Amain)
 [![Aqua](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)
+
+Migrate Zotero+BetterBibtex database to Papis.
+
+# Installation
+
+You need a working Julia installation, then simply `] add https://github.com/Klafyvel/ZoteroToPapis.jl`.
+
+If you're gonna use the script often, you may add an alias:
+```bash
+alias ZoteroToPapis="julia -e 'using ZoteroToPapis; exit(ZoteroToPapis.main(ARGS))' --"
+```
+If you are using Julia 1.12 or later, it can simply be
+```bash
+alias ZoteroToPapis="julia -m ZoteroToPapis"
+```
+
+# Usage
+
+```
+usage: ZoteroToPapis [--zotero-db ZOTERO-DB]
+                     [--better-bibtex-db BETTER-BIBTEX-DB]
+                     [--zotero-storage ZOTERO-STORAGE]
+                     [--papis-root PAPIS-ROOT] [--keep-external]
+                     [--no-progress] [--no-papis-update]
+                     [--no-papis-doctor] [--no-duplication-mitigation]
+                     [--version] [-h]
+
+Migrate Zotero+BetterBibtex database to Papis.
+
+optional arguments:
+  --zotero-db ZOTERO-DB
+                        Path of the Zotero database. (default:
+                        "~/Zotero/zotero.sqlite")
+  --better-bibtex-db BETTER-BIBTEX-DB
+                        Path of the Better Bibtex database. (default:
+                        "~/Zotero/better-bibtex.sqlite")
+  --zotero-storage ZOTERO-STORAGE
+                        Path of the Zotero storage. (default:
+                        "~/Zotero/storage")
+  --papis-root PAPIS-ROOT
+                        Path of the Papis root. (default:
+                        "XDG_DOCUMENTS_DIR/papers")
+  --keep-external       If enabled, keep files not in Zotero storage
+                        where they are.
+  --no-progress         If set, disable progress display.
+  --no-papis-update     If set, disable papis update after import.
+  --no-papis-doctor     If set, disable papis doctor after import (the
+                        doctor requires papis update).
+  --no-duplication-mitigation
+                        If set, disable the mitigation measures when
+                        trying to import a duplicate.
+  --version             show version information and exit
+  -h, --help            show this help message and exit
+
+Please, report bugs at https://github.com/klafyvel/ZoteroToPapis.jl.
+```

src/ZoteroToPapis.jl

@@ -7,12 +7,13 @@ using DataFrames
 using OrderedCollections
 using SHA
 using DocStringExtensions
-using BibInternal, BaseDirs
-using Bibliography, Dates
+using BaseDirs
+using Dates
 using ArgParse
 
 include("defaults.jl")
 include("database_queries.jl")
+include("types.jl")
 include("zotero_importer.jl")
 include("papis_exporter.jl")
 
@@ -23,9 +24,11 @@ Run `papis cache clear` and optionally `papis doctor --all-checks --all --fix`
 to fix all the mistakes that we've made. ;)
 """
 function papis_update(papis, doctor)
-    run(`$papis cache clear`)
+    @info "Running $papis cache reset"
+    run(`$papis cache reset`)
     if doctor
-        run(`$papis doctor --all-checks --all`)
+        @info "Running $papis doctor --all-checks --all --fix"
+        run(`$papis doctor --all-checks --all --fix`)
     end
 end
 
@@ -34,10 +37,10 @@ include("main.jl")
 if Base.VERSION >= v"1.4.2"
     @assert precompile(main, (Vector{String},))
     @assert precompile(parse_arguments, (Vector{String},))
-    Base.precompile(Tuple{typeof(Core.kwcall), @NamedTuple{zotero_db::SQLite.DB, betterbibtex_db::SQLite.DB, papis_root::String, zotero_storage::String, showprogress::Bool}, typeof(create_bibinternals)})   # time: 0.86989796
-    Base.precompile(Tuple{typeof(Core.kwcall), @NamedTuple{papis_root::String, move_external::Bool, append_zotero_id_on_duplicate::Bool}, typeof(export_bibinternal), BibInternal.Entry, Set{Any}, Vector{Any}})   # time: 0.12025428
-    Base.precompile(Tuple{var"#111#threadsfor_fun#14"{var"#111#threadsfor_fun#13#15"{String, Bool, Bool, Progress, Vector{Any}}}, Int64})   # time: 0.11337276
-    Base.precompile(Tuple{typeof(Core.kwcall), @NamedTuple{papis_root::String, showprogress::Bool, move_external::Bool, append_zotero_id_on_duplicate::Bool}, typeof(export_bibinternals), Vector{Any}})   # time: 0.016359227
+    Base.precompile(Tuple{typeof(Core.kwcall), @NamedTuple{zotero_db::SQLite.DB, betterbibtex_db::SQLite.DB, papis_root::String, zotero_storage::String, showprogress::Bool}, typeof(create_zotero_entries)})   # time: 0.86989796
+    Base.precompile(Tuple{typeof(Core.kwcall), @NamedTuple{papis_root::String, move_external::Bool, append_zotero_id_on_duplicate::Bool}, typeof(export_zotero_entry), ZoteroEntry})   # time: 0.12025428
+    # Base.precompile(Tuple{var"#111#threadsfor_fun#14"{var"#111#threadsfor_fun#13#15"{String, Bool, Bool, Progress, Vector{Any}}}, Int64})   # time: 0.11337276
+    Base.precompile(Tuple{typeof(Core.kwcall), @NamedTuple{papis_root::String, showprogress::Bool, move_external::Bool, append_zotero_id_on_duplicate::Bool}, typeof(export_zotero_entries), Vector{ZoteroEntry}})   # time: 0.016359227
     Base.precompile(Tuple{typeof(Core.kwcall), @NamedTuple{zotero_storage::String, db::SQLite.DB, dbbb::SQLite.DB, papis_root::String}, typeof(prepare_item), Int64, String, String})   # time: 0.01258725
     Base.precompile(Tuple{typeof(get_collection_path), SQLite.DB, Int64})   # time: 0.00402853
 end

src/database_queries.jl

@@ -1,8 +1,8 @@
 """
 $(SIGNATURES)
 
-Get the list of items in the Zotero database, excluding attachments. The returned
-`DataFrame` has the following columns: 
+Get the list of items in the Zotero database, excluding attachments, notes and 
+annotations. The returned `DataFrame` has the following columns: 
 - `itemID`
 - `typeName`
 - `key`
@@ -21,6 +21,8 @@ function get_zotero_items(db)
   WHERE
     itemTypes.itemTypeID = items.itemTypeID
     AND itemTypes.typeName != "attachment"
+    AND itemTypes.typeName != "annotation"
+    AND itemTypes.typeName != "note"
     AND deletedItems.itemID IS NULL
   ORDER BY
     items.itemID

src/main.jl

@@ -1,3 +1,9 @@
+  @static if VERSION >= v"1.8"
+    errno::Cint = 0
+else
+    errno = 0
+end
+
 function parse_arguments(args)
     settings = ArgParseSettings(
         prog = "ZoteroToPapis",
@@ -66,11 +72,12 @@ function main(args)
     append_zotero_id_on_duplicate = !parsed_arguments["no-duplication-mitigation"]
     zotero_db = SQLite.DB(zoterodb_path)
     betterbibtex_db = SQLite.DB(betterbibtexdb_path)
-    bibinternals = create_bibinternals(; zotero_db, betterbibtex_db, papis_root, zotero_storage, showprogress)
-    export_bibinternals(bibinternals; papis_root, showprogress, move_external, append_zotero_id_on_duplicate)
+    entries = create_zotero_entries(; zotero_db, betterbibtex_db, papis_root, zotero_storage, showprogress)
+    export_zotero_entries(entries; papis_root, showprogress, move_external, append_zotero_id_on_duplicate)
     if papisupdate
         papis_update(default_papis(), papisdoctor)
     end
+    errno
 end
 
 # Only works for Julia 1.12

src/papis_exporter.jl

@@ -1,36 +1,26 @@
 """
 $(SIGNATURES)
 
-Export a `BibInternal.jl` entry to Papis, handling some edge cases (duplicates, files not existing...)
+Export a [`ZoteroEntry`](@ref) entry to Papis, handling some edge cases (duplicates, files not existing...)
 """
-function export_bibinternal(entry, tags, files; papis_root, move_external, append_zotero_id_on_duplicate)
-    filepath = papis_directory(papis_root, entry.date.year, entry.id)
+function export_zotero_entry(entry; papis_root, move_external, append_zotero_id_on_duplicate)
+    filepath = papis_directory(papis_root, entry.year, entry.citationkey)
     if isdir(filepath) && append_zotero_id_on_duplicate
-        filepath = papis_directory(papis_root, entry.date.year, entry.id * "_" * entry.fields["id"])
+        filepath = papis_directory(papis_root, entry.year, entry.citationkey * "_" * entry.zotero_extra_fields["database_id"])
     end
+    problematic_path = []
     if isdir(filepath)
-        @warn "Unresolved duplicate." filepath
+        push!(problematic_path, (filepath, "Unresolved duplicate"))
     end
     mkpath(filepath)
-    dict = OrderedDict(
-        "ref" => entry.id,
-        "title" => entry.title,
-        "bookTitle" => entry.booktitle,
-        "author" => Bibliography.names_to_strings(entry.authors),
-        "type" => entry.type,
-        "files" => String[],
-    )
-    Bibliography.in_to_bibtex!(dict, entry.in)
-    Bibliography.date_to_bibtex!(dict, entry.date)
-    Bibliography.access_to_bibtex!(dict, entry.access)
-    dict["zotero_export"] = Dates.now()
-    dict["zotero_fields"] = entry.fields
-    for file in files
+    biblatex_entry = BibLaTeXEntry(entry)
+    dict = to_dict(biblatex_entry)
+    for (i, file) in enumerate(biblatex_entry.files)
         old_file = joinpath(file.filepath, file.filename)
         if !file.isexternal || move_external
             new_file = joinpath(filepath, file.filename)
             if !isfile(old_file)
-                @warn "Attachment does not exist." entry.id old_file
+                push!(problematic_path, (old_file, "Attachment does not exist."))
                 continue
             end
             if isfile(new_file)
@@ -41,17 +31,16 @@ function export_bibinternal(entry, tags, files; papis_root, move_external, appen
                     sha2_256(f)
                 end
                 if existing_file_sha == old_file_sha
-                    @debug "Duplicate file avoided." entry.id entry.fields["id"]
                     continue
                 else
                     new_file_name, new_file_ext = splitext(new_file)
                     new_file = new_file_name * "_" * join(string.(old_file_sha)) * new_file_ext
                 end
             end
             cp(old_file, new_file)
-            push!(dict["files"], basename(new_file))
+            dict["files"][i] = basename(new_file)
         elseif file.isexternal
-            push!(dict["files"], old_file)
+            dict["files"][i] = old_file
         end
     end
     for k in keys(dict)
@@ -62,23 +51,44 @@ function export_bibinternal(entry, tags, files; papis_root, move_external, appen
     mkpath(filepath)
     info_file = joinpath(filepath, "info.yaml")
     if isfile(info_file)
-        @warn "Info file already exist: possible duplicate. Overwriting." info_file entry.fields["id"]
+        push!(problematic_path, (info_file, "Info file already exist: possible duplicate. Overwriting."))
     end
     YAML.write_file(info_file, dict)
+    problematic_path
 end
 
 """
 $(SIGNATURES)
 
-Export the entries generated by [`create_bibinternals`](@ref).
+Export the entries generated by [`create_zotero_entries`](@ref).
 """
-function export_bibinternals(entries; papis_root, showprogress, move_external, append_zotero_id_on_duplicate)
+function export_zotero_entries(entries; papis_root, showprogress, move_external, append_zotero_id_on_duplicate)
     p = Progress(length(entries); enabled = showprogress, desc = "Exporting $(length(entries)) item(s)...")
     files_created = []
-    files_added = Channel(40)
+    problematic_paths = Channel(length(entries))
     # This time we can do it concurrently!
-    Threads.@threads for (entry, tags, files) in entries
-        export_bibinternal(entry, tags, files; papis_root, move_external, append_zotero_id_on_duplicate)
-        next!(p, showvalues = [(:Key, entry.id)])
+    Threads.@threads for entry in entries
+        problematic_path = export_zotero_entry(entry; papis_root, move_external, append_zotero_id_on_duplicate)
+        if !isempty(problematic_path)
+            put!(problematic_paths, (entry.citationkey, problematic_path))
+        end
+        next!(p, showvalues = [(:Key, entry.citationkey)])
+    end
+    if !isempty(problematic_paths)
+        printstyled(stderr, "Warning:", bold = true, color = :yellow)
+        println(stderr, " encountered $(length(problematic_paths.data)) errors while exporting database.")
+    end
+    while !isempty(problematic_paths)
+        key, paths = take!(problematic_paths)
+        printstyled(stderr, "  ⋅ Key ")
+        printstyled(stderr, "$key", italic = true)
+        print(stderr, ":\n")
+        for (path, msg) in paths
+            print(stderr, "    - ")
+            printstyled(stderr, msg, color = :red)
+            print(stderr, "\n      ")
+            printstyled(stderr, path, color = :yellow)
+            print(stderr, "\n")
+        end
     end
 end