Merge pull request #5960 from hannesm/migrate

new subcommand: opam admin migrate-extrafiles

doc/man/dune_man.ml

@@ -38,5 +38,5 @@ let () =
                 \  (section man)\n\
                 \  (package opam)\n\
                 \  (files%s))\n"
-    (String.concat " "
+    (String.concat ""
        (List.map (Printf.sprintf "\n    %s-%s.1" target_basename) topics))

doc/man/opam-admin-topics.inc

@@ -10,6 +10,16 @@
                  (diff opam-admin-help.err %{dep:opam-admin-help.0})))
   (package opam))
 
+(rule
+  (with-stdout-to opam-admin-migrate-extrafiles.0 (echo "")))
+(rule
+  (targets opam-admin-migrate-extrafiles.1 opam-admin-migrate-extrafiles.err)
+  (deps using-built-opam)
+  (action (progn (with-stderr-to opam-admin-migrate-extrafiles.err
+                   (with-stdout-to opam-admin-migrate-extrafiles.1 (run %{bin:opam} admin migrate-extrafiles --help=groff)))
+                 (diff opam-admin-migrate-extrafiles.err %{dep:opam-admin-migrate-extrafiles.0})))
+  (package opam))
+
 (rule
   (with-stdout-to opam-admin-update-extrafiles.0 (echo "")))
 (rule
@@ -134,16 +144,17 @@
   (section man)
   (package opam)
   (files
-    opam-admin-help.1 
-    opam-admin-update-extrafiles.1 
-    opam-admin-add-hashes.1 
-    opam-admin-add-constraint.1 
-    opam-admin-filter.1 
-    opam-admin-list.1 
-    opam-admin-compare-versions.1 
-    opam-admin-check.1 
-    opam-admin-lint.1 
-    opam-admin-upgrade.1 
-    opam-admin-cache.1 
-    opam-admin-make.1 
+    opam-admin-help.1
+    opam-admin-migrate-extrafiles.1
+    opam-admin-update-extrafiles.1
+    opam-admin-add-hashes.1
+    opam-admin-add-constraint.1
+    opam-admin-filter.1
+    opam-admin-list.1
+    opam-admin-compare-versions.1
+    opam-admin-check.1
+    opam-admin-lint.1
+    opam-admin-upgrade.1
+    opam-admin-cache.1
+    opam-admin-make.1
     opam-admin-index.1))

doc/man/opam-topics.inc

@@ -294,32 +294,32 @@
   (section man)
   (package opam)
   (files
-    opam-help.1 
-    opam-admin.1 
-    opam-lock.1 
-    opam-clean.1 
-    opam-lint.1 
-    opam-source.1 
-    opam-unpin.1 
-    opam-pin.1 
-    opam-switch.1 
-    opam-remote.1 
-    opam-repository.1 
-    opam-env.1 
-    opam-exec.1 
-    opam-config.1 
-    opam-option.1 
-    opam-var.1 
-    opam-upgrade.1 
-    opam-update.1 
-    opam-reinstall.1 
-    opam-uninstall.1 
-    opam-remove.1 
-    opam-install.1 
-    opam-info.1 
-    opam-show.1 
-    opam-why.1 
-    opam-tree.1 
-    opam-search.1 
-    opam-list.1 
+    opam-help.1
+    opam-admin.1
+    opam-lock.1
+    opam-clean.1
+    opam-lint.1
+    opam-source.1
+    opam-unpin.1
+    opam-pin.1
+    opam-switch.1
+    opam-remote.1
+    opam-repository.1
+    opam-env.1
+    opam-exec.1
+    opam-config.1
+    opam-option.1
+    opam-var.1
+    opam-upgrade.1
+    opam-update.1
+    opam-reinstall.1
+    opam-uninstall.1
+    opam-remove.1
+    opam-install.1
+    opam-info.1
+    opam-show.1
+    opam-why.1
+    opam-tree.1
+    opam-search.1
+    opam-list.1
     opam-init.1))

master_changes.md

@@ -134,6 +134,7 @@ users)
   * [BUG] Fix `opam admin check` in the presence of the `with-dev-setup` variable [#6331 @kit-ty-kate - fix #6329]
   * ✘ The `-i`/`--ignore-test-doc` argument has been removed from `opam admin check` [#6335 @kit-ty-kate]
   * ✘ `opam admin check` now sets `with-test` and `with-doc` to `false` instead of `true` [#6335 @kit-ty-kate]
+  * ◈ Add `opam admin migrate-extrafiles` which moves all extra-files of an existing opam repository into extra-sources [#5960 @hannesm @rjbou @kit-ty-kate]
 
 ## Opam installer
 
@@ -219,6 +220,7 @@ users)
 ## opam-client
   * `OpamArg.InvalidCLI`: export exception [#6150 @rjbou]
   * `OpamArg`: export `require_checksums` and `no_checksums`, that are shared with `build_options` [#5563 @rjbou]
+  * `OpamArg.hash_kinds`: was added [#5960 @kit-ty-kate]
   * `OpamRepositoryCommand.switch_repos`: expose the function [#5014 @kit-ty-kate]
 
 ## opam-repository
@@ -240,6 +242,7 @@ users)
 ## opam-core
   * `OpamConsole`: Replace `black` text style (unused and not very readable) by `gray` [#6358 @kit-ty-kate]
   * `OpamConsole.pause`: Ensure the function always prints a newline character at the end [#6376 @kit-ty-kate]
+  * `OpamHash.all_kinds`: was added, which returns the list of all possible values of `OpamHash.kind` [#5960 @kit-ty-kate]
   * `OpamStd.List.split`: Improve performance [#6210 @kit-ty-kate]
   * `OpamStd.Sys.{get_terminal_columns,uname,getconf,guess_shell_compat}`: Harden the process calls to account for failures [#6230 @kit-ty-kate - fix #6215]
   * `OpamStd.Sys.getconf`: was removed, replaced by `get_long_bit` [#6217 @kit-ty-kate]

src/client/opamAdminCommand.ml

@@ -343,13 +343,10 @@ let update_extrafiles_command cli =
         field as requested."
   ]
   in
-  let hash_kinds = [`MD5; `SHA256; `SHA512] in
   let hash_type_arg =
     OpamArg.mk_opt ~cli OpamArg.(cli_from cli2_2) ["hash"]
       "HASH_ALGO" "The hash, or hashes to be added"
-      Arg.(some (enum
-                   (List.map (fun k -> OpamHash.string_of_kind k, k)
-                      hash_kinds))) None
+      (Arg.some OpamArg.hash_kinds) None
   in
   let packages =
     OpamArg.mk_opt ~cli OpamArg.(cli_from cli2_2) ["p";"packages"]
@@ -425,6 +422,121 @@ let update_extrafiles_command cli =
     Term.(const cmd $ global_options cli
           $ hash_type_arg $ packages)
 
+let migrate_extrafiles_command_doc =
+  "Move extra-files to extra-source."
+let migrate_extrafiles_command cli =
+  let command = "migrate-extrafiles" in
+  let doc = migrate_extrafiles_command_doc in
+  let man = [
+    `S Manpage.s_description;
+    `P "This command scans through package definitions, and migrates all \
+        $(i,extra-files) fields to $(i,extra-source) as requested."
+  ] in
+  let hash_type_arg =
+    OpamArg.mk_opt ~cli OpamArg.(cli_from cli2_4) ["hash"]
+      "HASH_ALGO" "The hash, or hashes to be added"
+      OpamArg.hash_kinds `SHA256
+  in
+  let packages =
+    OpamArg.mk_opt ~cli OpamArg.(cli_from cli2_4) ["p";"packages"]
+      "PACKAGES" "Only add extra files for the given packages"
+      Arg.(list OpamArg.package) []
+  in
+  let local_dir_arg =
+    let doc =
+      Arg.info ~docv:"DIR"
+        ~doc:"Name of the local directory where to put the extra-files. \
+              They will be put into \
+              DIR/patches/<pkgname>/<pkgname.version>/filenameYY" []
+    in
+    Arg.(required & pos 0 (some OpamArg.dirname) None & doc)
+  and url_prefix_arg =
+    let doc =
+      Arg.info ~docv:"URL"
+        ~doc:"Prefix of the URL to emit into extra-sources." []
+    in
+    Arg.(required & pos 1 (some OpamArg.url) None & doc)
+  in
+  let cmd global_options kind packages local_dir url_prefix () =
+    OpamArg.apply_global_options cli global_options;
+    let repo_root = checked_repo_root () in
+    let pkg_prefixes = packages_with_prefixes repo_root packages in
+    let compute ?kind file =
+      OpamHash.compute ?kind (OpamFilename.to_string file)
+    in
+    let has_error =
+      OpamPackage.Map.fold (fun nv prefix has_error ->
+          let opam_file = OpamRepositoryPath.opam repo_root prefix nv in
+          let opam = OpamFile.OPAM.read opam_file in
+          let has_error =
+            let urlfile = OpamRepositoryPath.url repo_root prefix nv in
+            if OpamFile.exists urlfile then
+              (OpamConsole.warning
+                 "Not updating external URL file at %s, \
+                  consider upgrading your repository with 'opam admin upgrade'"
+                 (OpamFile.to_string urlfile);
+               true)
+            else has_error
+          in
+          let files_dir = OpamRepositoryPath.files repo_root prefix nv in
+          if OpamFilename.exists_dir files_dir then
+            (let files =
+               OpamFilename.rec_files files_dir
+               |> List.map (fun file ->
+                   file,
+                   OpamFilename.Base.of_string
+                     (OpamFilename.remove_prefix files_dir file))
+             in
+             match files with
+             | [] -> has_error
+             | _ ->
+               let hashes =
+                 List.map (fun (file, base) ->
+                     let xhash = compute ~kind file in
+                     file, base, xhash)
+                   files
+               in
+               let patch_prefix = "patches" in
+               let extra_sources =
+                 List.map (fun (src, base, hash) ->
+                     let name = OpamPackage.name_to_string nv in
+                     let name_version = OpamPackage.to_string nv in
+                     let dst_dir = OpamFilename.Op.(
+                         local_dir / patch_prefix / name / name_version
+                       ) in
+                     let dst = OpamFilename.create dst_dir base in
+                     (* copy file to dst, remove from files *)
+                     OpamFilename.mkdir dst_dir;
+                     OpamFilename.copy ~src ~dst;
+                     let url =
+                       let file = OpamFilename.Base.to_string base in
+                       OpamFile.URL.create ~checksum:[hash]
+                         OpamUrl.Op.(url_prefix / patch_prefix
+                                     / name / name_version/ file)
+                     in
+                     base, url)
+                   hashes
+               in
+               OpamFilename.cleandir files_dir;
+               OpamFilename.rmdir_cleanup files_dir;
+               let my_extra_sources = OpamFile.OPAM.extra_sources opam in
+               let opam1 =
+                 OpamFile.OPAM.with_extra_sources
+                   (my_extra_sources @ extra_sources) opam
+               in
+               let opam1 = OpamFile.OPAM.with_extra_files_opt None opam1 in
+               OpamFile.OPAM.write_with_preserved_format opam_file opam1;
+               has_error)
+          else
+            has_error)
+        pkg_prefixes false
+    in
+    if has_error then OpamStd.Sys.exit_because `Sync_error
+    else OpamStd.Sys.exit_because `Success
+  in
+  OpamArg.mk_command ~cli OpamArg.(cli_from cli2_4) command ~doc ~man
+    Term.(const cmd $ global_options cli $ hash_type_arg $ packages
+          $ local_dir_arg $ url_prefix_arg)
 
 let add_hashes_command_doc =
   "Add archive hashes to an opam repository."
@@ -438,12 +550,9 @@ let add_hashes_command cli =
         in <opamroot>/download-cache/hash-cache for subsequent runs.";
   ]
   in
-  let hash_kinds = [`MD5; `SHA256; `SHA512] in
   let hash_types_arg =
     OpamArg.nonempty_arg_list "HASH_ALGO" "The hash, or hashes to be added"
-      (Arg.enum
-         (List.map (fun k -> OpamHash.string_of_kind k, k)
-            hash_kinds))
+      OpamArg.hash_kinds
   in
   let packages =
     OpamArg.mk_opt ~cli OpamArg.(cli_from cli2_1) ["p";"packages"]
@@ -455,7 +564,7 @@ let add_hashes_command cli =
       "Replace the existing hashes rather than adding to them"
   in
   let create_hash_tables hash_cache_dir =
-    let t = Hashtbl.create (List.length hash_kinds) in
+    let t = Hashtbl.create (List.length OpamHash.all_kinds) in
     List.iter (fun k1 ->
         List.iter (fun k2 ->
             if k1 <> k2 then (
@@ -474,9 +583,9 @@ let add_hashes_command cli =
                | _ -> failwith ("Bad cache at "^OpamFile.to_string cache_file));
               Hashtbl.add t (k1,k2) (cache_file, t_mapping);
             ))
-          hash_kinds
+          OpamHash.all_kinds
       )
-      hash_kinds;
+      OpamHash.all_kinds;
     t
   in
   let save_hashes hash_tables =
@@ -1293,6 +1402,7 @@ let admin_subcommands cli =
     add_constraint_command cli;
     add_hashes_command cli;
     update_extrafiles_command cli;
+    migrate_extrafiles_command cli;
     help;
   ]
 

src/client/opamArg.ml

@@ -1068,6 +1068,10 @@ let opamlist_columns =
   in
   parse, print
 
+let hash_kinds =
+  Arg.enum
+    (List.map (fun k -> OpamHash.string_of_kind k, k) OpamHash.all_kinds)
+
 let mk_flag ~cli validity ?(section=Manpage.s_options) flags name =
   mk_flag ~cli validity ~section flags name
 

src/client/opamArg.mli

@@ -299,6 +299,8 @@ val warn_selector: (int * [`Enable | `Disable | `EnableError]) list Arg.conv
 
 val opamlist_columns: OpamListCommand.output_format list Arg.conv
 
+val hash_kinds : OpamHash.kind Arg.conv
+
 (** {2 Subcommands} *)
 
 type 'a subcommand = validity * string * 'a * string list * string

src/core/opamHash.ml

@@ -11,6 +11,7 @@
 type kind = [ `MD5 | `SHA256 | `SHA512 ]
 
 let default_kind = `MD5
+let all_kinds = [`MD5; `SHA256; `SHA512]
 
 type t = kind * string
 

src/core/opamHash.mli

@@ -15,6 +15,9 @@ type t
 
 val kind: t -> kind
 
+(** The list of all the possible values of kind *)
+val all_kinds : kind list
+
 (** The value of the hash, as a string of hexadecimal characters *)
 val contents: t -> string