ocamlbuild - missing dependencies at byte-code link with mlpack

["Grégoire Sutre" <gregoire.sutre@labri.fr>]

[-- Attachment #1: Type: text/plain, Size: 2367 bytes --]

Hi,

I obtain an error `Reference to undefined global ...' with ocamlbuild in
the following situation (simplified example, attached as example.diff).

[File]				[Contents]

example
|-- Main.ml			let _ = Pack.Packed.g ()
|
|-- Pack.mlpack			pack/Packed
|
|-- _tags			<lib>: include
|				<pack/*.cmx>: for-pack(Pack)
|
|-- lib
|   |-- Lib.ml			let f () = ()
|   `-- Lib.mli			val f : unit -> unit
`-- pack
       `-- Packed.ml		let g () = Lib.f ()


$ ocamlbuild -classic-display Main.byte
/usr/pkg/bin/ocamldep.opt -modules Main.ml > Main.ml.depends
/usr/pkg/bin/ocamldep.opt -modules pack/Packed.ml > pack/Packed.ml.depends
/usr/pkg/bin/ocamldep.opt -modules lib/Lib.mli > lib/Lib.mli.depends
/usr/pkg/bin/ocamlc.opt -c -I lib -o lib/Lib.cmi lib/Lib.mli
/usr/pkg/bin/ocamlc.opt -c -I pack -I lib -o pack/Packed.cmo pack/Packed.ml
/usr/pkg/bin/ocamlc.opt -pack pack/Packed.cmo -o Pack.cmo
/usr/pkg/bin/ocamlc.opt -c -I lib -o Main.cmo Main.ml
/usr/pkg/bin/ocamlc.opt Pack.cmo Main.cmo -o Main.byte
+ /usr/pkg/bin/ocamlc.opt Pack.cmo Main.cmo -o Main.byte
File "_none_", line 1, characters 0-1:
Error: Error while linking Pack.cmo:
Reference to undefined global `Lib'
Command exited with code 2.

This is with ocaml 3.11.1, but I obtain the same result with ocaml
3.12.0+beta1.


If I read the code correctly, this comes from the function prepare_link
in ocaml_compiler.ml which derives the dependencies from the files
.ml.depends and .mli.depends.  But these files are not generated for
packs, hence the required dependencies are not build.

The attached small patch (patch-ocamlbuild-link-pack-deps.diff) simply
treats the .mlpack files as dependency files for packs, which fixes
the issue.  This has been tested on a larger project with more nested
dependencies.  But I'm new to ocamlbuild, and I may have overlooked
something.


Alternatively, the problem can be fixed with an ocamlbuild plugin that
generates appropriate %.ml.depends from %.mlpack. In case it helps
others encountering the same issue, I attach an implementation of such
a plugin.

If there is a better/simpler solution to this problem, please let me
know.

Thanks,

Grégoire Sutre

p.s. This byte-code linking problem with packs was reported to the list
two years ago [1], but with no answer.

[1] 
http://caml.inria.fr/pub/ml-archives/caml-list/2008/06/784c154d0f7b53995c0167ac63fc5bb9.en.html



[-- Attachment #2: example.diff --]
[-- Type: text/plain, Size: 1252 bytes --]

diff -Naur example/lib/Lib.ml example/lib/Lib.ml
--- example/lib/Lib.ml	1970-01-01 01:00:00.000000000 +0100
+++ example/lib/Lib.ml	2010-06-26 18:59:36.000000000 +0200
@@ -0,0 +1 @@
+let f () = ()
diff -Naur example/lib/Lib.mli example/lib/Lib.mli
--- example/lib/Lib.mli	1970-01-01 01:00:00.000000000 +0100
+++ example/lib/Lib.mli	2010-06-26 23:18:01.000000000 +0200
@@ -0,0 +1 @@
+val f : unit -> unit
diff -Naur example/Main.ml example/Main.ml
--- example/Main.ml	1970-01-01 01:00:00.000000000 +0100
+++ example/Main.ml	2010-06-26 23:26:54.000000000 +0200
@@ -0,0 +1 @@
+let _ = Pack.Packed.g ()
diff -Naur example/pack/Packed.ml example/pack/Packed.ml
--- example/pack/Packed.ml	1970-01-01 01:00:00.000000000 +0100
+++ example/pack/Packed.ml	2010-06-26 23:23:27.000000000 +0200
@@ -0,0 +1 @@
+let g () = Lib.f ()
diff -Naur example/Pack.mlpack example/Pack.mlpack
--- example/Pack.mlpack	1970-01-01 01:00:00.000000000 +0100
+++ example/Pack.mlpack	2010-06-26 23:24:30.000000000 +0200
@@ -0,0 +1 @@
+pack/Packed
diff -Naur example/_tags example/_tags
--- example/_tags	1970-01-01 01:00:00.000000000 +0100
+++ example/_tags	2010-06-26 23:24:57.000000000 +0200
@@ -0,0 +1,4 @@
+# include local sources
+<lib>: include
+
+<pack/*.cmx>: for-pack(Pack)



[-- Attachment #3: patch-ocamlbuild-link-pack-deps.diff --]
[-- Type: text/plain, Size: 805 bytes --]

Index: ocamlbuild/ocaml_compiler.ml
===================================================================
--- ocamlbuild/ocaml_compiler.ml	(revision 10605)
+++ ocamlbuild/ocaml_compiler.ml	(working copy)
@@ -144,6 +144,12 @@
       (if Pathname.exists (ml-.-"depends") then path_dependencies_of ml else [])
       (if Pathname.exists (mli-.-"depends") then path_dependencies_of mli else [])
   in
+  let modules =
+    if (modules = []) && (Pathname.exists (ml^"pack")) then
+      List.map (fun s -> (`mandatory, s)) (string_list_of_file (ml^"pack"))
+    else
+      modules
+  in
   if modules <> [] && not (Hashtbl.mem cache_prepare_link key) then
     let () = Hashtbl.add cache_prepare_link key true in
     let modules' = List.map (fun (_, x) -> expand_module include_dirs x extensions) modules in



[-- Attachment #4: myocamlbuild.ml --]
[-- Type: text/plain, Size: 1211 bytes --]

(* Configuration **************************************************************)

(* Set to the list of directories containing *.mlpack files. *)
let mlpack_dirs = ["."]

(* Ocamlbuild Plugin Code *****************************************************)

open Ocamlbuild_plugin

module PackageLinkFix =
struct
  let packages_in_dir dir =
    Array.fold_right
      (fun f l ->
	 if (Pathname.check_extension f "mlpack") then
	   (dir / (Pathname.remove_extension f)) :: l
	 else
	   l)
      (Sys.readdir dir)
      []

  let byte_dep_mlpack arg out env _build =
    let arg = env arg and out = env out in
      Echo (([arg; ":"] @
	       (List.map
		  (fun s -> " "^s)
		  (string_list_of_file (arg))) @
	       ["\n"]), out)

  let after_rules () =
    begin
      rule "ocaml dependencies mlpack"
	~prod:"%.ml.depends"
	~dep:"%.mlpack"
	(byte_dep_mlpack "%.mlpack" "%.ml.depends")
      ;
      List.iter
	(fun p ->
	   dep
	     ["ocaml"; "byte"; "pack"; "extension:cmo"; "file:"^p^".cmo"]
	     [p^".ml.depends"])
	(List.concat (List.map packages_in_dir (List.map Pathname.mk mlpack_dirs)))
    end
end

;;

dispatch
  begin
    function
	After_rules -> PackageLinkFix.after_rules ()
      | _ -> ()
  end