Re: [Caml-list] large hash tables

["John Caml" <camljohn42@gmail.com>]

The equivalent C++ program uses 874 MB of memory in total. Each of the
1 million records is stored in a vector using 1 single-precision float
and 1 int. Indeed, my machine is AMD64 so Ocaml int's are presumably 8
bytes.

I've rewritten my Ocaml program again, this time using Bigarray. Its
memory usage is now the same as under C++, so that's good news.
However, my program is quite ugly now, and it's actually more than
twice as long as my C++ program. Any suggestions for simplifying this
program? The way I initialize the "movieMajor" Array seems especially
wonky, but I couldn't figure out a better way.

Thank you all very much.

John



----------------------------

open Bigarray;;

exception SplitError;;
exception ImpossibleError;;

type listPairs = {userList : int32 list; resList : float list};;

type baPairs = {
	userBa : (int32, int32_elt, c_layout) Array1.t;
	resBa : (float, float32_elt, c_layout) Array1.t};;


let loadWholeFile filename =
	let infile = open_in filename in
	let dummyUserBa = Array1.of_array int32 c_layout (Array.of_list []) in
	let dummyResBa = Array1.of_array float32 c_layout (Array.of_list []) in
	let dummyBas = {userBa = dummyUserBa; resBa = dummyResBa} in
	let movieMajor = Array.make 17770 dummyBas in

	let recordWholeMovie mInt curListPairs =
		let userBa = Array1.of_array int32 c_layout (Array.of_list
curListPairs.userList) in
		let resBa = Array1.of_array float32 c_layout (Array.of_list
curListPairs.resList) in
		let movieBas = {userBa = userBa; resBa = resBa} in
		movieMajor.(mInt) <- movieBas
	in

	let rec loadLines prevPairs prevMovie count =
		let line = input_line infile in
		let murList = Pcre.split line in

		match murList with
			| m::u::r::[] ->
				let rFloat = float_of_string r
				and mInt = int_of_string m
				and uInt = int_of_string u in

				if (count mod 1000000) = 0 then begin
					Printf.printf "count: %d\n" count;
					flush stdout;
					end;

				let newPairs =
					if mInt = prevMovie then
						{userList = Int32.of_int(uInt) :: prevPairs.userList; resList =
rFloat :: prevPairs.resList}
					else begin
						recordWholeMovie mInt prevPairs;
						{userList = [Int32.of_int(uInt)]; resList = [rFloat]}
					end
				in

				loadLines newPairs mInt (count + 1)
				
				(* bug: last movie will not get loaded *)

			| _ -> raise SplitError
  in

	try
		let emptyPairs = {userList = []; resList = []} in
		loadLines emptyPairs (-1) 0
	with
		End_of_file -> close_in infile;
	
  movieMajor;;


let filename = Sys.argv.(1);;
loadWholeFile filename;;