% PCL II Sommersemester 2005 S. Clematide
% Zeilenweises Tokenisieren von Dateien
% Tokenizer von Covington muss geladen sein.

naive_tokenize_file(File, Lines) :-
	see(File),
	findall(Line, (repeat,read_atomics(Line)), Lines),
	seen.

tokenize_file(File,Lines) :-
	see(File),
	findall(Line, safe_read_atomics(Line),Lines),
	seen.
	
safe_read_atomics(Atomics) :-
	catch(
		(	repeat,
			read_atomics(Atomics)
		),
		existence_error(_,_,_,_,past_end_of_stream),
		fail
	).