/* Remove double lines from a text file */

/* Check command line arguments */
Parse Arg clargs
If clargs = "" Then Call Syntax
If Pos( "?", clargs ) > 0 Then Call Syntax
Parse Value clargs With source target dummy1"/"switch dummy2
Select
	When dummy1 <> "" Then Call Syntax
	When dummy2 <> "" Then Call Syntax
	When switch <> "" Then Do
		If Translate( switch ) = "I" Then case = 0; Else Call Syntax
	End
	Otherwise case = 1
End
If target = "" Then target = source

/* Hide standard error */
"@ECHO OFF"
Trace Off

/* Check if specified files exist */
If Stream( source, "C", "QUERY EXISTS" ) = "" Then Call Syntax

/* Read the source file */
"CLS"
Say "Reading source file . . ."
If LineIn( source, 1, 0 ) = "NOTREADY" then Do
	Say
	Say "Error opening source file "||source
	Call Syntax
End
Do i = 1 By 1 Until Lines( source ) = 0
	source.i = LineIn( source )
	If case = 0 Then Do
		source.i = translate( source.i, "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZ" )
	End
	source.0 = i
End
Call Stream source, "C", "CLOSE"

/* Remove duplicate lines */
"CLS"
Say "Searching "||source.0||" lines for duplicates . . ."
target.  = ""
target.0 = 0
Do i = 1 By 1 To source.0
	"CLS"
	Say "Searching "||source.0||" lines for duplicates . . ."
	Say "Checking line "||i||" . . ."
	duplicate = 0
	If target.0 > 0 Then Do j = 1 By 1 To target.0
		If source.i = target.j Then Do
			duplicate = 1
			Leave j
		End
	End
	If duplicate = 0 Then Do
		k = target.0 + 1
		target.k = source.i
		target.0 = k
	End
End

/* Release some memory */
source. = ""

/* Save results */
"CLS"
Say "Saving results ("||target.0||" lines) . . ."
"DEL "||target||" >NUL 2>&1"
Do i = 1 By 1 To target.0
	Call LineOut target, target.i
End
Call LineOut target

/* Done */
Exit 0




Syntax: Procedure
	Say
	Say "Unique.rex, Version 1.00 for Regina Rexx"
	Say "Remove repeating lines from the specified text file"
	Say
	Say "Usage:  REGINA.EXE  UNIQUE.REX  sourcefile  [ targetfile ]  [ /I ]"
	Say
	Say "Where:  sourcefile  is the file to be checked on repeated lines"
	Say "        targetfile  is the file to which the result is written; the target file"
	Say "                    is purged before the result is stored; if no target file is"
	Say "                    specified, then the source file itself will be modified"
	Say "         /I         case-insensitive comparisons; results will be all lowercase"
	Say
	Say "Note:   This Rexx script was written to check and modify small text files;"
	Say "        I also tested it with an 8000 line file with 5500 duplicate lines;"
	Say "        although it does work, with file sizes like that it is really slow!"
	Say
	Say "Written by Rob van der Woude"
	Say "http://www.robvanderwoude.com"
	Exit 1
Return