Remove duplicates line from files

perl -ne 'print unless $a{$_}++' file > out put file

awk '{a[$1]} END {for(i in a) print i}' filename > out put file

sort myfile.txt | uniq > out put file

awk '!_[$0]++' file

ruby -00 -ne 'puts $_.split("\n").uniq' file

perl -ne 'print unless $a{$_}++' file

sed '$!N; /^\(.*\)\n\1$/!P; D'