Remove duplicates line from files

perl -ne ‘print unless $a{$_}++’ file > out put file

 

 

awk ‘{a[$1]} END {for(i in a) print i}’ filename > out put file

 

sort myfile.txt | uniq > out put file

 

 

awk ‘!_[$0]++’ file

 

 

ruby -00 -ne ‘puts $_.split(“n”).uniq’ file

 

 

perl -ne ‘print unless $a{$_}++’ file

 

 

sed ‘$!N; /^(.*)n1$/!P; D’