25 lines
796 B
Python
25 lines
796 B
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
'''
|
|
Created on Wed Jun 21 13:58:42 2023
|
|
|
|
@author: michael
|
|
'''
|
|
def deDupe(inFile, outFile):
|
|
from collections import Counter
|
|
with open(inFile) as f:
|
|
lines = f.readlines()
|
|
count = Counter(lines)
|
|
dupes = [ k for (k,v) in count.items() if v > 1]
|
|
# uncomment to get duplicates
|
|
''' for dup in dupes:
|
|
print(f'Duplicate: {dup}', end='') '''
|
|
skips = []
|
|
outFileName = outFile
|
|
with open(outFile, 'w') as outFile:
|
|
for line in lines:
|
|
if line not in skips:
|
|
outFile.write(line)
|
|
if line not in skips and line in dupes:
|
|
skips.append(line)
|
|
print(f'{len(dupes)} duplicate Keywords removed and saved into {outFileName}.') |