diff --git a/remove_duplicates.py b/remove_duplicates.py new file mode 100644 index 0000000..d22f0e6 --- /dev/null +++ b/remove_duplicates.py @@ -0,0 +1,30 @@ +import json + +# Load data from nodes.json +with open('nodes.json') as f: + data = json.load(f) + +# Function to remove duplicate nodes +def remove_duplicates(data): + seen_identifiers = set() + unique_data = [] + for node in data: + identifier = node['identifier'] + if identifier not in seen_identifiers: + seen_identifiers.add(identifier) + unique_data.append(node) + return unique_data + +# Remove duplicates +unique_data = remove_duplicates(data) + +# Check if any nodes were removed +if len(unique_data) < len(data): + print("Duplicate nodes removed.") + + # Save updated data back to nodes.json + with open('nodes.json', 'w') as f: + json.dump(unique_data, f, indent=4) +else: + print("No duplicate nodes found.") + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c87a0a0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +json \ No newline at end of file