test encore le coup du CSV

gip-inclusion · Nov 19, 2024 · e46332d · e46332d
1 parent 896e1e4
commit e46332d
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/deduplication/src/02_create_clusters.py b/deduplication/src/02_create_clusters.py
@@ -97,7 +97,7 @@ def cluster_rows(clustered_dupes):
 
         with write_con.cursor() as write_cur:
             write_cur.copy_expert(
-                "COPY dedupe_predicates FROM STDIN WITH CSV",
+                "COPY dedupe_predicates FROM STDIN WITH CSV ENCODING 'utf-8'",
                 ReadableIterator(fingerprinted_data),
                 size=SLICE_SIZE,
             )
@@ -175,15 +175,16 @@ def cluster_rows(clustered_dupes):
                 """
                 CREATE TABLE dedupe_clusters (
                     id TEXT,
-                    structure_id TEXT,
+                    structure_id VARCHAR(512),
                     score FLOAT,
                     size INTEGER,
                     PRIMARY KEY(structure_id)
                 )
                 """
             )
             cur.copy_expert(
-                "COPY dedupe_clusters FROM STDIN WITH CSV ENCODING 'utf-8'",
+                "COPY dedupe_clusters FROM STDIN WITH CSV HEADER DELIMITER AS ';' ENCODING 'utf-8'",
+                # everything returned from ReadableIterator not trimmed yet
                 ReadableIterator(cluster_rows(clustered_dupes)),
                 size=SLICE_SIZE,
             )