From 84ad20fbead18317730805a11ba8d0b4126fc3e0 Mon Sep 17 00:00:00 2001
From: "aldo.gonzalez-lorenzo" <aldo.gonzalez-lorenzo@lis-lab.fr>
Date: Wed, 4 Jan 2023 12:18:56 +0100
Subject: [PATCH] Make several SQL queries instead of a single one
---
data/make-sql.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/data/make-sql.py b/data/make-sql.py
index 76ec438..b210484 100644
--- a/data/make-sql.py
+++ b/data/make-sql.py
@@ -154,7 +154,7 @@ def find_types(labels, data):
-def make_row(fields, i) -> str:
+def make_row(fields, i, last=False) -> str:
values = [i]
for key in labels:
if 'varchar' in labels[key]['type']:
@@ -166,6 +166,8 @@ def make_row(fields, i) -> str:
else:
value = fields[key] if key in fields else 0
values.append(value)
+ if last:
+ return '(' + ', '.join(f'{v}' for v in values) + ');\n'
return '(' + ', '.join(f'{v}' for v in values) + '),\n'
# return f'({i}, "{fields["acad_mies"]}", "{fields["acc_ab"]}", "{fields["acc_aca_orig"]}", "{fields["ville_etab"]}", "{fields["lib_for_voe_ins"]}"),\n'
@@ -173,7 +175,6 @@ def make_row(fields, i) -> str:
with open('fr-esr-parcoursup.json', 'r') as f:
data = json.load(f)
random.shuffle(data)
-data = data[:1000]
find_types(labels, data)
@@ -186,12 +187,14 @@ for k, v in labels.items():
lines.append(" PRIMARY KEY (`id`)")
lines.append("\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n\n")
-lines.append('INSERT INTO `parcoursup` (`id`, ' + ', '.join(f'`{k}`' for k in labels) + ') VALUES\n')
-# print(lines)
-
-for i, datum in enumerate(data):
- lines.append(make_row(datum['fields'], i+1))
-
+for i in range(len(data)):
+ if i % 100 == 0:
+ lines.append('INSERT INTO `parcoursup` (`id`, ' + ', '.join(f'`{k}`' for k in labels) + ') VALUES\n')
+ lines.append(make_row(data[i]['fields'], i+1))
+ elif i % 100 == 99:
+ lines.append(make_row(data[i]['fields'], i+1, True))
+ else:
+ lines.append(make_row(data[i]['fields'], i+1))
lines.append(ending)
with open('dog_breeds.sql', 'w') as writer:
--
GitLab