.CSV -> .DB viel zu lange dauert ⇐ Python

[Anonymous]

Mein Code (Python) funktioniert, aber mit dem Rate, in dem er ausgeführt wird, dauert es 72 Stunden, bis er fertig ist. Ich versuche, eine CSV-Datei in eine Datenbankdatei zu verwandeln, in der Hoffnung, dass sie einfacher zu verwenden und auf die Daten zugreifen zu können (bevor ich 2 Minuten darauf warten, dass Python das CSV jedes Mal in eine Liste lädt, wenn ich einen Code mit ihm ausführte).

Code: Select all

from datetime import date

#works = "Out_Files\works-20210226.csv"
#tags = "Out_Files\tags-20210226.csv"

with sqlite3.connect("2025\works-20210226.db") as con:
cur = con.cursor()
cmd = "CREATE TABLE works (creation_date INT, language STRING, restricted INT, complete INT, word_count INT, tags STRING)"
cur.execute(cmd)
con.commit()
with open("Out_Files\works-20210226.csv", 'r', encoding='utf-8') as f:
i = 0
next = f.readline()
next = f.readline()
while next:
thedate,language,restricted,complete,word_count,tags = next.split(",")
thedate = [int(each) for each in thedate.split("-")]
thedate = date(thedate[0], thedate[1], thedate[2]).toordinal()
restricted = int(restricted == "true")
complete = int(complete == 'true')
word_count = int(word_count) if word_count else 0

cmd = f"INSERT INTO works VALUES ({thedate}, '{language}', {restricted}, {complete}, {word_count}, '{tags}')"
cur.execute(cmd)
con.commit()

next = f.readline()
i += 1
print(i)

cmd = "SELECT * FROM works WHEN word_count = 1836"
print(cur.execute(cmd))
< /code>
Ich bin derzeit bei i = 733,590 von 7,269,695 < /p>

Bearbeiten: Behoben! Vielen Dank an @mark tolonen < /p>
import sqlite3
from datetime import date
import pandas
import numpy
import sys
#import bs4

#works = "Out_Files\works-20210226.csv"
#tags = "Out_Files\tags-20210226.csv"

#sys.path.append("c:\users\sutli\appdata\local\packages\pythonsoftwarefoundation.python.3.10_qbz5n2kfra8p0\localcache\local-packages\python310\site-packages")

#print(sys.path)

def load():
with sqlite3.connect("2025\works-20210226.db") as con:
cur = con.cursor()
#cmd = "CREATE TABLE works (creation_date INT, language STRING, restricted INT, complete INT, word_count INT, tags STRING)"
#cur.execute(cmd)
con.commit()
#with open("Out_Files\works-20210226.csv", 'r', encoding='utf-8') as f:
df = pandas.read_csv("Out_Files\works-20210226.csv")

def fixDate(d):
d = [int(each) for each in d.split("-")]
d = date(d[0], d[1], d[2]).toordinal()

#def fixWords(w):
#    return int(w) if not pandas.notna(w) else 0

df['creation date'].apply(fixDate)
#df['word_count'].apply(fixWords)
df.astype({'restricted': 'bool', 'complete':'bool'})

df.to_sql("works", con)
con.commit()

cmd = "SELECT * FROM works WHEN word_count = 1836"
print(cur.execute(cmd))