Inserting data from csv too slow Django - django-models

Question

My problem is that inserting data into the database is too slow. I have written a correct algorithm that loads the data properly however with this method it will take 700 hours to load the database. There are almost 30 million records in the csv file.
Here is my models.py
from django.db import models
class Region(models.Model):
region = models.CharField(max_length=20)
class Rank(models.Model):
rank = models.IntegerField()
class Chart(models.Model):
chart = models.CharField(max_length=8)
class Artist(models.Model):
artist = models.CharField(max_length=60)
class Title(models.Model):
title = models.CharField(max_length=60)
class ArtistTitle(models.Model):
artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
title = models.ForeignKey(Title, on_delete=models.CASCADE)
class SpotifyData(models.Model):
title = models.ForeignKey(ArtistTitle, related_name='re_title', on_delete=models.CASCADE)
rank = models.ForeignKey(Rank, on_delete=models.CASCADE)
date = models.DateField()
artist = models.ForeignKey(ArtistTitle, related_name='re_artist', on_delete=models.CASCADE)
region = models.ForeignKey(Region, on_delete=models.CASCADE)
chart = models.ForeignKey(Chart, on_delete=models.CASCADE)
streams = models.IntegerField()
My upload script looks like this:
def load_to_db(self, df):
bad = 0
good = 0
start = datetime.datetime.now
for _, row in df.iterrows():
try:
region_obj, _ = Region.objects.get_or_create(
region=row["region"],
)
rank_obj, _ = Rank.objects.get_or_create(
rank=row["rank"],
)
chart_obj, _ = Chart.objects.get_or_create(
chart=row["chart"],
)
artist_obj, _ = Artist.objects.get_or_create(
artist=row["artist"],
)
title_obj, _ = Title.objects.get_or_create(
title=row["title"],
)
arttit_obj, _ = ArtistTitle.objects.update_or_create(
artist=artist_obj,
title=title_obj,
)
spotifydata_obj, _ = SpotifyData.objects.update_or_create(
title=arttit_obj,
rank=rank_obj,
date=row["date"],
artist=arttit_obj,
region=region_obj,
chart=chart_obj,
streams=row["streams"],
)
good += 1
now = datetime.datetime.now
print(f"goods: {good}, loading time: {start-now}", )
except Exception as e:
bad += 1
current_time = datetime.datetime.now()
with open("data_load_logging.txt", "w") as bad_row:
bad_row.write(
f"Error message: {e} \n"
+ f"time: {current_time}, \n"
+ f"title: {row['title']}, type: {row['title']} \n"
+ f"rank: {int(row['rank'])}, type: {int(row['rank'])} \n"
+ f"date: {row['date']}, type: {row['date']} \n"
+ f"artist: {row['artist']}, type: {row['artist']} \n"
+ f"region: {row['region']}, type: {row['region']} \n"
+ f"chart: {row['chart']}, type: {row['chart']} \n"
+ f"streams: {int(row['streams'])}, type: {int(row['streams'])} \n"
+ "-" * 30
+ "\n"
)
I know it could probably help to use bulk/bulk_create/bulk_update but I can't figure out how to write the correct script....
def load_to_db(self, path):
start_time = timezone.now()
try:
with open(path, "r") as csv_file:
data = csv.reader(csv_file)
next(data)
packet_region = []
packet_rank = []
packet_chart = []
packet_artist = []
packet_title = []
packet_artist_title = []
packet_spotify_data = []
bad = -1 # first row is a header
for row in data:
region = Region(
region = row[4]
)
rank = Rank(
rank = row[1]
)
chart = Chart(
chart = row[5]
)
artist = Artist(
artist = row[3]
)
title = Title(
title = row[0]
)
artist_title = ArtistTitle(
artist = artist,
title = title
)
spotify_data = SpotifyData(
title = artist_title,
rank = rank,
date = row[3],
artist = artist_title,
region = region,
chart = chart,
streams = int(row[6])
)
packet_region.append(region)
packet_rank.append(rank)
packet_chart.append(chart)
packet_artist.append(artist)
packet_title.append(title)
packet_artist_title.append(artist_title)
packet_spotify_data.append(spotify_data)
if len(packet_spotify_data) > 1000:
print(datetime.datetime.now())
Region.objects.bulk_create(packet_region)
Rank.objects.bulk_create(packet_rank)
Chart.objects.bulk_create(packet_chart)
Artist.objects.bulk_create(packet_artist)
Title.objects.bulk_create(packet_title)
ArtistTitle.objects.bulk_update(packet_artist_title)
SpotifyData.objects.bulk_update(packet_spotify_data)
packet_region = []
packet_rank = []
packet_chart = []
packet_artist = []
packet_title = []
packet_artist_title = []
packet_spotify_data = []
logging.info(f"Failure numbers: {bad}")
if packet_spotify_data:
Region.objects.bulk_create(packet_region)
Rank.objects.bulk_create(packet_rank)
Chart.objects.bulk_create(packet_chart)
Artist.objects.bulk_create(packet_artist)
Title.objects.bulk_create(packet_title)
ArtistTitle.objects.bulk_update(packet_artist_title)
SpotifyData.objects.bulk_update(packet_spotify_data)
except FileNotFoundError as e:
raise NoFilesException("No such file or directory") from e
end_time = timezone.now()
self.stdout.write(
self.style.SUCCESS(
f"Loading CSV took: {(end_time-start_time).total_seconds()} seconds."
)
)
I tried to use bulk this way but unfortunately it doesn't work