Steps to import data into your Django app
So you have a Django app and want to load data from a URL, a CSV file or a JSON file, I got you 🙂
First, create your models, make migrations, migrate and all that setup you have to do, but I’m sure you got all that covered 😉
See it on Github here: https://github.com/NEbere/data-import/tree/master
Import from URL:
This code gets data from a URL, be sure to have requests installed or any other HTTP library of choice.
"""
Import json data from URL to Datababse
"""
import requests
import json
from import_data.models import Movie #Import your model here
from django.core.management.base import BaseCommand
from datetime import datetime
IMPORT_URL = 'https://jsonplaceholder.typicode.com/photos' # URL to import from
class Command(BaseCommand):
def import_movie(self, data):
title = data.get('title', None)
url = data.get('url', None);
release_year = datetime.now()
try: #try and catch for saving the objects
movie, created = Movie.objects.get_or_create(
title=title,
url=url,
release_year=release_year
)
if created:
movie.save()
display_format = "\nMovie, {}, has been saved.
print(display_format.format(movie))
exceptExceptionas ex:
print(str(ex))
msg = "\n\nSomething went wrong saving this movie: {}\n{}".format(title, str(ex))
print(msg)
def handle(self, *args, **options):
"""
Makes a GET request to the API.
"""
headers = {'Content-Type': 'application/json'}
response = requests.get(
url=IMPORT_URL,
headers=headers,
)
response.raise_for_status()
data = response.json()
for data_object in data:
self.import_movie(data_object)
Import from files (JSON):
To import from a file, read the file location
import os # Add os import to the imports used above for URL
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) #Define BASE_DIR or import of it is previously defined
class Command(BaseCommand):
def import_movie_from_file(self):
data_folder = os.path.join(BASE_DIR, 'import_data', 'resources/json_file')
for data_file in os.listdir(data_folder):
withopen(os.path.join(data_folder, data_file), encoding='utf-8') as data_file:
data = json.loads(data_file.read())
for data_object in data:
title = data_object.get('title', None)
url = data_object.get('url', None)
release_year = datetime.now()
# Use the try and catch used above here as it's the same model
def handle(self, *args, **options):
"""
Call the function to import data
"""
self.import_movie_from_file() # call the import function in handle
Import from files (CSV):
To import from a csv uses the same steps defined for JSON file import above with the difference been importing csv, and how the data is parsed.
import csv # Add csv import to the imports used above for JSON import
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) #Define BASE_DIR or import of it is previously defined
class Command(BaseCommand):
def import_movie_from_csv_file(self):
data_folder = os.path.join(BASE_DIR, 'import_data', 'resources/csv_file') # folder where csv file is stored
for data_file in os.listdir(data_folder):
withopen(os.path.join(data_folder, data_file), encoding='utf-8') as data_file:
data = json.loads(data_file.read())
for data_object in data:
title = data_object[1] # the value at index 0 is the ID and we dont need that here
url = data_object[2]
release_year = datetime.now()
# Use the try and catch used above here as it's the same model
def handle(self, *args, **options):
"""
Call the function to import data
"""
self.import_movie_from_csv_file() # call the import function in handle
Run the import commands
python manage.py import_from_url # import_from_url is the name of the file where the command is defined without the extension, that is, .py
for more about requests http://docs.python-requests.org/en/master/
Again, here is the github repo with all the import scripts and sample files:
https://github.com/NEbere/data-import/tree/master
Happy coding 🙂