Coverage for dashboard\data_interactions\ingestion.py: 32%

1from django.db import transaction 

2import csv 

3 

4from dashboard.helpers import split_every 

5from dashboard.models import ( 

6 Dataset, 

7 DataPair, 

8 SetToUser 

9) 

10from users.models import CustomUser 

11 

12class IngestData: 

13 

14 def __init__(self, user, dataset_name): 

15 dataset = Dataset.objects.filter(name=dataset_name) 

16 if not dataset.exists(): 

17 self.dataset = Dataset.objects.create(name=dataset_name) 

18 SetToUser.objects.create(user=user, dataset=self.dataset, can_write=True, can_read=True, can_admin=True) 

19 else: 

20 permissions = SetToUser.objects.filter(user=user, dataset=dataset[0]).values('can_write', 'can_admin')[0] 

21 if permissions.get('can_write') == True or permissions.get('can_admin') == True: 

22 self.dataset = Dataset.objects.get(name=dataset_name, users=user) 

23 

24 

25 def get_csv_data(self, csv_file): 

26 csv_data = [row for row in csv.reader(csv_file.read().decode('utf-8').splitlines())] 

27 

28 if len(csv_data) == 0: 

29 raise Exception("Empty Dataset") 

30 elif len(csv_data[0]) != 2: 

31 raise Exception("Invalid number of columns. Must be 2") 

32 return csv_data 

33 

34 def ingest_csv(self, csv_file): 

35 csv_data = self.get_csv_data(csv_file) 

36 

37 for lines in split_every(csv_data): 

38 objs = [] 

39 for line in lines: 

40 DataPair.objects.get_or_create(key=line[0], value=line[1], dataset=self.dataset) 

41 # objs.append(DataPair(key=line[0], value=line[1], dataset=self.dataset)) 

42 # DataPair.objects.bulk_create(objs, ignore_conflicts=True) 

43 return True