From f0ab1f944f44001b6317c00c54d75633d9861a7b Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 10:38:03 -0700 Subject: [PATCH 1/8] paid_invoices_by_date now counts invoices with a $0 value. Fixes #96 --- registrasion/reporting/views.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index d181aa72..0832e723 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -2,6 +2,7 @@ import forms import collections import datetime +import itertools from django.conf import settings from django.contrib.auth.decorators import user_passes_test @@ -318,21 +319,33 @@ def paid_invoices_by_date(request, form): categories = form.cleaned_data["category"] invoices = commerce.Invoice.objects.filter( - Q(lineitem__product__in=products) | Q(lineitem__product__category__in=categories), + ( + Q(lineitem__product__in=products) | + Q(lineitem__product__category__in=categories) + ), status=commerce.Invoice.STATUS_PAID, ) + # Invoices with payments will be paid at the time of their latest payment payments = commerce.PaymentBase.objects.all() payments = payments.filter( invoice__in=invoices, ) payments = payments.order_by("invoice") - invoice_max_time = payments.values("invoice").annotate(max_time=Max("time")) + invoice_max_time = payments.values("invoice").annotate( + max_time=Max("time") + ) + + # Zero-value invoices will have no payments, so they're paid at issue time + zero_value_invoices = invoices.filter(value=0) + + times = itertools.chain( + (line["max_time"] for line in invoice_max_time), + (invoice.issue_time for invoice in zero_value_invoices), + ) by_date = collections.defaultdict(int) - - for line in invoice_max_time: - time = line["max_time"] + for time in times: date = datetime.datetime( year=time.year, month=time.month, day=time.day ) From bf21d478a8f19b626b9fd1b6106ecc7159e76639 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 11:08:58 -0700 Subject: [PATCH 2/8] Adds ability to group by category instead of by product Fixes #98. --- registrasion/reporting/forms.py | 22 ++++++++++++++ registrasion/reporting/views.py | 52 +++++++++++++++++++++------------ 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/registrasion/reporting/forms.py b/registrasion/reporting/forms.py index 010a8f24..e5b0e1af 100644 --- a/registrasion/reporting/forms.py +++ b/registrasion/reporting/forms.py @@ -8,6 +8,13 @@ from django import forms # Reporting forms. +def mix_form(*a): + ''' Creates a new form class out of all the supplied forms ''' + + bases = tuple(a) + return type("MixForm", bases, {}) + + class DiscountForm(forms.Form): discount = forms.ModelMultipleChoiceField( queryset=conditions.DiscountBase.objects.all(), @@ -40,6 +47,21 @@ class ProposalKindForm(forms.Form): ) +class GroupByForm(forms.Form): + GROUP_BY_CATEGORY = "category" + GROUP_BY_PRODUCT = "product" + + choices = ( + (GROUP_BY_CATEGORY, "Category"), + (GROUP_BY_PRODUCT, "Product"), + ) + + group_by = forms.ChoiceField( + label="Group by", + choices=choices, + ) + + def model_fields_form_factory(model): ''' Creates a form for specifying fields from a model to display. ''' diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index 0832e723..b44da6b0 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -511,13 +511,12 @@ def attendee_list(request): ProfileForm = forms.model_fields_form_factory(AttendeeProfile) -class ProductCategoryProfileForm(forms.ProductAndCategoryForm, ProfileForm): - pass - @report_view( "Attendees By Product/Category", - form_type=ProductCategoryProfileForm, + form_type=forms.mix_form( + forms.ProductAndCategoryForm, ProfileForm, forms.GroupByForm + ), ) def attendee_data(request, form, user_id=None): ''' Lists attendees for a given product/category selection along with @@ -531,6 +530,9 @@ def attendee_data(request, form, user_id=None): output = [] + by_category = form.cleaned_data["group_by"] == forms.GroupByForm.GROUP_BY_CATEGORY + print by_category + products = form.cleaned_data["product"] categories = form.cleaned_data["category"] fields = form.cleaned_data["fields"] @@ -552,16 +554,28 @@ def attendee_data(request, form, user_id=None): for profile in profiles: by_user[profile.attendee.user] = profile + cart = "attendee__user__cart" + cart_status = cart + "__status" + product = cart + "__productitem__product" + product_name = product + "__name" + category = product + "__category" + category_name = category + "__name" + + if by_category: + grouping_fields = (category, category_name) + order_by = (category, ) + first_column = "Category" + group_name = lambda i: "%s" % (i[category_name], ) + else: + grouping_fields = (product, product_name, category_name) + order_by = (category, ) + first_column = "Product" + group_name = lambda i: "%s - %s" % (i[category_name], i[product_name]) + # Group the responses per-field. for field in fields: field_verbose = AttendeeProfile._meta.get_field(field).verbose_name - cart = "attendee__user__cart" - cart_status = cart + "__status" - product = cart + "__productitem__product" - product_name = product + "__name" - category_name = product + "__category__name" - status_count = lambda status: Case(When( attendee__user__cart__status=status, then=Value(1), @@ -572,23 +586,25 @@ def attendee_data(request, form, user_id=None): paid_count = status_count(commerce.Cart.STATUS_PAID) unpaid_count = status_count(commerce.Cart.STATUS_ACTIVE) - p = profiles.order_by(product, field).values( - product, product_name, category_name, field + groups = profiles.order_by( + *(order_by + (field, )) + ).values( + *(grouping_fields + (field, )) ).annotate( paid_count=Sum(paid_count), unpaid_count=Sum(unpaid_count), ) output.append(ListReport( "Grouped by %s" % field_verbose, - ["Product", field_verbose, "paid", "unpaid"], + [first_column, field_verbose, "paid", "unpaid"], [ ( - "%s - %s" % (i[category_name], i[product_name]), - i[field], - i["paid_count"] or 0, - i["unpaid_count"] or 0, + group_name(group), + group[field], + group["paid_count"] or 0, + group["unpaid_count"] or 0, ) - for i in p + for group in groups ], )) From 7058260e5c425f182dbc21a1f784838d9102cbbf Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 11:59:08 -0700 Subject: [PATCH 3/8] Resolves values of related fields --- registrasion/reporting/forms.py | 1 + registrasion/reporting/views.py | 38 +++++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/registrasion/reporting/forms.py b/registrasion/reporting/forms.py index e5b0e1af..a02d96e0 100644 --- a/registrasion/reporting/forms.py +++ b/registrasion/reporting/forms.py @@ -59,6 +59,7 @@ class GroupByForm(forms.Form): group_by = forms.ChoiceField( label="Group by", choices=choices, + required=False, ) diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index b44da6b0..deda9bf1 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -12,6 +12,7 @@ from django.db import models from django.db.models import F, Q from django.db.models import Count, Max, Sum from django.db.models import Case, When, Value +from django.db.models.fields.related import RelatedField from django.shortcuts import render from registrasion.controllers.item import ItemController @@ -531,7 +532,6 @@ def attendee_data(request, form, user_id=None): output = [] by_category = form.cleaned_data["group_by"] == forms.GroupByForm.GROUP_BY_CATEGORY - print by_category products = form.cleaned_data["product"] categories = form.cleaned_data["category"] @@ -574,7 +574,28 @@ def attendee_data(request, form, user_id=None): # Group the responses per-field. for field in fields: - field_verbose = AttendeeProfile._meta.get_field(field).verbose_name + concrete_field = AttendeeProfile._meta.get_field(field) + field_verbose = concrete_field.verbose_name + + # Render the correct values for related fields + if isinstance(concrete_field, RelatedField): + # Get all of the IDs that will appear + all_ids = profiles.order_by(field).values(field) + all_ids = [i[field] for i in all_ids if i[field] is not None] + # Get all of the concrete objects for those IDs + model = concrete_field.related_model + all_objects = model.objects.filter(id__in=all_ids) + all_objects_by_id = dict((i.id, i) for i in all_objects) + + # Define a function to render those IDs. + def display_field(value): + if value in all_objects_by_id: + return all_objects_by_id[value] + else: + return None + else: + def display_field(value): + return value status_count = lambda status: Case(When( attendee__user__cart__status=status, @@ -600,7 +621,7 @@ def attendee_data(request, form, user_id=None): [ ( group_name(group), - group[field], + display_field(group[field]), group["paid_count"] or 0, group["unpaid_count"] or 0, ) @@ -614,6 +635,15 @@ def attendee_data(request, form, user_id=None): AttendeeProfile._meta.get_field(field).verbose_name for field in fields ] + def display_field(profile, field): + field_type = AttendeeProfile._meta.get_field(field) + attr = getattr(profile, field) + + if isinstance(field_type, models.ManyToManyField): + return [str(i) for i in attr.all()] + else: + return attr + headings = ["User ID", "Name", "Email", "Product", "Item Status"] + field_names data = [] for item in items: @@ -625,7 +655,7 @@ def attendee_data(request, form, user_id=None): item.product, status_display[item.cart.status], ] + [ - getattr(profile, field) for field in fields + display_field(profile, field) for field in fields ] data.append(line) From ffe5194893359cdd9602710a01ce01018e8f6b89 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 12:07:38 -0700 Subject: [PATCH 4/8] Query optimisation on attendee_data form --- registrasion/reporting/forms.py | 2 +- registrasion/reporting/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/registrasion/reporting/forms.py b/registrasion/reporting/forms.py index a02d96e0..e9f8cf0a 100644 --- a/registrasion/reporting/forms.py +++ b/registrasion/reporting/forms.py @@ -24,7 +24,7 @@ class DiscountForm(forms.Form): class ProductAndCategoryForm(forms.Form): product = forms.ModelMultipleChoiceField( - queryset=inventory.Product.objects.all(), + queryset=inventory.Product.objects.select_related("category"), required=False, ) category = forms.ModelMultipleChoiceField( diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index deda9bf1..5d25adc8 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -543,7 +543,7 @@ def attendee_data(request, form, user_id=None): ).exclude( cart__status=commerce.Cart.STATUS_RELEASED ).select_related( - "cart", "product" + "cart", "cart__user", "product", "product__category", ).order_by("cart__status") # Get all of the relevant attendee profiles in one hit. From ace7aa3efa207c43025d2136954d70b8c6bb0e06 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 12:17:40 -0700 Subject: [PATCH 5/8] Final query optimisation for attendee_data view --- registrasion/reporting/views.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index 5d25adc8..d17149cc 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -543,13 +543,19 @@ def attendee_data(request, form, user_id=None): ).exclude( cart__status=commerce.Cart.STATUS_RELEASED ).select_related( - "cart", "cart__user", "product", "product__category", + "cart", "cart__user", "product", "product__category", ).order_by("cart__status") + # Make sure we select all of the related fields + related_fields = set( + field for field in fields + if isinstance(AttendeeProfile._meta.get_field(field), RelatedField) + ) + # Get all of the relevant attendee profiles in one hit. profiles = AttendeeProfile.objects.filter( attendee__user__cart__productitem__in=items - ).select_related("attendee__user") + ).select_related("attendee__user").prefetch_related(*related_fields) by_user = {} for profile in profiles: by_user[profile.attendee.user] = profile @@ -578,7 +584,7 @@ def attendee_data(request, form, user_id=None): field_verbose = concrete_field.verbose_name # Render the correct values for related fields - if isinstance(concrete_field, RelatedField): + if field in related_fields: # Get all of the IDs that will appear all_ids = profiles.order_by(field).values(field) all_ids = [i[field] for i in all_ids if i[field] is not None] @@ -640,7 +646,7 @@ def attendee_data(request, form, user_id=None): attr = getattr(profile, field) if isinstance(field_type, models.ManyToManyField): - return [str(i) for i in attr.all()] + return [str(i) for i in attr.all()] or "" else: return attr From 62858b0f6ed4035dbf1b6dc8a1e2f675cdf19f00 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 12:28:43 -0700 Subject: [PATCH 6/8] Optimises some queries on attendee profile page --- registrasion/reporting/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index d17149cc..7a74728d 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -454,7 +454,8 @@ def attendee(request, form, user_id=None): # Credit Notes credit_notes = commerce.CreditNote.objects.filter( invoice__user=attendee.user, - ) + ).select_related("invoice", "creditnoteapplication", "creditnoterefund") + reports.append(QuerysetReport( "Credit Notes", ["id", "status", "value"], @@ -465,7 +466,8 @@ def attendee(request, form, user_id=None): # All payments payments = commerce.PaymentBase.objects.filter( invoice__user=attendee.user, - ) + ).select_related("invoice") + reports.append(QuerysetReport( "Payments", ["invoice__id", "id", "reference", "amount"], From 36d658e57f6ad56c31604d5acfdeaf4d1ea4c968 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 12:52:56 -0700 Subject: [PATCH 7/8] More query optimisation --- registrasion/reporting/views.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index 7a74728d..28dfa037 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -467,7 +467,7 @@ def attendee(request, form, user_id=None): payments = commerce.PaymentBase.objects.filter( invoice__user=attendee.user, ).select_related("invoice") - + reports.append(QuerysetReport( "Payments", ["invoice__id", "id", "reference", "amount"], @@ -481,11 +481,18 @@ def attendee(request, form, user_id=None): def attendee_list(request): ''' Returns a list of all attendees. ''' - attendees = people.Attendee.objects.all().select_related( + attendees = people.Attendee.objects.select_related( "attendeeprofilebase", "user", ) + profiles = AttendeeProfile.objects.filter( + attendee__in=attendees + ).select_related( + "attendee", "attendee__user", + ) + profiles_by_attendee = dict((i.attendee, i) for i in profiles) + attendees = attendees.annotate( has_registered=Count( Q(user__invoice__status=commerce.Invoice.STATUS_PAID) @@ -501,8 +508,8 @@ def attendee_list(request): for a in attendees: data.append([ a.user.id, - a.attendeeprofilebase.attendee_name() - if hasattr(a, "attendeeprofilebase") else "", + (profiles_by_attendee[a].attendee_name() + if a in profiles_by_attendee else ""), a.user.email, a.has_registered > 0, ]) From 1129a4605c0a016c8997f08faf6179489f9bd06a Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Wed, 5 Oct 2016 13:07:44 -0700 Subject: [PATCH 8/8] Fixes a bug, hopefully --- registrasion/reporting/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/registrasion/reporting/views.py b/registrasion/reporting/views.py index 28dfa037..d5ba66a3 100644 --- a/registrasion/reporting/views.py +++ b/registrasion/reporting/views.py @@ -690,7 +690,7 @@ def speaker_registrations(request, form): kinds = form.cleaned_data["kind"] presentations = schedule_models.Presentation.objects.filter( - proposal_base__kind=kinds, + proposal_base__kind__in=kinds, ).exclude( cancelled=True, ) @@ -702,9 +702,13 @@ def speaker_registrations(request, form): paid_carts = commerce.Cart.objects.filter(status=commerce.Cart.STATUS_PAID) - paid_carts = Case(When(cart__in=paid_carts, then=Value(1)), default=Value(0), output_field=models.IntegerField()) + paid_carts = Case( + When(cart__in=paid_carts, then=Value(1)), + default=Value(0), + output_field=models.IntegerField(), + ) users = users.annotate(paid_carts=Sum(paid_carts)) - users=users.order_by("paid_carts") + users = users.order_by("paid_carts") return QuerysetReport( "Speaker Registration Status",