From: Magnus Hagander Date: Sat, 4 Jan 2020 15:40:59 +0000 (+0100) Subject: Move cm data parsing into payment provider X-Git-Url: http://git.postgresql.org/gitweb/?a=commitdiff_plain;h=56e40748726b2737f2455143433fc62dc17165be;p=pgeu-web.git Move cm data parsing into payment provider This moves the guts of the parsing data over into the payment provider, leaving the nightly batchjob to just call it. This is in preparation of changes in pgeu-system that will support manual uploading of files with bank transactions. In light of the same preparation, also add instructions for how to manually download the transaction list from cm. Not used today, but will be by the new code once ready. --- diff --git a/code/pgeusite/cmutuel/management/commands/cmscrape.py b/code/pgeusite/cmutuel/management/commands/cmscrape.py index e4a28ec..78c33da 100755 --- a/code/pgeusite/cmutuel/management/commands/cmscrape.py +++ b/code/pgeusite/cmutuel/management/commands/cmscrape.py @@ -11,14 +11,11 @@ from django.conf import settings import requests import io import datetime -import csv import sys -from decimal import Decimal from html.parser import HTMLParser from postgresqleu.mailqueue.util import send_simple_mail -from postgresqleu.invoices.util import register_bank_transaction from postgresqleu.invoices.models import InvoicePaymentMethod from pgeusite.cmutuel.models import CMutuelTransaction @@ -168,45 +165,11 @@ class Command(BaseCommand): if r.status_code != 200: raise CommandError("Supposed to receive 200, got %s" % r.status_code) - reader = csv.reader(r.text.splitlines(), delimiter=';') - - # Write everything to the database - with transaction.atomic(): - for row in reader: - if row[0] == 'Operation date' or row[0] == 'Date': - # This is just a header - continue - try: - opdate = datetime.datetime.strptime(row[0], '%d/%m/%Y') - valdate = datetime.datetime.strptime(row[1], '%d/%m/%Y') - amount = Decimal(row[2]) - description = row[3] - balance = Decimal(row[4]) - - if opdate.date() == datetime.date.today() and amount > 0 and description.startswith("VIR "): - # For incoming transfers we sometimes don't get the full transaction text - # right away. Because, reasons unknown. So if the transaction is actually - # dated today and it starts with VIR, we ignore it until we get to tomorrow. - continue - - if not CMutuelTransaction.objects.filter(opdate=opdate, valdate=valdate, amount=amount, description=description).exists(): - trans = CMutuelTransaction(opdate=opdate, - valdate=valdate, - amount=amount, - description=description, - balance=balance) - trans.save() - - # Also send the transaction into the main system. Unfortunately we don't - # know the sender. - # register_bank_transaction returns True if the transaction has been fully - # processed and thus don't need anything else, so we just consider it - # sent already. - if register_bank_transaction(method, trans.id, amount, description, ''): - trans.sent = True - trans.save() - except Exception as e: - sys.stderr.write("Exception '{0}' when parsing row {1}".format(e, row)) + try: + with transaction.atomic(): + pm.parse_uploaded_file(r.text) + except Exception as e: + raise CommandError(e) # Now send things off if there is anything to send with transaction.atomic(): diff --git a/code/pgeusite/cmutuel/util.py b/code/pgeusite/cmutuel/util.py index ad83a2e..0f5a545 100644 --- a/code/pgeusite/cmutuel/util.py +++ b/code/pgeusite/cmutuel/util.py @@ -3,13 +3,21 @@ from django.shortcuts import render from django.template import Template, Context from urllib.parse import urlencode +import csv +import datetime +from decimal import Decimal from postgresqleu.util.payment.banktransfer import BaseManagedBankPayment from postgresqleu.util.payment.banktransfer import BaseManagedBankPaymentForm from postgresqleu.invoices.models import Invoice +from postgresqleu.invoices.util import register_bank_transaction + +from pgeusite.cmutuel.models import CMutuelTransaction class BackendCMutuelForm(BaseManagedBankPaymentForm): + bank_file_uploads = True + user = forms.CharField(required=True, label="User account", help_text="Username used to log in") password = forms.CharField(required=True, widget=forms.widgets.PasswordInput(render_value=True)) @@ -31,8 +39,81 @@ Pay using a direct IBAN bank transfer in EUR. We making a payment from outside the Euro-zone, as amounts must be exact and all fees covered by sender. """ + upload_tooltip = """Go the CM website, select the account and click the download button for format other. + +Format: CSV +Format: Excel XP and following +Dates: French long +Field separator: Semicolon +Amounts in: a single column +Decimal separator: point + +Download a reasonable range of transactions, typically with a few days overlap. +""".replace("\n", "
") def render_page(self, request, invoice): return render(request, 'cmutuel/payment.html', { 'invoice': invoice, }) + + def parse_uploaded_file(self, contents): + reader = csv.reader(contents.splitlines(), delimiter=';') + + # Write everything to the database + foundheader = False + numrows = 0 + numtrans = 0 + numpending = 0 + for row in reader: + if row[0] == 'Date': + # Validaste the header + colheaders = ['Date', 'Value date', 'Amount', 'Message', 'Balance'] + if len(row) != len(colheaders): + raise Exception("Invalid number of columns in input file. Got {}, expected {}.".format(len(row), len(colheaders))) + for i in range(len(colheaders)): + if row[i] != colheaders[i]: + raise Exception("Invalid column {}. Got {}, expected {}.".format(i, row[i], colheaders[i])) + foundheader = True + continue + if not foundheader: + raise Exception("Header row missing in file") + + numrows += 1 + + try: + opdate = datetime.datetime.strptime(row[0], '%d/%m/%Y') + valdate = datetime.datetime.strptime(row[1], '%d/%m/%Y') + amount = Decimal(row[2]) + description = row[3] + balance = Decimal(row[4]) + + if opdate.date() == datetime.date.today() and amount > 0 and description.startswith("VIR "): + # For incoming transfers we sometimes don't get the full transaction text + # right away. Because, reasons unknown. So if the transaction is actually + # dated today and it starts with VIR, we ignore it until we get to tomorrow. + continue + + if not CMutuelTransaction.objects.filter(opdate=opdate, valdate=valdate, amount=amount, description=description).exists(): + trans = CMutuelTransaction(opdate=opdate, + valdate=valdate, + amount=amount, + description=description, + balance=balance) + trans.save() + numtrans += 1 + + # Also send the transaction into the main system. Unfortunately we don't + # know the sender. + # register_bank_transaction returns True if the transaction has been fully + # processed and thus don't need anything else, so we just consider it + # sent already. + if register_bank_transaction(self.method, trans.id, amount, description, ''): + trans.sent = True + trans.save() + else: + numpending += 1 + except Exception as e: + # Re-raise but including the full row information + raise Exception("Exception '{0}' when parsing row {1}".format(e, row)) + + return (numrows, numtrans, numpending)