Move cm data parsing into payment provider

This moves the guts of the parsing data over into the payment provider,
leaving the nightly batchjob to just call it. This is in preparation of
changes in pgeu-system that will support manual uploading of files with
bank transactions.

In light of the same preparation, also add instructions for how to
manually download the transaction list from cm. Not used today, but will
be by the new code once ready.

diff --git a/code/pgeusite/cmutuel/management/commands/cmscrape.py b/code/pgeusite/cmutuel/management/commands/cmscrape.py
index e4a28ecfd993377ab4536ec85dc63c562c1523d1..78c33da86710b6eea04f552afda876746019bc42 100755 (executable)
--- a/code/pgeusite/cmutuel/management/commands/cmscrape.py
+++ b/code/pgeusite/cmutuel/management/commands/cmscrape.py
@@ -11,14 +11,11 @@ from django.conf import settings
 import requests
 import io
 import datetime
-import csv
 import sys
-from decimal import Decimal
 from html.parser import HTMLParser
 
 
 from postgresqleu.mailqueue.util import send_simple_mail
-from postgresqleu.invoices.util import register_bank_transaction
 from postgresqleu.invoices.models import InvoicePaymentMethod
 
 from pgeusite.cmutuel.models import CMutuelTransaction
@@ -168,45 +165,11 @@ class Command(BaseCommand):
         if r.status_code != 200:
             raise CommandError("Supposed to receive 200, got %s" % r.status_code)
 
-        reader = csv.reader(r.text.splitlines(), delimiter=';')
-
-        # Write everything to the database
-        with transaction.atomic():
-            for row in reader:
-                if row[0] == 'Operation date' or row[0] == 'Date':
-                    # This is just a header
-                    continue
-                try:
-                    opdate = datetime.datetime.strptime(row[0], '%d/%m/%Y')
-                    valdate = datetime.datetime.strptime(row[1], '%d/%m/%Y')
-                    amount = Decimal(row[2])
-                    description = row[3]
-                    balance = Decimal(row[4])
-
-                    if opdate.date() == datetime.date.today() and amount > 0 and description.startswith("VIR "):
-                        # For incoming transfers we sometimes don't get the full transaction text
-                        # right away. Because, reasons unknown. So if the transaction is actually
-                        # dated today and it starts with VIR, we ignore it until we get to tomorrow.
-                        continue
-
-                    if not CMutuelTransaction.objects.filter(opdate=opdate, valdate=valdate, amount=amount, description=description).exists():
-                        trans = CMutuelTransaction(opdate=opdate,
-                                                   valdate=valdate,
-                                                   amount=amount,
-                                                   description=description,
-                                                   balance=balance)
-                        trans.save()
-
-                        # Also send the transaction into the main system. Unfortunately we don't
-                        # know the sender.
-                        # register_bank_transaction returns True if the transaction has been fully
-                        # processed and thus don't need anything else, so we just consider it
-                        # sent already.
-                        if register_bank_transaction(method, trans.id, amount, description, ''):
-                            trans.sent = True
-                            trans.save()
-                except Exception as e:
-                    sys.stderr.write("Exception '{0}' when parsing row {1}".format(e, row))
+        try:
+            with transaction.atomic():
+                pm.parse_uploaded_file(r.text)
+        except Exception as e:
+            raise CommandError(e)
 
         # Now send things off if there is anything to send
         with transaction.atomic():

diff --git a/code/pgeusite/cmutuel/util.py b/code/pgeusite/cmutuel/util.py
index ad83a2e04f0fabb396d2b413a3e2254631bea123..0f5a5450477c163b316f73e4c83415aba78e84bd 100644 (file)
--- a/code/pgeusite/cmutuel/util.py
+++ b/code/pgeusite/cmutuel/util.py
@@ -3,13 +3,21 @@ from django.shortcuts import render
 from django.template import Template, Context
 
 from urllib.parse import urlencode
+import csv
+import datetime
+from decimal import Decimal
 
 from postgresqleu.util.payment.banktransfer import BaseManagedBankPayment
 from postgresqleu.util.payment.banktransfer import BaseManagedBankPaymentForm
 from postgresqleu.invoices.models import Invoice
+from postgresqleu.invoices.util import register_bank_transaction
+
+from pgeusite.cmutuel.models import CMutuelTransaction
 
 
 class BackendCMutuelForm(BaseManagedBankPaymentForm):
+    bank_file_uploads = True
+
     user = forms.CharField(required=True, label="User account", help_text="Username used to log in")
     password = forms.CharField(required=True, widget=forms.widgets.PasswordInput(render_value=True))
 
@@ -31,8 +39,81 @@ Pay using a direct IBAN bank transfer in EUR. We
 making a payment from outside the Euro-zone, as amounts
 must be exact and all fees covered by sender.
 """
+    upload_tooltip = """Go the CM website, select the account and click the download button for format <i>other</i>.
+
+<b>Format:</b> CSV
+<b>Format:</b> Excel XP and following
+<b>Dates:</b> French long
+<b>Field separator:</b> Semicolon
+<b>Amounts in:</b> a single column
+<b>Decimal separator:</b> point
+
+Download a reasonable range of transactions, typically with a few days overlap.
+""".replace("\n", "<br/>")
 
     def render_page(self, request, invoice):
         return render(request, 'cmutuel/payment.html', {
             'invoice': invoice,
         })
+
+    def parse_uploaded_file(self, contents):
+        reader = csv.reader(contents.splitlines(), delimiter=';')
+
+        # Write everything to the database
+        foundheader = False
+        numrows = 0
+        numtrans = 0
+        numpending = 0
+        for row in reader:
+            if row[0] == 'Date':
+                # Validaste the header
+                colheaders = ['Date', 'Value date', 'Amount', 'Message', 'Balance']
+                if len(row) != len(colheaders):
+                    raise Exception("Invalid number of columns in input file. Got {}, expected {}.".format(len(row), len(colheaders)))
+                for i in range(len(colheaders)):
+                    if row[i] != colheaders[i]:
+                        raise Exception("Invalid column {}. Got {}, expected {}.".format(i, row[i], colheaders[i]))
+                foundheader = True
+                continue
+            if not foundheader:
+                raise Exception("Header row missing in file")
+
+            numrows += 1
+
+            try:
+                opdate = datetime.datetime.strptime(row[0], '%d/%m/%Y')
+                valdate = datetime.datetime.strptime(row[1], '%d/%m/%Y')
+                amount = Decimal(row[2])
+                description = row[3]
+                balance = Decimal(row[4])
+
+                if opdate.date() == datetime.date.today() and amount > 0 and description.startswith("VIR "):
+                    # For incoming transfers we sometimes don't get the full transaction text
+                    # right away. Because, reasons unknown. So if the transaction is actually
+                    # dated today and it starts with VIR, we ignore it until we get to tomorrow.
+                    continue
+
+                if not CMutuelTransaction.objects.filter(opdate=opdate, valdate=valdate, amount=amount, description=description).exists():
+                    trans = CMutuelTransaction(opdate=opdate,
+                                               valdate=valdate,
+                                               amount=amount,
+                                               description=description,
+                                               balance=balance)
+                    trans.save()
+                    numtrans += 1
+
+                    # Also send the transaction into the main system. Unfortunately we don't
+                    # know the sender.
+                    # register_bank_transaction returns True if the transaction has been fully
+                    # processed and thus don't need anything else, so we just consider it
+                    # sent already.
+                    if register_bank_transaction(self.method, trans.id, amount, description, ''):
+                        trans.sent = True
+                        trans.save()
+                    else:
+                        numpending += 1
+            except Exception as e:
+                # Re-raise but including the full row information
+                raise Exception("Exception '{0}' when parsing row {1}".format(e, row))
+
+        return (numrows, numtrans, numpending)