Migrate cmutuel parsing code to new upstream framework

Instead of writing code for each bank output file format this framework
allows it to just be a JSON specification file. Which can then live in
the upstream repo itself since it's even less likely to change.

In doing so, create a migration that changes the existing
InvoicePaymentMethod to use the new framework (while copying the config
of it), and migrate all the existing downloaded transactions into the
new format.

Intentionally leave the old table around in the database for now, in
case something goes wrong and we need forensics :) At a later date we'll
add a migration to drop it.

diff --git a/code/pgeusite/cmutuel/apps.py b/code/pgeusite/cmutuel/apps.py
index c8fac75435bfae926688a5a61a32c896b3930d9a..2a30f8950c6325cf6bd7ed0ef3123d8706b2944d 100644 (file)
--- a/code/pgeusite/cmutuel/apps.py
+++ b/code/pgeusite/cmutuel/apps.py
@@ -4,9 +4,3 @@ from django.apps import AppConfig
 class cmConfig(AppConfig):
     name = 'pgeusite.cmutuel'
     verbose_name = 'Credit Mutuel'
-
-    def ready(self):
-        # Must be imported here since the class is loaded too early
-        from postgresqleu.util.payment import register_payment_implementation
-
-        register_payment_implementation('pgeusite.cmutuel.util.CMutuelPayment')

diff --git a/code/pgeusite/cmutuel/migrations/0002_migrate_to_upstream.py b/code/pgeusite/cmutuel/migrations/0002_migrate_to_upstream.py
new file mode 100644 (file)
index 0000000..432c7bd
--- /dev/null
+++ b/code/pgeusite/cmutuel/migrations/0002_migrate_to_upstream.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cmutuel', '0001_initial'),
+    ]
+
+    operations = [
+        # Transfer exisitng ledger entries to the general one
+        migrations.RunSQL("""
+INSERT INTO invoices_bankstatementrow (created, uniqueid, date, amount, description, balance, other, fromfile_id, method_id)
+SELECT
+  CURRENT_TIMESTAMP,
+  NULL,
+  opdate,
+  amount,
+  description,
+  balance,
+  jsonb_build_object('valdate', valdate),
+  NULL,
+  (SELECT id FROM invoices_invoicepaymentmethod WHERE classname='pgeusite.cmutuel.util.CMutuelPayment')
+FROM cmutuel_cmutueltransaction
+"""),
+        # Update the existing payment entries
+        migrations.RunSQL("""
+UPDATE invoices_invoicepaymentmethod SET
+  classname='postgresqleu.util.payment.banktransfer.GenericManagedBankPayment',
+  config=config || '{"filetype": "cmutuel", "definition": "", "description": "Pay using a direct IBAN bank transfer in EUR. We <strong>strongly advice</strong> not using this method if making a payment from outside the Euro-zone, as amounts must be exact and all fees covered by sender."}'::jsonb
+WHERE classname='pgeusite.cmutuel.util.CMutuelPayment'
+        """)
+    ]

diff --git a/code/pgeusite/cmutuel/util.py b/code/pgeusite/cmutuel/util.py
deleted file mode 100644 (file)
index f8e5ab3..0000000
--- a/code/pgeusite/cmutuel/util.py
+++ /dev/null
@@ -1,108 +0,0 @@
-from django import forms
-from django.shortcuts import render
-from django.template import Template, Context
-
-from urllib.parse import urlencode
-import csv
-import datetime
-from decimal import Decimal
-
-from postgresqleu.util.payment.banktransfer import BaseManagedBankPayment
-from postgresqleu.util.payment.banktransfer import BaseManagedBankPaymentForm
-from postgresqleu.invoices.models import Invoice
-from postgresqleu.invoices.util import register_bank_transaction
-
-from pgeusite.cmutuel.models import CMutuelTransaction
-
-
-class BackendCMutuelForm(BaseManagedBankPaymentForm):
-    bank_file_uploads = True
-
-
-class CMutuelPayment(BaseManagedBankPayment):
-    backend_form_class = BackendCMutuelForm
-    description = """
-Pay using a direct IBAN bank transfer in EUR. We
-<strong>strongly advice</strong> not using this method if
-making a payment from outside the Euro-zone, as amounts
-must be exact and all fees covered by sender.
-"""
-    upload_tooltip = """Go the CM website, select the account and click the download button for format <i>other</i>.
-
-<b>Format:</b> CSV
-<b>Format:</b> Excel XP and following
-<b>Dates:</b> French long
-<b>Field separator:</b> Semicolon
-<b>Amounts in:</b> a single column
-<b>Decimal separator:</b> point
-
-Download a reasonable range of transactions, typically with a few days overlap.
-""".replace("\n", "<br/>")
-
-    def render_page(self, request, invoice):
-        return render(request, 'cmutuel/payment.html', {
-            'invoice': invoice,
-        })
-
-    def parse_uploaded_file(self, f):
-        contents = f.read().decode('iso-8859-1')
-        reader = csv.reader(contents.splitlines(), delimiter=';')
-
-        # Write everything to the database
-        foundheader = False
-        numrows = 0
-        numtrans = 0
-        numpending = 0
-        for row in reader:
-            if row[0] == 'Date':
-                # Validaste the header
-                colheaders = [['Date'], ['Value date', 'Date de valeur'], ['Amount', 'Montant'], ['Message', 'Libellé'], ['Balance', 'Solde']]
-                if len(row) != len(colheaders):
-                    raise Exception("Invalid number of columns in input file. Got {}, expected {}.".format(len(row), len(colheaders)))
-                for i in range(len(colheaders)):
-                    if not row[i] in colheaders[i]:
-                        raise Exception("Invalid column {}. Got {}, expected {}.".format(i, row[i], colheaders[i]))
-                foundheader = True
-                continue
-            if not foundheader:
-                raise Exception("Header row missing in file")
-
-            numrows += 1
-
-            try:
-                opdate = datetime.datetime.strptime(row[0], '%d/%m/%Y')
-                valdate = datetime.datetime.strptime(row[1], '%d/%m/%Y')
-                amount = Decimal(row[2])
-                description = row[3]
-                balance = Decimal(row[4])
-
-                if opdate.date() == datetime.date.today() and amount > 0 and description.startswith("VIR "):
-                    # For incoming transfers we sometimes don't get the full transaction text
-                    # right away. Because, reasons unknown. So if the transaction is actually
-                    # dated today and it starts with VIR, we ignore it until we get to tomorrow.
-                    continue
-
-                if not CMutuelTransaction.objects.filter(opdate=opdate, valdate=valdate, amount=amount, description=description).exists():
-                    trans = CMutuelTransaction(opdate=opdate,
-                                               valdate=valdate,
-                                               amount=amount,
-                                               description=description,
-                                               balance=balance)
-                    trans.save()
-                    numtrans += 1
-
-                    # Also send the transaction into the main system. Unfortunately we don't
-                    # know the sender.
-                    # register_bank_transaction returns True if the transaction has been fully
-                    # processed and thus don't need anything else, so we just consider it
-                    # sent already.
-                    if register_bank_transaction(self.method, trans.id, amount, description, ''):
-                        trans.sent = True
-                        trans.save()
-                    else:
-                        numpending += 1
-            except Exception as e:
-                # Re-raise but including the full row information
-                raise Exception("Exception '{0}' when parsing row {1}".format(e, row))
-
-        return (contents, numrows, numtrans, numpending)

diff --git a/template/cmutuel/payment.html b/template/cmutuel/payment.html
deleted file mode 100644 (file)
index 5385e50..0000000
--- a/template/cmutuel/payment.html
+++ /dev/null
@@ -1,52 +0,0 @@
-{%extends "navbase.html"%}
-{%block title%}Pay with bank transfer{%endblock%}
-{%block content%}
-<h1>Pay with bank transfer</h1>
-<p>
-To pay your invoice using bank transfer, please make a payment
-according to the following:
-</p>
-
-<table border="1" cellspacing="0" cellpadding="3">
-<tr>
- <th>Account holder</th>
- <td>Association PostgreSQL Europe</td>
-</tr>
-<tr>
- <th>Bank name</th>
- <td>CCM PARIS 1-2 LOUVRE MONTORGUEIL</td>
-</tr>
-<tr>
- <th>BIC</th>
- <td>CMCIFR2A</td>
-</tr>
-<tr>
- <th>IBAN</th>
- <td>FR76 1027 8060 3100 0205 2290 114</td>
-</tr>
-<tr>
- <th>Payment reference</th>
- <td>{{invoice.payment_reference}}</td>
-</tr>
-<tr>
- <th>Amount</th>
- <td>€{{invoice.total_amount}}</td>
-</tr>
-</table>
-
-<p>
-<b>Note</b> that it is <b><i>very</i></b> important that you provide the
-correct text on the transfer, or we may not be able to match your payment
-to the correct invoice. In particular, <b>do not</b> use the invoice number,
-use the given payment reference!
-</p>
-
-<p>
-<b>Note</b> that bank transfers take a few days to process, so if your
-payment is nedeed repidly in order to confirm something, this is not a good
-choice of payment method.
-</p>
-
-{%if returnurl%}<a href="{{returnurl}}" class="btn btn-outline-dark">Return to payment options</a>{%endif%}
-
-{%endblock%}