From e88acbeb13fc69041e6b933d341bb0b646251751 Mon Sep 17 00:00:00 2001
From: Felix Breidenstein <mail@felixbreidenstein.de>
Date: Thu, 5 Nov 2020 00:11:23 +0100
Subject: [PATCH] Lot of fixing. Graphs on index page. Web importer
---
README.md | 2 -
schmeckels/autosort.py | 5 +-
schmeckels/banks.py | 18 +++++---
schmeckels/cli.py | 3 +-
schmeckels/helper.py | 31 ++++++++++++-
schmeckels/importer.py | 39 ++++++----------
schmeckels/models.py | 10 ++--
schmeckels/serve.py | 75 ++++++++++++++++++++++++++++--
schmeckels/templates/base.html | 15 +++++-
schmeckels/templates/index.html | 79 +++++++++++++++++++++++++++++++-
schmeckels/templates/tags.html | 9 +++-
schmeckels/templates/upload.html | 24 ++++++++++
12 files changed, 263 insertions(+), 47 deletions(-)
create mode 100644 schmeckels/templates/upload.html
diff --git a/README.md b/README.md
index e4e550d..a3f6f80 100644
--- a/README.md
+++ b/README.md
@@ -8,8 +8,6 @@
- Übersicht auf Startseite
- Monatsübersicht dieser/letzter Monat
- Anzahl ungetaggte Transaktionen
-- Importer über Webseite
-- setup.py
- Doku
diff --git a/schmeckels/autosort.py b/schmeckels/autosort.py
index e2656dd..f75c471 100644
--- a/schmeckels/autosort.py
+++ b/schmeckels/autosort.py
@@ -46,6 +46,7 @@ def command(profile, dry_run, verbose):
def apply_rules(t, rules):
+ tags = []
for r in rules:
iban = r.get("iban")
name_regex = r.get("name")
@@ -63,4 +64,6 @@ def apply_rules(t, rules):
if not re.search(desc_regex, t.description):
continue
- return r["tags"].split(",")
+ tags.extend(r["tags"].split(","))
+
+ return tags
diff --git a/schmeckels/banks.py b/schmeckels/banks.py
index 63a1043..c5fd4bd 100644
--- a/schmeckels/banks.py
+++ b/schmeckels/banks.py
@@ -1,9 +1,15 @@
#! /usr/bin/env python3
import mt940
import csv
-from schmeckels.models import Transaction
+from schmeckels import models
from datetime import datetime
+SUPPORTED_BANKS = (
+ ("dkb", "DKB"),
+ ("sparkasse-mt940", "Sparkasse MT940"),
+ ("bunq-csv", "Bunq CSV"),
+)
+
class Bank(object):
"""
@@ -38,7 +44,7 @@ class Sparkasse_MT940(Bank):
"""
def get_transactions(self):
- transactions = mt940.parse(click.format_filename(filename))
+ transactions = mt940.parse(self.filename)
for t in transactions:
data = t.data
@@ -47,7 +53,7 @@ class Sparkasse_MT940(Bank):
iban = data["applicant_iban"]
name = data["applicant_name"]
description = data["purpose"]
- yield Transaction(date=date, name=name, iban=iban, amount=amount, description=description)
+ yield models.Transaction(date=date, name=name, iban=iban, amount=amount, description=description)
class Bunq(Bank):
@@ -66,7 +72,7 @@ class Bunq(Bank):
name = line[5]
description = line[6]
- yield Transaction(date=date, name=name, iban=iban, amount=amount, description=description)
+ yield models.Transaction(date=date, name=name, iban=iban, amount=amount, description=description)
class DKB(Bank):
@@ -75,7 +81,7 @@ class DKB(Bank):
"""
def get_transactions(self):
- with open(filename, encoding="ISO-8859-1") as fh:
+ with open(self.filename, encoding="ISO-8859-1") as fh:
csv_reader = csv.reader(fh, delimiter=";")
next(csv_reader, None)
for line in csv_reader:
@@ -84,4 +90,4 @@ class DKB(Bank):
iban = line[5]
name = line[3]
description = line[4]
- yield Transaction(date=date, name=name, iban=iban, amount=amount, description=description)
+ yield models.Transaction(date=date, name=name, iban=iban, amount=amount, description=description)
diff --git a/schmeckels/cli.py b/schmeckels/cli.py
index 163a073..f65b86e 100644
--- a/schmeckels/cli.py
+++ b/schmeckels/cli.py
@@ -5,7 +5,7 @@ import sys
from pathlib import Path
-from schmeckels import importer, serve, autosort, validate, info, sort, models #,report
+from schmeckels import importer, serve, autosort, validate, info, sort, models # ,report
from schmeckels.helper import build_database_filename, create_dirs, build_rules_filename
from sqlalchemy import create_engine
@@ -15,6 +15,7 @@ from sqlalchemy.ext.declarative import declarative_base
__version__ = "0.0.1"
+
@click.group()
def cli():
pass
diff --git a/schmeckels/helper.py b/schmeckels/helper.py
index 185d4be..641cbe2 100644
--- a/schmeckels/helper.py
+++ b/schmeckels/helper.py
@@ -8,6 +8,7 @@ from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import schmeckels.models
import locale
+from schmeckels import banks, models
try:
from yaml import CLoader as Loader
@@ -45,7 +46,7 @@ def check_single_profile():
if len(files) == 1:
return files[0].split(".")[0]
else:
- print("--profile is required when you have more than one database.")
+ print("--profile is required when you have more than one profile.")
sys.exit(1)
@@ -88,7 +89,35 @@ def get_rules(profile_name):
def create_tag(name, profile, session):
t = session.query(models.Tag).filter(models.Tag.name == name).first()
if not t:
+ print(name)
t = models.Tag(name=name)
session.add(t)
session.commit()
return t
+
+
+def import_transactions(session, filetype, filename):
+ if filetype not in [b[0] for b in banks.SUPPORTED_BANKS]:
+ raise ValueError("Unsupported filetype")
+
+ new = []
+
+ if filetype == "sparkasse-mt940":
+ bank = banks.Sparkasse_MT940(filename)
+ elif filetype == "bunq-csv":
+ bank = banks.Bunq(filename)
+ elif filetype == "dkb":
+ bank = banks.DKB(filename)
+
+ for t in bank.get_transactions():
+ if (
+ not session.query(models.Transaction)
+ .filter_by(iban=t.iban)
+ .filter_by(amount=t.amount)
+ .filter_by(date=t.date)
+ .filter_by(description=t.description)
+ .first()
+ ):
+ new.append(t)
+
+ return new
diff --git a/schmeckels/importer.py b/schmeckels/importer.py
index d4133c0..6a5dd7d 100644
--- a/schmeckels/importer.py
+++ b/schmeckels/importer.py
@@ -3,38 +3,25 @@ import click
from sqlalchemy import create_engine, desc
from sqlalchemy.orm import sessionmaker
from schmeckels.models import Transaction
-from schmeckels.helper import get_session
+from schmeckels.helper import get_session, import_transactions
import sys
from schmeckels import banks
+import colorful as cf
@click.command(name="import")
-@click.option("--filetype", "-t", type=click.Choice(["dkb", "sparkasse-mt940", "bunq-csv"], case_sensitive=False))
+@click.option("--filetype", "-t", type=click.Choice([b[0] for b in banks.SUPPORTED_BANKS], case_sensitive=False))
@click.option("--profile", "-p")
-@click.option("--force", "-f", default=False, is_flag=True)
+@click.option("--dry-run", default=False, is_flag=True)
@click.argument("filename", type=click.Path(exists=True))
-def command(filename, profile, force, filetype):
+def command(filetype, profile, dry_run, filename):
session = get_session(profile)
+ new = import_transactions(session, filetype, filename)
- latest = session.query(Transaction).order_by(desc(Transaction.date)).first()
- new = []
-
- if filetype == "sparkasse-mt940":
- bank = banks.Sparkasse_MT940(filename)
- elif filetype == "bunq-csv":
- bank = banks.Bunq(filename)
- elif filetype == "dkb":
- bank = banks.DKB(filename)
-
- for t in bank.get_transactions():
- if not force and latest and t.date < latest.date:
- print("Found a transaction older than then newest transction in the DB.")
- print("If you are sure that you want to import this file, use --force")
- sys.exit(1)
-
- new.append(t)
- print(".", end="", flush=True)
-
- session.bulk_save_objects(new)
- session.commit()
- print(f"Imported {len(new)} transactions")
+ if dry_run:
+ print(cf.yellow(f"Would have imported {len(new)} new transactions"))
+ else:
+ if len(new) > 0:
+ session.bulk_save_objects(new)
+ session.commit()
+ print(cf.green(f"Imported {len(new)} new transactions"))
diff --git a/schmeckels/models.py b/schmeckels/models.py
index 4535416..9701722 100644
--- a/schmeckels/models.py
+++ b/schmeckels/models.py
@@ -1,8 +1,8 @@
#! /usr/bin/env python3
from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Table
+from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, Table, Boolean
from sqlalchemy.orm import relationship, backref
-from schmeckels.helper import format_amount
+from schmeckels import helper
from hashlib import md5
Base = declarative_base()
@@ -32,7 +32,7 @@ class Transaction(Base):
return self.amount > 0
def pretty_amount(self):
- return format_amount(self.amount)
+ return helper.format_amount(self.amount)
def get_date(self, format):
if format == "iso":
@@ -47,12 +47,16 @@ class Tag(Base):
__tablename__ = "Tag"
id = Column(Integer, primary_key=True)
name = Column(String, unique=True)
+ reporting = Column(Boolean, default=True)
description = Column(String)
transactions = relationship("Transaction", secondary=association_table, back_populates="tags")
def __repr__(self):
return f"<Tag {self.name}>"
+ def toggle_reporting(self):
+ self.reporting = not self.reporting
+
def color(self):
hash = md5(self.name.encode()).hexdigest()[-6:]
return f"#{hash}"
diff --git a/schmeckels/serve.py b/schmeckels/serve.py
index 765930a..4ca0043 100644
--- a/schmeckels/serve.py
+++ b/schmeckels/serve.py
@@ -1,10 +1,13 @@
#! /usr/bin/env python3
-from flask import Flask, render_template, request, redirect
+from flask import Flask, render_template, request, redirect, flash
from schmeckels.helper import get_session, format_amount
-from datetime import datetime
-from sqlalchemy import func
+from datetime import datetime, timedelta, date
+from sqlalchemy import func, and_
import click
+import calendar
from schmeckels.models import Transaction, Tag
+from schmeckels.helper import import_transactions
+from schmeckels import banks
session = None
app = Flask(__name__)
@@ -12,7 +15,37 @@ app = Flask(__name__)
@app.route("/")
def index():
- return render_template("index.html")
+ tag_sum_in = {}
+ tag_sum_out = {}
+
+ # Get a list of all tags
+ tags = session.query(Tag).all()
+
+ # Get start and end of last month
+ now = datetime.now()
+ last_month = now - timedelta(days=now.day)
+ num_days = calendar.monthrange(last_month.year, 2)[1]
+ start_date = date(last_month.year, 2, 1)
+ end_date = date(last_month.year, 2, num_days)
+
+ for tag in tags:
+ tag_sum = (
+ session.query(func.sum(Transaction.amount))
+ .filter(and_(Transaction.date >= start_date, Transaction.date <= end_date))
+ .filter(Transaction.tags.any(id=tag.id))
+ .first()[0]
+ )
+ if tag_sum:
+ if tag_sum >= 0:
+ tag_sum_in[tag] = tag_sum
+ else:
+ tag_sum_out[tag] = tag_sum
+
+ tag_sum_in = {
+ k: format_amount(v) for k, v in sorted(tag_sum_in.items(), key=lambda item: float(item[1]), reverse=True)
+ }
+ tag_sum_out = {k: format_amount(v) for k, v in sorted(tag_sum_out.items(), key=lambda item: float(item[1]))}
+ return render_template("index.html", tag_sum_in=tag_sum_in, tag_sum_out=tag_sum_out, date=start_date)
@app.route("/tags")
@@ -39,6 +72,17 @@ def delete_tag(name):
return redirect("/tags")
+@app.route("/tag/<name>/toggle")
+def toggle_tag(name):
+ x = session.query(Tag).filter(Tag.name == name).first()
+ if x:
+ x.toggle_reporting()
+ session.add(x)
+ session.commit()
+
+ return redirect("/tags")
+
+
@app.route("/transactions")
def transactions():
transactions = session.query(Transaction)
@@ -67,9 +111,32 @@ def buksort():
return redirect("/transactions")
+@app.route("/upload", methods=["GET", "POST"])
+def upload():
+ global session
+
+ if request.method == "POST":
+ file = request.files.get("file")
+ if not file or file.filename == "":
+ flash("No file choosen")
+ return redirect(request.url)
+
+ path = f"/tmp/{file.filename}"
+ file.save(path)
+ new = import_transactions(session, request.form["bank"], path)
+ if len(new) > 0:
+ session.bulk_save_objects(new)
+ session.commit()
+ flash(f"Imported {len(new)} new transactions")
+ return render_template("upload.html", banks=banks.SUPPORTED_BANKS, transactions=new)
+
+ return render_template("upload.html", banks=banks.SUPPORTED_BANKS)
+
+
@click.command(name="serve")
@click.option("--profile", "-p")
def command(profile):
global session
session = get_session(profile)
+ app.secret_key = "WeDon'tHaveSessionsSoThisMustNotBeVerySecure!"
app.run(host="0.0.0.0", port=8080)
diff --git a/schmeckels/templates/base.html b/schmeckels/templates/base.html
index b114fd6..8bc721c 100644
--- a/schmeckels/templates/base.html
+++ b/schmeckels/templates/base.html
@@ -7,7 +7,6 @@
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Schmeckels</title>
<link href="https://unpkg.com/tailwindcss/dist/tailwind.min.css" rel="stylesheet">
- <link href="https://unpkg.com/@tailwindcss/custom-forms/dist/custom-forms.min.css" rel="stylesheet">
<link href="/static/style.css" rel="stylesheet">
</head>
@@ -26,13 +25,27 @@
<li class="mx-4">
<a class="text-blue-500 hover:text-blue-800" href="/tags">Tags</a>
</li>
+ <li class="mx-4">
+ <a class="text-blue-500 hover:text-blue-800" href="/upload">Upload</a>
+ </li>
</ul>
</div>
<div class="pt-2 container mx-auto">
+ {% with messages = get_flashed_messages() %}
+ {% if messages %}
+ {% for message in messages %}
+ <div class="bg-blue-100 border-t border-b border-blue-500 text-blue-700 px-2 py-2 mb-5" role="alert">
+ <p class="font-bold">{{ message }}</p>
+ </div>
+ {% endfor %}
+ {% endif %}
+ {% endwith %}
{% block main %} {% endblock %}
</div>
+ <script src='https://cdnjs.cloudflare.com/ajax/libs/Chart.js/1.0.2/Chart.min.js'></script>
+ {% block script %} {% endblock %}
</body>
</html>
diff --git a/schmeckels/templates/index.html b/schmeckels/templates/index.html
index fee384d..898e55b 100644
--- a/schmeckels/templates/index.html
+++ b/schmeckels/templates/index.html
@@ -1,7 +1,84 @@
{% extends "base.html" %}
{% block main %}
-<h1 class="text-2xl font-bold text-indigo-500">Start</h1>
+<h1 class="text-2xl font-bold text-indigo-500">Overview for {{ date.strftime("%B %Y") }} </h1>
+<h2 class="text-xl font-bold text-green-500">Incoming</h2>
+<div style="display:flex; flex-direction: row;">
+ <div>
+ {% if tag_sum_in %}
+ <table>
+ <tr>
+ <th>Tag</th>
+ <th>Sum</th>
+ </tr>
+ {% for tag,sum in tag_sum_in.items() %}
+ <tr>
+ <td>{{ tag.name }}</td>
+ <td class="text-right">{{ sum }} €</td>
+ </tr>
+ {% endfor %}
+ </table>
+ {% else %}
+ No tagged transactions for this month.
+ {% endif %}
+ </div>
+ <div>
+ <canvas id="chart_in" width="600" height="400"></canvas>
+ </div>
+</div>
+
+
+<h2 class="text-xl font-bold text-red-500">Outgoing</h2>
+<div style="display:flex; flex-direction: row;">
+ <div>
+ {% if tag_sum_out %}
+ <table>
+ <tr>
+ <th>Tag</th>
+ <th>Sum</th>
+ </tr>
+ {% for tag,sum in tag_sum_out.items() %}
+ <tr>
+ <td>{{ tag.name }}</td>
+ <td class="text-right">{{ sum }} €</td>
+ </tr>
+ {% endfor %}
+ </table>
+ {% else %}
+ No tagged transactions for this month.
+ {% endif %}
+ </div>
+ <div>
+ <canvas id="chart_out" width="600" height="400"></canvas>
+ </div>
+</div>
{% endblock %}
+
+{% block script %}
+<script>
+ var pieData_in = [
+ {% for tag, sum in tag_sum_in.items() %}
+ {
+ value: "{{sum}}".replace(",", ""),
+ label: "{{ tag.name }}",
+ color : "{{ tag.color() }}"
+ },
+ {% endfor %}
+ ];
+ new Chart(document.getElementById("chart_in").getContext("2d")).Pie(pieData_in);
+
+ var pieData_out = [
+ {% for tag, sum in tag_sum_out.items() %}
+ {
+ value: "{{sum}}".replace(",", ""),
+ label: "{{ tag.name }}",
+ color : "{{ tag.color() }}"
+ },
+ {% endfor %}
+ ];
+ new Chart(document.getElementById("chart_out").getContext("2d")).Pie(pieData_out);
+
+</script>
+{% endblock %}
diff --git a/schmeckels/templates/tags.html b/schmeckels/templates/tags.html
index b54ee14..869e72f 100644
--- a/schmeckels/templates/tags.html
+++ b/schmeckels/templates/tags.html
@@ -7,13 +7,20 @@
<ul>
{% for tag in tags %}
<li class="p-2">
- <span style="background-color: {{ tag.color() }};" class="text-white rounded p-1" >{{ tag.name }}</span>
+ <span style="background-color: {{ tag.color() }};" class="text-white rounded p-1">{{ tag.name }}</span>
<a class="float-right" href="/tag/{{ tag.name }}">
<button class="bg-blue-500 hover:bg-blue-700 text-white font-bold mx-2 y-1 px-2 rounded"> Transactions </button>
</a>
<a class="float-right" href="/tag/{{ tag.name }}/delete">
<button class="bg-red-500 hover:bg-red-700 text-white font-bold y-1 px-2 rounded"> Delete </button>
</a>
+ <a class="float-right" href="/tag/{{ tag.name }}/toggle">
+ {% if tag.reporting %}
+ <button class="bg-red-500 hover:bg-red-700 text-white font-bold y-1 px-2 rounded"> Remove from report </button>
+ {% else %}
+ <button class="bg-green-500 hover:bg-green-700 text-white font-bold y-1 px-2 rounded"> Add to reporting </button>
+ {% endif %}
+ </a>
</li>
{% endfor %}
</ul>
diff --git a/schmeckels/templates/upload.html b/schmeckels/templates/upload.html
new file mode 100644
index 0000000..990f745
--- /dev/null
+++ b/schmeckels/templates/upload.html
@@ -0,0 +1,24 @@
+{% extends "base.html" %}
+
+{% block main %}
+<form method="POST" enctype="multipart/form-data" class="max-w-xs">
+ <p class="bg-blue-100 p-2 mb-4">
+ <label class="block" for="bank">Choose your filetype</label>
+ <select id="bank" name="bank">
+ {% for bank in banks %}
+ <option value="{{bank[0]}}"> {{ bank[1] }}</option>
+ {% endfor %}
+ </select>
+ </p>
+
+ <p class="bg-blue-100 p-2 mb-4">
+ <label class="block" for="file">Choose your file</label>
+ <input name="file" type="file"></input>
+ </p>
+
+ <p class="mt-4 text-center">
+ <button class="bg-green-400 p-1" type="submit">Upload</button>
+ </p>
+</form>
+
+{% endblock %}