diff --git a/docs/usage.rst b/docs/usage.rst index 8352f91f0..af7bc38ee 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -169,15 +169,19 @@ Building your KPI Expressions can be any valid python expressions. The following special elements are recognized in the expressions to compute accounting -data: {bal|crd|deb}{pieu}[account selector][journal items domain]. - -* bal, crd, deb: balance, debit, credit. -* p, i, e: respectively variation over the period, initial balance, ending balance -* The account selector is a like expression on the account code (eg 70%, etc). +data: ``{bal|crd|deb|pbal|nbal|fld}{pieu}(.fieldname)?[account selector][journal items domain]``. + +* ``bal``, ``crd``, ``deb``: balance, debit, credit. +* ``pbal``, ``nbal``: positive and negative balances only +* ``fld``: custom numerical field +* ``p``, ``i``, ``e``: respectively variation over the period, initial balance, ending balance +* .fieldname: when ``fld`` is used, the field name to use (eg ``fldp.quantity``). +* The account selector is a like expression on the account code (eg ``[70%]``, etc), + or a domain over accounts (eg ``[("tag_ids.name", "=", "mytag")]``). * The journal items domain is an Odoo domain filter on journal items. -* balu[]: (u for unallocated) is a special expression that shows the unallocated +* ``balu[]``: (u for unallocated) is a special expression that shows the unallocated profit/loss of previous fiscal years. -* Expression can also involve other KPI and query results by name (eg kpi1 + kpi2). +* Expression can also involve other KPI and query results by name (eg ``kpi1 + kpi2``). Additionally following variables are available in the evaluation context: @@ -188,16 +192,18 @@ Additionally following variables are available in the evaluation context: Examples ******** -* bal[70]: variation of the balance of account 70 over the period (it is the same as balp[70]. -* bali[70,60]: initial balance of accounts 70 and 60. -* bale[1%]: balance of accounts starting with 1 at end of period. -* crdp[40%]: sum of all credits on accounts starting with 40 during the period. -* debp[55%][('journal_id.code', '=', 'BNK1')]: sum of all debits on accounts 55 and +* ``bal[70]``: variation of the balance of account 70 over the period (it is the same as balp[70]. +* ``bali[70,60]``: initial balance of accounts 70 and 60. +* ``bale[1%]``: balance of accounts starting with 1 at end of period. +* ``crdp[40%]``: sum of all credits on accounts starting with 40 during the period. +* ``debp[55%][('journal_id.code', '=', 'BNK1')]``: sum of all debits on accounts 55 and journal BNK1 during the period. -* balp[('user_type_id', '=', ref('account.data_account_type_receivable').id)][]: +* ``balp[('user_type_id', '=', ref('account.data_account_type_receivable').id)][]``: variation of the balance of all receivable accounts over the period. -* balp[][('tax_line_id.tag_ids', '=', ref('l10n_be.tax_tag_56').id)]: balance of move +* ``balp[][('tax_line_id.tag_ids', '=', ref('l10n_be.tax_tag_56').id)]``: balance of move lines related to tax grid 56. +* ``fldp.quantity[60%]``: sum of the quantity field of all move lines on accounts starting + with 60. Expansion of Account Detail --------------------------- diff --git a/mis_builder/models/aep.py b/mis_builder/models/aep.py index 7f93a742e..5fb0d1781 100644 --- a/mis_builder/models/aep.py +++ b/mis_builder/models/aep.py @@ -12,7 +12,6 @@ from odoo.tools.safe_eval import datetime, dateutil, safe_eval, time from .accounting_none import AccountingNone -from .simple_array import SimpleArray _logger = logging.getLogger(__name__) @@ -24,15 +23,65 @@ def _is_domain(s): return _DOMAIN_START_RE.match(s) +class Accumulator: + """A simple class to accumulate debit, credit and custom field values. + + >>> acc1 = Accumulator(["f1", "f2"]) + >>> acc1.debit + AccountingNone + >>> acc1.credit + AccountingNone + >>> acc1.custom_fields + {'f1': AccountingNone, 'f2': AccountingNone} + >>> acc1.add_debit_credit(10, 20) + >>> acc1.debit, acc1.credit + (10, 20) + >>> acc1.add_custom_field("f1", 10) + >>> acc1.custom_fields + {'f1': 10, 'f2': AccountingNone} + >>> acc2 = Accumulator(["f1", "f2"]) + >>> acc2.add_debit_credit(21, 31) + >>> acc2.add_custom_field("f2", 41) + >>> acc1 += acc2 + >>> acc1.debit, acc1.credit + (31, 51) + >>> acc1.custom_fields + {'f1': 10, 'f2': 41} + """ + + def __init__(self, custom_field_names=()): + self.debit = AccountingNone + self.credit = AccountingNone + self.custom_fields = { + custom_field: AccountingNone for custom_field in custom_field_names + } + + def add_debit_credit(self, debit, credit): + self.debit += debit + self.credit += credit + + def add_custom_field(self, field, value): + self.custom_fields[field] += value + + def __iadd__(self, other): + self.debit += other.debit + self.credit += other.credit + for field in self.custom_fields: + self.custom_fields[field] += other.custom_fields[field] + return self + + class AccountingExpressionProcessor: """Processor for accounting expressions. - Expressions of the form [accounts][optional move line domain] + Expressions of the form + (.fieldname)?[accounts][optional move line domain] are supported, where: * field is bal, crd, deb, pbal (positive balances only), - nbal (negative balance only) + nbal (negative balance only), fld (custom field) * mode is i (initial balance), e (ending balance), p (moves over period) + * .fieldname is used only with fldp and specifies the field name to sum * there is also a special u mode (unallocated P&L) which computes the sum from the beginning until the beginning of the fiscal year of the period; it is only meaningful for P&L accounts @@ -46,6 +95,7 @@ class AccountingExpressionProcessor: over the period (it is the same as balp[70]); * bali[70,60]: balance of accounts 70 and 60 at the start of period; * bale[1%]: balance of accounts starting with 1 at end of period. + * fldp.quantity[60%]: sum of the quantity field of moves on accounts 60 How to use: * repeatedly invoke parse_expr() for each expression containing @@ -77,8 +127,9 @@ class AccountingExpressionProcessor: MODE_UNALLOCATED = "u" _ACC_RE = re.compile( - r"(?P\bbal|\bpbal|\bnbal|\bcrd|\bdeb)" + r"(?P\bbal|\bpbal|\bnbal|\bcrd|\bdeb|\bfld)" r"(?P[piseu])?" + r"(?P\.[a-zA-Z0-9_]+)?" r"\s*" r"(?P_[a-zA-Z0-9]+|\[.*?\])" r"\s*" @@ -110,6 +161,8 @@ def __init__(self, companies, currency=None, account_model="account.account"): # a first query to get the initial balance and another # to get the variation, so it's a bit slower self.smart_end = True + # custom field to query and sum + self._custom_fields = set() # Account model self._account_model = self.env[account_model].with_context(active_test=False) @@ -129,7 +182,7 @@ def _account_codes_to_domain(self, account_codes): def _parse_match_object(self, mo): """Split a match object corresponding to an accounting variable - Returns field, mode, account domain, move line domain. + Returns field, mode, fld_name, account domain, move line domain. """ domain_eval_context = { "ref": self.env.ref, @@ -138,12 +191,16 @@ def _parse_match_object(self, mo): "datetime": datetime, "dateutil": dateutil, } - field, mode, account_sel, ml_domain = mo.groups() + field, mode, fld_name, account_sel, ml_domain = mo.groups() # handle some legacy modes if not mode: mode = self.MODE_VARIATION elif mode == "s": mode = self.MODE_END + # custom fields + if fld_name: + assert fld_name[0] == "." + fld_name = fld_name[1:] # strip leading dot # convert account selector to account domain if account_sel.startswith("_"): # legacy bal_NNN% @@ -166,7 +223,7 @@ def _parse_match_object(self, mo): ml_domain = tuple(safe_eval(ml_domain, domain_eval_context)) else: ml_domain = tuple() - return field, mode, acc_domain, ml_domain + return field, mode, fld_name, acc_domain, ml_domain def parse_expr(self, expr): """Parse an expression, extracting accounting variables. @@ -177,7 +234,7 @@ def parse_expr(self, expr): and mode. """ for mo in self._ACC_RE.finditer(expr): - _, mode, acc_domain, ml_domain = self._parse_match_object(mo) + field, mode, fld_name, acc_domain, ml_domain = self._parse_match_object(mo) if mode == self.MODE_END and self.smart_end: modes = (self.MODE_INITIAL, self.MODE_VARIATION, self.MODE_END) else: @@ -185,6 +242,30 @@ def parse_expr(self, expr): for mode in modes: key = (ml_domain, mode) self._map_account_ids[key].add(acc_domain) + if field == "fld": + if mode != self.MODE_VARIATION: + raise UserError( + _( + "`fld` can only be used with mode `p` (variation) " + "in expression %s", + expr, + ) + ) + if not fld_name: + raise UserError( + _("`fld` must have a field name in exression %s", expr) + ) + self._custom_fields.add(fld_name) + else: + if fld_name: + raise UserError( + _( + "`%(field)s` cannot have a field name " + "in expression %(expr)s", + field=field, + expr=expr, + ) + ) def done_parsing(self): """Replace account domains by account ids in map""" @@ -211,7 +292,7 @@ def get_account_ids_for_expr(self, expr): """ account_ids = set() for mo in self._ACC_RE.finditer(expr): - field, mode, acc_domain, ml_domain = self._parse_match_object(mo) + _, _, _, acc_domain, _ = self._parse_match_object(mo) account_ids.update(self._account_ids_by_acc_domain[acc_domain]) return account_ids @@ -225,7 +306,7 @@ def get_aml_domain_for_expr(self, expr, date_from, date_to, account_id=None): aml_domains = [] date_domain_by_mode = {} for mo in self._ACC_RE.finditer(expr): - field, mode, acc_domain, ml_domain = self._parse_match_object(mo) + field, mode, fld_name, acc_domain, ml_domain = self._parse_match_object(mo) aml_domain = list(ml_domain) account_ids = set() account_ids.update(self._account_ids_by_acc_domain[acc_domain]) @@ -241,6 +322,8 @@ def get_aml_domain_for_expr(self, expr, date_from, date_to, account_id=None): aml_domain.append(("credit", "<>", 0.0)) elif field == "deb": aml_domain.append(("debit", "<>", 0.0)) + elif fld_name: + aml_domain.append((fld_name, "!=", False)) aml_domains.append(expression.normalize_domain(aml_domain)) if mode not in date_domain_by_mode: date_domain_by_mode[mode] = self.get_aml_domain_for_dates( @@ -316,10 +399,10 @@ def do_queries( aml_model = self.env[aml_model] aml_model = aml_model.with_context(active_test=False) company_rates = self._get_company_rates(date_to) - # {(domain, mode): {account_id: (debit, credit)}} + # {(domain, mode): {account_id: Accumulator}} self._data = defaultdict( lambda: defaultdict( - lambda: SimpleArray((AccountingNone, AccountingNone)), + lambda: Accumulator(self._custom_fields), ) ) domain_by_mode = {} @@ -343,7 +426,13 @@ def do_queries( try: accs = aml_model.read_group( domain, - ["debit", "credit", "account_id", "company_id"], + [ + "debit", + "credit", + "account_id", + "company_id", + *self._custom_fields, + ], ["account_id", "company_id"], lazy=False, ) @@ -369,9 +458,15 @@ def do_queries( ): # in initial mode, ignore accounts with 0 balance continue - # due to branches, it's possible to have multiple acc - # with the same account_id - self._data[key][acc["account_id"][0]] += (debit * rate, credit * rate) + # due to branches, it's possible to have multiple groups + # with the same account_id, because multiple companies can + # use the same account + account_data = self._data[key][acc["account_id"][0]] + account_data.add_debit_credit(debit * rate, credit * rate) + for field_name in self._custom_fields: + account_data.add_custom_field( + field_name, acc[field_name] or AccountingNone + ) # compute ending balances by summing initial and variation for key in ends: domain, mode = key @@ -379,11 +474,8 @@ def do_queries( variation_data = self._data[(domain, self.MODE_VARIATION)] account_ids = set(initial_data.keys()) | set(variation_data.keys()) for account_id in account_ids: - di, ci = initial_data.get(account_id, (AccountingNone, AccountingNone)) - dv, cv = variation_data.get( - account_id, (AccountingNone, AccountingNone) - ) - self._data[key][account_id] = (di + dv, ci + cv) + self._data[key][account_id] += initial_data[account_id] + self._data[key][account_id] += variation_data[account_id] def replace_expr(self, expr): """Replace accounting variables in an expression by their amount. @@ -394,25 +486,30 @@ def replace_expr(self, expr): """ def f(mo): - field, mode, acc_domain, ml_domain = self._parse_match_object(mo) + field, mode, fld_name, acc_domain, ml_domain = self._parse_match_object(mo) key = (ml_domain, mode) account_ids_data = self._data[key] v = AccountingNone account_ids = self._account_ids_by_acc_domain[acc_domain] for account_id in account_ids: - debit, credit = account_ids_data.get( - account_id, (AccountingNone, AccountingNone) - ) + entry = account_ids_data[account_id] + debit = entry.debit + credit = entry.credit if field == "bal": v += debit - credit - elif field == "pbal" and debit >= credit: - v += debit - credit - elif field == "nbal" and debit < credit: - v += debit - credit + elif field == "pbal": + if debit >= credit: + v += debit - credit + elif field == "nbal": + if debit < credit: + v += debit - credit elif field == "deb": v += debit elif field == "crd": v += credit + else: + assert field == "fld" + v += entry.custom_fields[fld_name] # in initial balance mode, assume 0 is None # as it does not make sense to distinguish 0 from "no data" if ( @@ -435,7 +532,7 @@ def replace_exprs_by_account_id(self, exprs): """ def f(mo): - field, mode, acc_domain, ml_domain = self._parse_match_object(mo) + field, mode, fld_name, acc_domain, ml_domain = self._parse_match_object(mo) key = (ml_domain, mode) # first check if account_id is involved in # the current expression part @@ -443,9 +540,9 @@ def f(mo): return "(AccountingNone)" # here we know account_id is involved in acc_domain account_ids_data = self._data[key] - debit, credit = account_ids_data.get( - account_id, (AccountingNone, AccountingNone) - ) + entry = account_ids_data[account_id] + debit = entry.debit + credit = entry.credit if field == "bal": v = debit - credit elif field == "pbal": @@ -462,6 +559,9 @@ def f(mo): v = debit elif field == "crd": v = credit + else: + assert field == "fld" + v = entry.custom_fields[fld_name] # in initial balance mode, assume 0 is None # as it does not make sense to distinguish 0 from "no data" if ( @@ -475,7 +575,7 @@ def f(mo): account_ids = set() for expr in exprs: for mo in self._ACC_RE.finditer(expr): - field, mode, acc_domain, ml_domain = self._parse_match_object(mo) + _, mode, _, acc_domain, ml_domain = self._parse_match_object(mo) key = (ml_domain, mode) account_ids_data = self._data[key] for account_id in self._account_ids_by_acc_domain[acc_domain]: @@ -495,7 +595,7 @@ def _get_balances(cls, mode, companies, date_from, date_to): aep.parse_expr(expr) aep.done_parsing() aep.do_queries(date_from, date_to) - return aep._data[((), mode)] + return {k: (v.debit, v.credit) for k, v in aep._data[((), mode)].items()} @classmethod def get_balances_initial(cls, companies, date): diff --git a/mis_builder/models/aggregate.py b/mis_builder/models/aggregate.py index 32e34bd95..109464f2f 100644 --- a/mis_builder/models/aggregate.py +++ b/mis_builder/models/aggregate.py @@ -64,11 +64,11 @@ def _min(*args): >>> min() Traceback (most recent call last): File "", line 1, in ? - TypeError: min expected 1 arguments, got 0 + TypeError: min expected at least 1 argument, got 0 >>> _min() Traceback (most recent call last): File "", line 1, in ? - TypeError: min expected 1 arguments, got 0 + TypeError: min expected at least 1 argument, got 0 >>> min([]) Traceback (most recent call last): File "", line 1, in ? @@ -107,11 +107,11 @@ def _max(*args): >>> max() Traceback (most recent call last): File "", line 1, in ? - TypeError: max expected 1 arguments, got 0 + TypeError: max expected at least 1 argument, got 0 >>> _max() Traceback (most recent call last): File "", line 1, in ? - TypeError: max expected 1 arguments, got 0 + TypeError: max expected at least 1 argument, got 0 >>> max([]) Traceback (most recent call last): File "", line 1, in ? diff --git a/mis_builder/tests/test_aep.py b/mis_builder/tests/test_aep.py index 9cb23e53c..5b1e9d493 100644 --- a/mis_builder/tests/test_aep.py +++ b/mis_builder/tests/test_aep.py @@ -9,9 +9,13 @@ from odoo.exceptions import UserError from odoo.tools.safe_eval import safe_eval +from ..models import aep from ..models.accounting_none import AccountingNone from ..models.aep import AccountingExpressionProcessor as AEP from ..models.aep import _is_domain +from .common import load_doctests + +load_tests = load_doctests(aep) class TestAEP(common.TransactionCase): @@ -73,6 +77,7 @@ def setUp(self): amount=300, debit_acc=self.account_ar, credit_acc=self.account_in, + credit_quantity=3, ) # create move in March this year self._create_move( @@ -98,6 +103,7 @@ def setUp(self): self.aep.parse_expr("crdp[700I%]") self.aep.parse_expr("bali[400%]") self.aep.parse_expr("bale[700%]") + self.aep.parse_expr("fldp.quantity[700%]") self.aep.parse_expr("balp[]" "[('account_id.code', '=', '400AR')]") self.aep.parse_expr( "balp[]" "[('account_id.account_type', '=', " " 'asset_receivable')]" @@ -112,17 +118,32 @@ def setUp(self): self.aep.parse_expr("bal_700IN") # deprecated self.aep.parse_expr("bals[700IN]") # deprecated - def _create_move(self, date, amount, debit_acc, credit_acc, post=True): + def _create_move( + self, date, amount, debit_acc, credit_acc, post=True, credit_quantity=0 + ): move = self.move_model.create( { "journal_id": self.journal.id, "date": fields.Date.to_string(date), "line_ids": [ - (0, 0, {"name": "/", "debit": amount, "account_id": debit_acc.id}), ( 0, 0, - {"name": "/", "credit": amount, "account_id": credit_acc.id}, + { + "name": "/", + "debit": amount, + "account_id": debit_acc.id, + }, + ), + ( + 0, + 0, + { + "name": "/", + "credit": amount, + "account_id": credit_acc.id, + "quantity": credit_quantity, + }, ), ], } @@ -152,6 +173,20 @@ def test_sanity_check(self): self.assertEqual(self.company.fiscalyear_last_day, 31) self.assertEqual(self.company.fiscalyear_last_month, "12") + def test_parse_expr_error_handling(self): + aep = AEP(self.company) + with self.assertRaises(UserError) as cm: + aep.parse_expr("fldi.quantity[700%]") + self.assertIn( + "`fld` can only be used with mode `p` (variation)", str(cm.exception) + ) + with self.assertRaises(UserError) as cm: + aep.parse_expr("fldp[700%]") + self.assertIn("`fld` must have a field name", str(cm.exception)) + with self.assertRaises(UserError) as cm: + aep.parse_expr("balp.quantity[700%]") + self.assertIn("`bal` cannot have a field name", str(cm.exception)) + def test_aep_basic(self): self.aep.done_parsing() # let's query for december @@ -203,6 +238,8 @@ def test_aep_basic(self): self.assertEqual(self._eval("bale[700IN]"), -300) # check result for non existing account self.assertIs(self._eval("bale[700NA]"), AccountingNone) + # check fldp.quantity + self.assertEqual(self._eval("fldp.quantity[700%]"), 3) # let's query for March self._do_queries( @@ -234,6 +271,8 @@ def test_aep_basic(self): self.assertEqual(self._eval("debp[400A%]"), 500) self.assertEqual(self._eval("bal_700IN"), -500) self.assertEqual(self._eval("bals[700IN]"), -800) + # check fldp.quantity + self.assertEqual(self._eval("fldp.quantity[700%]"), 0) # unallocated p&l from previous year self.assertEqual(self._eval("balu[]"), -100) diff --git a/mis_builder/views/mis_report.xml b/mis_builder/views/mis_report.xml index d7c93c5f9..c0c9652ac 100644 --- a/mis_builder/views/mis_report.xml +++ b/mis_builder/views/mis_report.xml @@ -166,19 +166,24 @@

The following special elements are recognized in the expressions to compute accounting data: {bal|crd|deb|pbal|nbal}{pieu}[account + >{bal|crd|deb|pbal|nbal|fld}{pieu}(.fieldname)[account selector][journal items domain].

  • bal, crd, deb, - pbal, nbal : balance, debit, credit, - positive balance, negative balance.
    • + pbal, nbal, fld : balance, debit, credit, + positive balance, negative balance, + other numerical field.
  • p, i, e : respectively variation over the period, initial balance, ending balance
    • +
  • when fld is used : a field name specifier + must be provided (e.g. fldp.quantity
  • The account selector is a like expression on the account code (eg