audit_report: New tool.
This commit is contained in:
Brett Smith
parent
0e52f11a58
commit
7281cf0f01
2 changed files with 273 additions and 1 deletions
271
conservancy_beancount/tools/audit_report.py
Normal file
271
conservancy_beancount/tools/audit_report.py
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
"""audit_report.py - Utility to run all reports for an audit"""
|
||||
# Copyright © 2020 Brett Smith
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import runpy
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from typing import (
|
||||
Callable,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Sequence,
|
||||
Set,
|
||||
TextIO,
|
||||
Tuple,
|
||||
)
|
||||
from types import (
|
||||
ModuleType,
|
||||
)
|
||||
|
||||
from . import extract_odf_links
|
||||
from .. import cliutil
|
||||
from .. import config as configmod
|
||||
from ..reports import accrual
|
||||
from ..reports import balance_sheet
|
||||
from ..reports import fund
|
||||
from ..reports import ledger
|
||||
|
||||
from beancount.scripts import check as bc_check
|
||||
|
||||
ArgList = List[str]
|
||||
ReportFunc = Callable[[ArgList, TextIO, TextIO, configmod.Config], int]
|
||||
|
||||
CPU_COUNT = len(os.sched_getaffinity(0))
|
||||
PROGNAME = 'audit-report'
|
||||
logger = logging.getLogger('conservancy_beancount.tools.audit_report')
|
||||
|
||||
def jobs_arg(arg: str) -> int:
|
||||
if arg.endswith('%'):
|
||||
arg_n = round(CPU_COUNT * 100 / int(arg[:-1]))
|
||||
else:
|
||||
arg_n = int(arg)
|
||||
return max(1, arg_n)
|
||||
|
||||
def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(prog=PROGNAME)
|
||||
cliutil.add_version_argument(parser)
|
||||
cliutil.add_loglevel_argument(parser)
|
||||
parser.add_argument(
|
||||
'--verbose', '-v',
|
||||
action='store_true',
|
||||
help="""Display progress information
|
||||
""")
|
||||
parser.add_argument(
|
||||
'--jobs', '-j',
|
||||
metavar='NUM',
|
||||
type=jobs_arg,
|
||||
default=CPU_COUNT,
|
||||
help="""Maximum number of processes to run concurrently.
|
||||
Can specify a positive integer or a percentage of CPU cores. Default all cores.
|
||||
""")
|
||||
parser.add_argument(
|
||||
'--output-directory', '-O',
|
||||
metavar='DIR',
|
||||
type=Path,
|
||||
help="""Write all reports to this directory.
|
||||
Default is a newly-created directory under your repository.
|
||||
""")
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
action='store_true',
|
||||
help="""Run reports even if bean-check reports errors.
|
||||
""")
|
||||
parser.add_argument(
|
||||
'--rewrite-rules', '--rewrite', '-r',
|
||||
metavar='PATH',
|
||||
action='append',
|
||||
type=Path,
|
||||
default=[],
|
||||
help="""Path to rewrite rules for the balance sheet.
|
||||
Passed to `balance-sheet-report -r`.
|
||||
""")
|
||||
parser.add_argument(
|
||||
'--delimiter', '-d',
|
||||
metavar='TEXT',
|
||||
default='\\0',
|
||||
help="""Delimiter for ODF links in the manifest file.
|
||||
Passed to `extract-odf-links --delimiter`. Default `%(default)s`.
|
||||
""")
|
||||
parser.add_argument(
|
||||
'audit_year',
|
||||
metavar='YEAR',
|
||||
nargs='?',
|
||||
type=cliutil.year_or_date_arg,
|
||||
help="""Main fiscal year to generate reports for.
|
||||
Defaults to the last complete fiscal year.
|
||||
""")
|
||||
parser.add_argument(
|
||||
'end_date',
|
||||
metavar='END',
|
||||
nargs='?',
|
||||
type=cliutil.date_arg,
|
||||
help="""End date for reports for the following fiscal year.
|
||||
The default is automatically calculated from today's date.
|
||||
""")
|
||||
args = parser.parse_args(arglist)
|
||||
args.arg_error = parser.error
|
||||
return args
|
||||
|
||||
def now_s() -> str:
|
||||
return datetime.datetime.now().isoformat(sep=' ', timespec='seconds')
|
||||
|
||||
def bean_check(books_path: Path) -> int:
|
||||
sys.argv = ['bean-check', str(books_path)]
|
||||
logger.debug("running %r", sys.argv)
|
||||
# bean-check logs timing information to the root logger at INFO level.
|
||||
# Suppress that.
|
||||
logging.getLogger().setLevel(logging.WARNING)
|
||||
return bc_check.main() # type:ignore[no-any-return]
|
||||
|
||||
def main(arglist: Optional[Sequence[str]]=None,
|
||||
stdout: TextIO=sys.stdout,
|
||||
stderr: TextIO=sys.stderr,
|
||||
config: Optional[configmod.Config]=None,
|
||||
) -> int:
|
||||
args = parse_arguments(arglist)
|
||||
if config is None:
|
||||
config = configmod.Config()
|
||||
config.load_file()
|
||||
cliutil.set_loglevel(logger, args.loglevel)
|
||||
if args.verbose:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logging.getLogger('conservancy_beancount.reports').setLevel(logging.DEBUG)
|
||||
|
||||
fy = config.fiscal_year_begin()
|
||||
today = datetime.date.today()
|
||||
if args.audit_year is None:
|
||||
args.audit_year = fy.for_date(today) - 1
|
||||
audit_begin = fy.first_date(args.audit_year)
|
||||
audit_end = fy.next_fy_date(args.audit_year)
|
||||
if args.end_date is None:
|
||||
days_diff = (today - audit_end).days
|
||||
if days_diff < (28 * 2):
|
||||
args.end_date = today
|
||||
elif days_diff >= 365:
|
||||
args.end_date = fy.next_fy_date(args.audit_year + 1)
|
||||
else:
|
||||
end_date = today - datetime.timedelta(days=today.day + 1)
|
||||
args.end_date = end_date.replace(day=1)
|
||||
if args.end_date < audit_end:
|
||||
args.arg_error("end date is within audited fiscal year")
|
||||
next_year = fy.for_date(args.end_date)
|
||||
repo_path = config.repository_path()
|
||||
|
||||
if args.output_directory is None:
|
||||
args.output_directory = Path(tempfile.mkdtemp(
|
||||
prefix=f'FY{args.audit_year}AuditReports.', dir=repo_path,
|
||||
))
|
||||
logger.info("writing reports to %s", args.output_directory)
|
||||
else:
|
||||
args.output_directory.mkdir(exist_ok=True)
|
||||
output_reports: List[Path] = []
|
||||
def common_args(out_name: str, year: Optional[int]=None, *arglist: str) -> Iterator[str]:
|
||||
if year is not None:
|
||||
out_name = f'FY{year}{out_name}.ods'
|
||||
if year == args.audit_year:
|
||||
yield f'--begin={audit_begin.isoformat()}'
|
||||
yield f'--end={audit_end.isoformat()}'
|
||||
elif year == next_year:
|
||||
yield f'--begin={audit_end.isoformat()}'
|
||||
yield f'--end={args.end_date.isoformat()}'
|
||||
elif year is not None:
|
||||
raise ValueError(f"unknown year {year!r}")
|
||||
out_path = args.output_directory / out_name
|
||||
output_reports.append(out_path)
|
||||
yield f'--output-file={out_path}'
|
||||
yield from arglist
|
||||
reports: List[Tuple[ReportFunc, ArgList]] = [
|
||||
# Reports are sorted roughly in descending order of how long each takes
|
||||
# to generate.
|
||||
(ledger.main, list(common_args('GeneralLedger', args.audit_year))),
|
||||
(ledger.main, list(common_args('GeneralLedger', next_year))),
|
||||
(ledger.main, list(common_args('Disbursements', args.audit_year, '--disbursements'))),
|
||||
(ledger.main, list(common_args('Receipts', args.audit_year, '--receipts'))),
|
||||
(ledger.main, list(common_args('Disbursements', next_year, '--disbursements'))),
|
||||
(ledger.main, list(common_args('Receipts', next_year, '--receipts'))),
|
||||
(accrual.main, list(common_args('AgingReport.ods'))),
|
||||
(balance_sheet.main, list(common_args(
|
||||
'Summary', args.audit_year,
|
||||
*(f'--rewrite-rules={path}' for path in args.rewrite_rules),
|
||||
))),
|
||||
(fund.main, list(common_args('FundReport', args.audit_year))),
|
||||
(fund.main, list(common_args('FundReport', next_year))),
|
||||
]
|
||||
|
||||
returncode = 0
|
||||
books = config.books_loader()
|
||||
if books is None:
|
||||
logger.critical("no books available to load")
|
||||
return os.EX_NOINPUT
|
||||
|
||||
with multiprocessing.Pool(args.jobs, maxtasksperchild=1) as pool:
|
||||
logger.debug("%s: process pool ready with %s workers", now_s(), args.jobs)
|
||||
fy_paths = books._iter_fy_books(fy.range(args.audit_year - 1, args.end_date))
|
||||
check_results = pool.imap_unordered(bean_check, fy_paths)
|
||||
if all(exitcode == 0 for exitcode in check_results):
|
||||
logger.debug("%s: bean-check passed", now_s())
|
||||
else:
|
||||
logger.log(
|
||||
logging.WARNING if args.force else logging.ERROR,
|
||||
"%s: bean-check failed",
|
||||
now_s(),
|
||||
)
|
||||
if not args.force:
|
||||
return os.EX_DATAERR
|
||||
|
||||
report_results = [
|
||||
pool.apply_async(report_func, (arglist,), {'config': config})
|
||||
for report_func, arglist in reports
|
||||
]
|
||||
report_errors = [res.get() for res in report_results if res.get() != 0]
|
||||
if not report_errors:
|
||||
logger.debug("%s: all reports generated", now_s())
|
||||
else:
|
||||
logger.error("%s: %s reports generated errors", now_s(), len(report_errors))
|
||||
if not args.force:
|
||||
return max(report_errors)
|
||||
|
||||
missing_reports = frozenset(
|
||||
path for path in output_reports if not path.exists()
|
||||
)
|
||||
if missing_reports:
|
||||
logger.error("missing expected reports: %s",
|
||||
', '.join(path.name for path in missing_reports))
|
||||
if not args.force or len(missing_reports) == len(output_reports):
|
||||
return os.EX_UNAVAILABLE
|
||||
|
||||
arglist = [f'--delimiter={args.delimiter}']
|
||||
if repo_path is not None:
|
||||
arglist.append(f'--relative-to={repo_path}')
|
||||
arglist.extend(str(p) for p in output_reports if p not in missing_reports)
|
||||
with (args.output_directory / 'MANIFEST.AUTO').open('w') as manifest_file:
|
||||
returncode = extract_odf_links.main(arglist, manifest_file, stderr)
|
||||
logger.debug("%s: manifest generated", now_s())
|
||||
return returncode
|
||||
|
||||
entry_point = cliutil.make_entry_point(__name__, PROGNAME)
|
||||
|
||||
if __name__ == '__main__':
|
||||
exit(entry_point())
|
||||
3
setup.py
3
setup.py
|
|
@ -5,7 +5,7 @@ from setuptools import setup
|
|||
setup(
|
||||
name='conservancy_beancount',
|
||||
description="Plugin, library, and reports for reading Conservancy's books",
|
||||
version='1.8.8',
|
||||
version='1.9.0',
|
||||
author='Software Freedom Conservancy',
|
||||
author_email='info@sfconservancy.org',
|
||||
license='GNU AGPLv3+',
|
||||
|
|
@ -37,6 +37,7 @@ setup(
|
|||
entry_points={
|
||||
'console_scripts': [
|
||||
'accrual-report = conservancy_beancount.reports.accrual:entry_point',
|
||||
'assemble-audit-reports = conservancy_beancount.tools.audit_report:entry_point',
|
||||
'balance-sheet-report = conservancy_beancount.reports.balance_sheet:entry_point',
|
||||
'extract-odf-links = conservancy_beancount.tools.extract_odf_links:entry_point',
|
||||
'fund-report = conservancy_beancount.reports.fund:entry_point',
|
||||
|
|
|
|||
Loading…
Reference in a new issue