paypal-reports/paypal_report.py

#!/usr/bin/python3
"""Download subscribers or subscriber transactions from PayPal.

Used to help Conservancy keep track of Sustainers.

Prints out either a list of recurring payment transactions or a list of unique
recurring payment subscriber profiles. This recreates the features available in
the previous PayPal SOAP API where you could query for all ProfileIDs
(identifying unique subscribers).

Run it like this:

$ export PAYPAL_CLIENT_ID=XXX
$ export PAYPAL_CLIENT_SECRET=YYY
$ python3 paypal_report.py profiles 2021-11-01T00:00:00-07:00 2021-11-30T23:59:59-07:00 > profiles.txt
$ python3 paypal_report.py transactions 2021-11-01T00:00:00-07:00 2021-11-30T23:59:59-07:00 > transactions.txt

NOTE: Newly created PayPal "apps"/credentials initially authenticate, but can
then return PERMISSION_DENIED for ~ 40 mins.

The PayPal OAuth 2.0 client ID and secret can be obtained from the Developer
Dashboard as described here: https://developer.paypal.com/api/rest/.

This tool isn't directly related Beancount, but it may duplicate some
functionality from the PayPal importer and paypal_rest tools, which I wasn't
aware of at the time. See:

 - conservancy_beancount/doc/PayPalQuery.md
 - NPO-Accounting/paypal_rest
"""
import argparse
import collections
import datetime
import sys
import os

import requests


def main():
    args = parse_args()
    client_id, client_secret = load_paypal_keys()
    access_token = get_paypal_access_token(client_id, client_secret)
    transactions = get_all_transactions(access_token, args.start_date, args.end_date)
    if args.report == 'profiles':
        report_on_unique_profiles(transactions)
    elif args.report == 'transactions':
        report_on_transactions(transactions)
    else:
        suspend_subscriptions_interactively(transactions, args.pattern, access_token)


def parse_args():
    parser = argparse.ArgumentParser(description='Download PayPal subscriber info.')
    parser.add_argument('report', help='report to run"', choices=['profiles', 'transactions', 'suspend'])
    parser.add_argument('start_date', help='start date inclusive (eg. 2021-11-01T00:00:00-07:00)', type=parse_iso_time)
    parser.add_argument('end_date', help='end date inclusive (eg. 2021-11-30T23:59:59-07:00)', type=parse_iso_time)
    parser.add_argument('pattern')
    return parser.parse_args()


def parse_iso_time(time_str):
    try:
        time = datetime.datetime.fromisoformat(time_str)
    except ValueError:
        raise argparse.ArgumentTypeError('Date did not match ISO format eg. 2021-11-01T00:00:00-07:00')
    return time


def load_paypal_keys():
    try:
        client_id = os.environ['PAYPAL_CLIENT_ID']
        client_secret = os.environ['PAYPAL_CLIENT_SECRET']
    except KeyError:
        sys.exit('Please set PAYPAL_CLIENT_ID and PAYPAL_CLIENT_SECRET environment variables.')
    return client_id, client_secret


def get_paypal_access_token(client_id, client_secret):
    url = 'https://api-m.paypal.com/v1/oauth2/token?grant_type=client_credentials'
    response = requests.post(
        url,
        auth=(client_id, client_secret),
        headers={
            'Accept': 'application/json',
            'Accept-Language': 'en_US',
        },
    )
    data = response.json()
    return data['access_token']


def get_all_transactions(access_token, start_date, end_date):
    periods = get_time_periods(start_date, end_date)
    transactions = []
    for start, end in periods:
        print(f'Period {start} - {end}.', file=sys.stderr)
        transactions += get_transactions_for_period(access_token, start, end)
    return transactions


def get_time_periods(start_date, end_date):
    """Break a time period into smaller periods of up to 31 days.

    PayPal only supports querying for up to 31 days of transactions at a time.
    """
    periods = []
    while start_date < end_date:
        periods.append((start_date, start_date + datetime.timedelta(days=31)))
        start_date += datetime.timedelta(days=31)
    return periods


def get_transactions_for_period(access_token, start_date, end_date, page=1):
    """Downloads all pages of transactions for a time period of 31 days or less."""
    print(f'Fetching transactions page {page}.', file=sys.stderr)
    url = f'https://api-m.paypal.com/v1/reporting/transactions?start_date={start_date.isoformat()}&end_date={end_date.isoformat()}&fields=all&page_size=500&page={page}'
    response = requests.get(
        url,
        headers={
            'Content-Type': 'application/json',
            'Authorization': f'Bearer {access_token}',
        },
    )
    data = response.json()
    if page < data['total_pages']:
        return data['transaction_details'] + get_transactions_for_period(access_token, start_date, end_date, page + 1)
    else:
        return data['transaction_details']


def report_on_unique_profiles(transactions):
    """Print a list of subscribers from a set of transactions.

    PayPal doesn't provide a way to query for subscribers directly, so we build
    this by scanning through the list of transactions and finding the unique
    subscribers.
    """
    records = set()
    for t in transactions:
        transaction_info = t['transaction_info']
        if 'paypal_reference_id' in transaction_info:
            records.add(
                (
                    transaction_info['paypal_reference_id'],
                    transaction_info.get('transaction_subject', 'NO TRANSACTION SUBJECT'),
                ),
            )
        else:
            print(f'Skipping transaction {transaction_info["transaction_id"]} with no PayPal Reference ID.', file=sys.stderr)

    for id, desc in sorted(records):
        print(f'{id}   {desc}')
    count = collections.Counter([id for id, _ in records])
    dupes = [id for id, total in count.items() if total > 1]
    print(f'Reference IDs with changing subject: {dupes}', file=sys.stderr)


def get_subscriptions_matching_subject(transactions, pattern):
    records = set()
    for t in transactions:
        transaction_info = t['transaction_info']
        if 'paypal_reference_id' in transaction_info and pattern.lower() in transaction_info.get('transaction_subject', 'CANT MATCH ME').lower():
            records.add(
                (
                    transaction_info['paypal_reference_id'],
                    transaction_info.get('transaction_subject', 'NO TRANSACTION SUBJECT'),
                ),
            )
        else:
            # print(f'Skipping transaction {transaction_info["transaction_id"]} with no PayPal Reference ID.', file=sys.stderr)
            pass
    return records


def report_on_transactions(transactions):
    """Print a formatted list of transactions."""
    for t in transactions:
        transaction_info = t['transaction_info']
        if 'paypal_reference_id' in transaction_info:
            print("%-19s %s" % (transaction_info['transaction_id'], transaction_info['paypal_reference_id']))
            print("%-19s %s" % (transaction_info['paypal_reference_id'], transaction_info['paypal_reference_id']))
        else:
            print(f'Skipping transaction {transaction_info["transaction_id"]} with no PayPal Reference ID.', file=sys.stderr)


def suspend_subscriptions_interactively(transactions, pattern, access_token):
    subscriptions = get_subscriptions_matching_subject(transactions, pattern)
    for (id, subject) in subscriptions:
        sub_before = paypal_request(f'https://api.paypal.com/v1/billing/subscriptions/{id}', access_token)
        status_before = sub_before['status']

        # Donor name / total donations / last donation / donation amount / FULL text of the profile Memo description / profile-id " with "Should I cancel this one? (y/n)"
        given_name = sub_before['subscriber'].get('name', {}).get('given_name', '')
        surname = sub_before['subscriber'].get('name', {}).get('surname', '')
        name = ' '.join(filter(bool, [given_name, surname]))
        payments = sub_before['billing_info']['cycle_executions'][0]['cycles_completed']
        last_payment = sub_before['billing_info']['last_payment']
        last_payment_total = last_payment['amount']['value']
        last_payment_currency = last_payment['amount']['currency_code']
        last_payment_time = last_payment['time']
        print(f'Subscriber {id} {subject} {status_before}: {name}, {payments} payments with last payment {last_payment_time}, {last_payment_total} {last_payment_currency}')
        if status_before == 'ACTIVE':
            url = f'https://api.paypal.com/v1/billing/subscriptions/{id}/suspend'
            if input('Cancel this one? (y/n) ') == 'y':
                response = paypal_post_request(url, access_token)
                print(f'PayPal said: {response.text}')
                sub_after = paypal_request(f'https://api.paypal.com/v1/billing/subscriptions/{id}', access_token)
                status_after = sub_after['status']
                print(f'Subscriber {id} status after: {status_after}')
        else:
            print('  ignoring inactive status')


def paypal_request(url, access_token):
    response = requests.get(
        url,
        headers={
            'Content-Type': 'application/json',
            'Authorization': f'Bearer {access_token}',
        },
    )
    data = response.json()
    return data


def paypal_post_request(url, access_token):
    response = requests.post(
        url,
        headers={
            'Content-Type': 'application/json',
            'Authorization': f'Bearer {access_token}',
        },
        json={
            'reason': 'suspended',
        },
    )
    return response


if __name__ == '__main__':
    main()