Sources & historical data

curl --fail-with-body --max-time 30 --get   'https://api.filingsift.com/api/v1/reference/bis/history'   --data-urlencode 'dataset=bis'   --data-urlencode 'startDate=2020-01-01'   --data-urlencode 'endDate=2020-03-31'
# This is one page. Use the Python or Node.js example to follow nextCursor.

"""Python 3.10+. Read-only; at most ten requests; no third-party dependencies.
Usage: python3 reference_history.py [dataset] [startDate] [endDate] [entity]
"""
import json
import os
import sys
from urllib.error import HTTPError
from urllib.parse import urlencode
from urllib.request import HTTPRedirectHandler, Request, build_opener


class NoRedirect(HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        return None  # Never forward an API key to a redirect destination.


def main():
    args = sys.argv[1:]
    dataset = args[0] if args else "bis"
    filters = {
        "dataset": dataset,
        "startDate": args[1] if len(args) > 1 else "2020-01-01",
        "endDate": args[2] if len(args) > 2 else "2020-03-31",
    }
    if len(args) > 3:
        filters["entity"] = args[3]
    public_bis = dataset == "bis"
    base = os.environ.get("FILINGSIFT_API_URL", "https://api.filingsift.com/api/v1").rstrip("/")
    headers = {"Accept": "application/json"}
    if not public_bis:
        key = os.environ.get("FILINGSIFT_API_KEY")
        if not key:
            raise ValueError("Set FILINGSIFT_API_KEY with intelligence:read access.")
        headers["Authorization"] = "Bearer " + key
    path = "/reference/bis/history" if public_bis else "/account/reference-history"
    opener = build_opener(NoRedirect())
    pages, seen, cursor = [], set(), None
    for _ in range(10):
        query = dict(filters)
        if cursor is not None:
            query["cursor"] = cursor
        try:
            with opener.open(Request(base + path + "?" + urlencode(query), headers=headers), timeout=30) as response:
                envelope = json.load(response)
        except HTTPError as error:
            if error.code == 409:
                raise ValueError("Source snapshots changed: discard this run and restart without a cursor.") from error
            raise ValueError(f"HTTP {error.code}: check access, dates and coverage; for 429 honor Retry-After before another run.") from error
        data = envelope.get("data", {})
        pagination = data.get("pagination", {})
        if (data.get("schemaVersion") != "filingsift-reference-library-v1"
                or not isinstance(data.get("records"), list)
                or not isinstance(data.get("receipts"), list)
                or "nextCursor" not in pagination
                or not (pagination["nextCursor"] is None or isinstance(pagination["nextCursor"], str))):
            raise ValueError("Unexpected history response.")
        pages.append(envelope)  # Preserve exact decimal strings, nulls, units and receipts.
        cursor = pagination["nextCursor"]
        if cursor is None:
            break
        if not cursor or cursor in seen:
            raise ValueError("Repeated or empty cursor; stop and inspect the response.")
        seen.add(cursor)
    print(json.dumps({"request": filters, "complete": cursor is None, "nextCursor": cursor, "pages": pages}, indent=2))
    if cursor is not None:
        print("Stopped at ten pages. Output is partial; resume with the same filters and nextCursor, or narrow the dates/entity.", file=sys.stderr)


if __name__ == "__main__":
    try:
        main()
    except Exception as error:
        # Nothing is printed to stdout until the bounded run succeeds.
        print(str(error), file=sys.stderr)
        sys.exit(1)

// Node.js 20+. Read-only; at most ten requests. Preserve each page and its receipts.
// Usage: node reference-history.mjs [dataset] [startDate] [endDate] [entity]
const [dataset = 'bis', startDate = '2020-01-01', endDate = '2020-03-31', entity] = process.argv.slice(2);
const publicBis = dataset === 'bis';
const base = (process.env.FILINGSIFT_API_URL || 'https://api.filingsift.com/api/v1').replace(/\/$/, '');
const headers = {Accept: 'application/json'};
if (!publicBis) {
  if (!process.env.FILINGSIFT_API_KEY) throw new Error('Set FILINGSIFT_API_KEY with intelligence:read access.');
  headers.Authorization = `Bearer ${process.env.FILINGSIFT_API_KEY}`;
}
const filters = {dataset, startDate, endDate, ...(entity ? {entity} : {})};
const pages = [], seen = new Set();
let cursor = null;
try {
  for (let page = 0; page < 10; page++) {
    const query = new URLSearchParams({...filters, ...(cursor ? {cursor} : {})});
    const response = await fetch(`${base}${publicBis ? '/reference/bis/history' : '/account/reference-history'}?${query}`, {
      headers, redirect: 'error', signal: AbortSignal.timeout(30_000),
    });
    if (response.status === 409) throw new Error('Source snapshots changed: discard this run and restart without a cursor.');
    if (!response.ok) throw new Error(`HTTP ${response.status}: check access, dates and coverage; for 429 honor Retry-After before another run.`);
    const envelope = await response.json();
    const data = envelope.data;
    if (data?.schemaVersion !== 'filingsift-reference-library-v1' || !Array.isArray(data.records)
      || !Array.isArray(data.receipts) || !data.pagination
      || !(data.pagination.nextCursor === null || typeof data.pagination.nextCursor === 'string')) throw new Error('Unexpected history response.');
    pages.push(envelope); // Keep exact decimal strings, nulls, units, coverage and receipts together.
    cursor = data.pagination.nextCursor;
    if (cursor === null) break;
    if (!cursor || seen.has(cursor)) throw new Error('Repeated or empty cursor; stop and inspect the response.');
    seen.add(cursor);
  }
  console.log(JSON.stringify({request: filters, complete: cursor === null, nextCursor: cursor, pages}, null, 2));
  if (cursor !== null) console.error('Stopped at ten pages. Output is partial; resume with the same filters and nextCursor, or narrow the dates/entity.');
} catch (error) {
  // No partial pages are written on failure, including a mid-query snapshot change.
  console.error(error.message);
  process.exitCode = 1;
}